Skip to content

Commit f60c9a7

Browse files
sokrasokra
authored
Merge pull request webpack#8272 from webpack/feature/cache-pack
add pack store mode for filesystem cache
2 parents c0d5387 + 044ad37 commit f60c9a7

11 files changed

+357
-77
lines changed

‎declarations/WebpackOptions.d.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -428,17 +428,17 @@ export interface FileCacheOptions {
428428
*/
429429
hashAlgorithm?: string;
430430
/**
431-
* Display log info. (debug: all access and errors with stack trace, info: all access, warning: only failed serialization)
431+
* Display log info. (debug: all access and errors with stack trace, verbose: all access, info: all write access, warning: only failed serialization)
432432
*/
433-
loglevel?: "debug" | "info" | "warning";
433+
loglevel?: "debug" | "verbose" | "info" | "warning";
434434
/**
435435
* Name for the cache. Different names will lead to different coexisting caches.
436436
*/
437437
name?: string;
438438
/**
439-
* When to store data to the filesystem. (idle: Store data when compiler is idle; background: Store data in background while compiling, but doesn't block the compilation; instant: Store data when creating blocking compilation until data is stored; defaults to idle)
439+
* When to store data to the filesystem. (pack: Store data when compiler is idle in a single file, idle: Store data when compiler is idle in multiple files; background: Store data in background while compiling, but doesn't block the compilation; instant: Store data when creating blocking compilation until data is stored; defaults to idle)
440440
*/
441-
store?: "idle" | "background" | "instant";
441+
store?: "pack" | "idle" | "background" | "instant";
442442
/**
443443
* Filesystem caching
444444
*/

‎lib/cache/FileCachePlugin.js

Lines changed: 232 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,87 @@
55

66
"use strict";
77

8+
const fs = require("fs");
89
const path = require("path");
910
const createHash = require("../util/createHash");
11+
const makeSerializable = require("../util/makeSerializable");
1012
const serializer = require("../util/serializer");
1113

1214
/** @typedef {import("webpack-sources").Source} Source */
1315
/** @typedef {import("../../declarations/WebpackOptions").FileCacheOptions} FileCacheOptions */
1416
/** @typedef {import("../Compiler")} Compiler */
1517
/** @typedef {import("../Module")} Module */
1618

19+
class Pack {
20+
constructor(version) {
21+
this.version = version;
22+
this.content = new Map();
23+
this.lastAccess = new Map();
24+
this.used = new Set();
25+
this.invalid = false;
26+
}
27+
28+
get(relativeFilename) {
29+
this.used.add(relativeFilename);
30+
return this.content.get(relativeFilename);
31+
}
32+
33+
set(relativeFilename, data) {
34+
this.used.add(relativeFilename);
35+
this.invalid = true;
36+
return this.content.set(relativeFilename, data);
37+
}
38+
39+
collectGarbage(maxAge) {
40+
this._updateLastAccess();
41+
const now = Date.now();
42+
for (const [relativeFilename, lastAccess] of this.lastAccess) {
43+
if (now - lastAccess > maxAge) {
44+
this.lastAccess.delete(relativeFilename);
45+
this.content.delete(relativeFilename);
46+
}
47+
}
48+
}
49+
50+
_updateLastAccess() {
51+
const now = Date.now();
52+
for (const relativeFilename of this.used) {
53+
this.lastAccess.set(relativeFilename, now);
54+
}
55+
this.used.clear();
56+
}
57+
58+
serialize({ write, snapshot, rollback }) {
59+
this._updateLastAccess();
60+
write(this.version);
61+
for (const [relativeFilename, data] of this.content) {
62+
const s = snapshot();
63+
try {
64+
write(relativeFilename);
65+
write(data);
66+
} catch (err) {
67+
rollback(s);
68+
continue;
69+
}
70+
}
71+
write(null);
72+
write(this.lastAccess);
73+
}
74+
75+
deserialize({ read }) {
76+
this.version = read();
77+
this.content = new Map();
78+
let relativeFilename = read();
79+
while (relativeFilename !== null) {
80+
this.content.set(relativeFilename, read());
81+
relativeFilename = read();
82+
}
83+
this.lastAccess = read();
84+
}
85+
}
86+
87+
makeSerializable(Pack, "webpack/lib/cache/FileCachePlugin", "Pack");
88+
1789
const memorize = fn => {
1890
let result = undefined;
1991
return () => {
@@ -48,9 +120,9 @@ class FileCachePlugin {
48120
const hashAlgorithm = this.options.hashAlgorithm || "md4";
49121
const version = this.options.version || "";
50122
const log = this.options.loglevel
51-
? { debug: 3, info: 2, warning: 1 }[this.options.loglevel]
123+
? { debug: 4, verbose: 3, info: 2, warning: 1 }[this.options.loglevel]
52124
: 0;
53-
const store = this.options.store || "idle";
125+
const store = this.options.store || "pack";
54126

55127
let pendingPromiseFactories = new Map();
56128
const toHash = str => {
@@ -59,33 +131,82 @@ class FileCachePlugin {
59131
const digest = hash.digest("hex");
60132
return `${digest.slice(0, 2)}/${digest.slice(2)}`;
61133
};
62-
compiler.cache.hooks.store.tapPromise(
63-
"FileCachePlugin",
64-
(identifier, etag, data) => {
65-
const entry = { identifier, data: () => data, etag, version };
66-
const filename = path.join(
67-
cacheDirectory,
68-
toHash(identifier) + ".data"
69-
);
70-
memoryCache.set(filename, entry);
71-
const promiseFactory = () =>
72-
serializer
73-
.serializeToFile(entry, filename)
74-
.then(() => {
75-
if (log >= 2) {
76-
console.warn(`Cached ${identifier} to ${filename}.`);
134+
let packPromise;
135+
if (store === "pack") {
136+
packPromise = serializer
137+
.deserializeFromFile(`${cacheDirectory}.pack`)
138+
.then(cacheEntry => {
139+
if (cacheEntry) {
140+
if (!(cacheEntry instanceof Pack)) {
141+
if (log >= 3) {
142+
console.warn(
143+
`Restored pack from ${cacheDirectory}.pack, but is not a Pack.`
144+
);
77145
}
78-
})
79-
.catch(err => {
80-
if (log >= 1) {
146+
return new Pack(version);
147+
}
148+
if (cacheEntry.version !== version) {
149+
if (log >= 3) {
81150
console.warn(
82-
`Caching failed for ${identifier}: ${
83-
log >= 3 ? err.stack : err
84-
}`
151+
`Restored pack from ${cacheDirectory}.pack, but version doesn't match.`
85152
);
86153
}
87-
});
88-
if (store === "instant") {
154+
return new Pack(version);
155+
}
156+
return cacheEntry;
157+
}
158+
return new Pack(version);
159+
})
160+
.catch(err => {
161+
if (log >= 1 && err && err.code !== "ENOENT") {
162+
console.warn(
163+
`Restoring pack failed from ${cacheDirectory}.pack: ${
164+
log >= 4 ? err.stack : err
165+
}`
166+
);
167+
}
168+
return new Pack(version);
169+
});
170+
}
171+
compiler.cache.hooks.store.tapPromise(
172+
"FileCachePlugin",
173+
(identifier, etag, data) => {
174+
const entry = {
175+
identifier,
176+
data: etag ? () => data : data,
177+
etag,
178+
version
179+
};
180+
const relativeFilename = toHash(identifier) + ".data";
181+
const filename = path.join(cacheDirectory, relativeFilename);
182+
memoryCache.set(filename, entry);
183+
const promiseFactory =
184+
store === "pack"
185+
? () =>
186+
packPromise.then(pack => {
187+
if (log >= 2) {
188+
console.warn(`Cached ${identifier} to pack.`);
189+
}
190+
pack.set(relativeFilename, entry);
191+
})
192+
: () =>
193+
serializer
194+
.serializeToFile(entry, filename)
195+
.then(() => {
196+
if (log >= 2) {
197+
console.warn(`Cached ${identifier} to ${filename}.`);
198+
}
199+
})
200+
.catch(err => {
201+
if (log >= 1) {
202+
console.warn(
203+
`Caching failed for ${identifier}: ${
204+
log >= 3 ? err.stack : err
205+
}`
206+
);
207+
}
208+
});
209+
if (store === "instant" || store === "pack") {
89210
return promiseFactory();
90211
} else if (store === "idle") {
91212
pendingPromiseFactories.set(filename, promiseFactory);
@@ -100,77 +221,133 @@ class FileCachePlugin {
100221
compiler.cache.hooks.get.tapPromise(
101222
"FileCachePlugin",
102223
(identifier, etag) => {
103-
const filename = path.join(
104-
cacheDirectory,
105-
toHash(identifier) + ".data"
106-
);
224+
const relativeFilename = toHash(identifier) + ".data";
225+
const filename = path.join(cacheDirectory, relativeFilename);
226+
const logMessage = store === "pack" ? "pack" : filename;
107227
const memory = memoryCache.get(filename);
108228
if (memory !== undefined) {
109229
return Promise.resolve(
110-
memory.etag === etag && memory.version === version
111-
? memory.data()
112-
: undefined
230+
memory.etag !== etag || memory.version !== version
231+
? undefined
232+
: typeof memory.data === "function"
233+
? memory.data()
234+
: memory.data
113235
);
114236
}
115-
return serializer.deserializeFromFile(filename).then(
237+
const cacheEntryPromise =
238+
store === "pack"
239+
? packPromise.then(pack => pack.get(relativeFilename))
240+
: serializer.deserializeFromFile(filename);
241+
return cacheEntryPromise.then(
116242
cacheEntry => {
117-
cacheEntry = {
118-
identifier: cacheEntry.identifier,
119-
etag: cacheEntry.etag,
120-
version: cacheEntry.version,
121-
data: memorize(cacheEntry.data)
122-
};
243+
if (cacheEntry === undefined) return;
244+
if (typeof cacheEntry.data === "function")
245+
cacheEntry.data = memorize(cacheEntry.data);
123246
memoryCache.set(filename, cacheEntry);
124247
if (cacheEntry === undefined) return;
125248
if (cacheEntry.identifier !== identifier) {
126-
if (log >= 2) {
249+
if (log >= 3) {
127250
console.warn(
128-
`Restored ${identifier} from ${filename}, but identifier doesn't match.`
251+
`Restored ${identifier} from ${logMessage}, but identifier doesn't match.`
129252
);
130253
}
131254
return;
132255
}
133256
if (cacheEntry.etag !== etag) {
134-
if (log >= 2) {
257+
if (log >= 3) {
135258
console.warn(
136-
`Restored ${etag} from ${filename}, but etag doesn't match.`
259+
`Restored ${identifier} from ${logMessage}, but etag doesn't match.`
137260
);
138261
}
139262
return;
140263
}
141264
if (cacheEntry.version !== version) {
142-
if (log >= 2) {
265+
if (log >= 3) {
143266
console.warn(
144-
`Restored ${version} from ${filename}, but version doesn't match.`
267+
`Restored ${identifier} from ${logMessage}, but version doesn't match.`
145268
);
146269
}
147270
return;
148271
}
149-
if (log >= 2) {
150-
console.warn(`Restored ${identifier} from ${filename}.`);
272+
if (log >= 3) {
273+
console.warn(`Restored ${identifier} from ${logMessage}.`);
151274
}
152-
return cacheEntry.data();
275+
if (typeof cacheEntry.data === "function") return cacheEntry.data();
276+
return cacheEntry.data;
153277
},
154278
err => {
155279
if (log >= 1 && err && err.code !== "ENOENT") {
156280
console.warn(
157-
`Restoring failed for ${identifier} from ${filename}: ${
158-
log >= 3 ? err.stack : err
281+
`Restoring failed for ${identifier} from ${logMessage}: ${
282+
log >= 4 ? err.stack : err
159283
}`
160284
);
161285
}
162286
}
163287
);
164288
}
165289
);
290+
const serializePack = () => {
291+
packPromise = packPromise.then(pack => {
292+
if (!pack.invalid) return pack;
293+
if (log >= 3) {
294+
console.warn(`Storing pack...`);
295+
}
296+
pack.collectGarbage(1000 * 60 * 60 * 24 * 2);
297+
return serializer
298+
.serializeToFile(pack, `${cacheDirectory}.pack~`)
299+
.then(
300+
result =>
301+
new Promise((resolve, reject) => {
302+
if (!result) {
303+
if (log >= 1) {
304+
console.warn(
305+
'Caching failed for pack, because content is flagged as not serializable. Use store: "idle" instead.'
306+
);
307+
}
308+
return resolve();
309+
}
310+
fs.unlink(`${cacheDirectory}.pack`, err => {
311+
fs.rename(
312+
`${cacheDirectory}.pack~`,
313+
`${cacheDirectory}.pack`,
314+
err => {
315+
if (err) return reject(err);
316+
if (log >= 3) {
317+
console.warn(`Stored pack`);
318+
}
319+
resolve();
320+
}
321+
);
322+
});
323+
})
324+
)
325+
.then(() => {
326+
return serializer.deserializeFromFile(`${cacheDirectory}.pack`);
327+
})
328+
.catch(err => {
329+
if (log >= 1) {
330+
console.warn(
331+
`Caching failed for pack: ${log >= 4 ? err.stack : err}`
332+
);
333+
}
334+
return new Pack(version);
335+
});
336+
});
337+
return packPromise;
338+
};
166339
compiler.cache.hooks.shutdown.tapPromise("FileCachePlugin", () => {
167340
isIdle = false;
168341
const promises = Array.from(pendingPromiseFactories.values()).map(fn =>
169342
fn()
170343
);
171344
pendingPromiseFactories.clear();
172345
if (currentIdlePromise !== undefined) promises.push(currentIdlePromise);
173-
return Promise.all(promises);
346+
const promise = Promise.all(promises);
347+
if (store === "pack") {
348+
return promise.then(serializePack);
349+
}
350+
return promise;
174351
});
175352

176353
let currentIdlePromise;
@@ -193,6 +370,10 @@ class FileCachePlugin {
193370
};
194371
compiler.cache.hooks.beginIdle.tap("FileCachePlugin", () => {
195372
isIdle = true;
373+
if (store === "pack") {
374+
pendingPromiseFactories.delete("pack");
375+
pendingPromiseFactories.set("pack", serializePack);
376+
}
196377
Promise.resolve().then(processIdleTasks);
197378
});
198379
compiler.cache.hooks.endIdle.tap("FileCachePlugin", () => {

0 commit comments

Comments
 (0)