Skip to content

Commit f2bce6f

Browse files
sokrasokra
authored
Merge pull request webpack#8651 from webpack/cache/improvements
Performance for caching
2 parents 9debdd3 + 324d46f commit f2bce6f

23 files changed

+250
-116
lines changed

‎declarations/WebpackOptions.d.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -431,6 +431,14 @@ export interface FileCacheOptions {
431431
* Algorithm used for generation the hash (see node.js crypto package)
432432
*/
433433
hashAlgorithm?: string;
434+
/**
435+
* Time in ms after which idle period the cache storing should happen (only for store: 'pack' or 'idle')
436+
*/
437+
idleTimeout?: number;
438+
/**
439+
* Time in ms after which idle period the initial cache storing should happen (only for store: 'pack' or 'idle')
440+
*/
441+
idleTimeoutForInitialStore?: number;
434442
/**
435443
* Display log info. (debug: all access and errors with stack trace, verbose: all access, info: all write access, warning: only failed serialization)
436444
*/

‎lib/Cache.js

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,23 @@ const { AsyncParallelHook, AsyncSeriesBailHook, SyncHook } = require("tapable");
99

1010
/** @typedef {(result: any, callback: (err?: Error) => void) => void} GotHandler */
1111

12+
const needCalls = (times, callback) => {
13+
return err => {
14+
if (--times === 0) {
15+
return callback(err);
16+
}
17+
if (err && times > 0) {
18+
times = NaN;
19+
return callback(err);
20+
}
21+
};
22+
};
23+
1224
class Cache {
1325
constructor() {
1426
this.hooks = {
1527
/** @type {AsyncSeriesBailHook<string, string>} */
16-
get: new AsyncSeriesBailHook(["identifier", "etag"]),
17-
/** @type {AsyncParallelHook<string, string, any>} */
18-
got: new AsyncParallelHook(["identifier", "etag", "data"]),
28+
get: new AsyncSeriesBailHook(["identifier", "etag", "gotHandlers"]),
1929
/** @type {AsyncParallelHook<string, string, any>} */
2030
store: new AsyncParallelHook(["identifier", "etag", "data"]),
2131
/** @type {SyncHook} */
@@ -28,18 +38,24 @@ class Cache {
2838
}
2939

3040
get(identifier, etag, callback) {
31-
this.hooks.get.callAsync(identifier, etag, (err, result) => {
41+
const gotHandlers = [];
42+
this.hooks.get.callAsync(identifier, etag, gotHandlers, (err, result) => {
3243
if (err) {
3344
callback(err);
3445
return;
3546
}
36-
this.hooks.got.callAsync(identifier, etag, result, err => {
37-
if (err) {
38-
callback(err);
39-
return;
47+
if (gotHandlers.length > 1) {
48+
const innerCallback = needCalls(gotHandlers.length, () =>
49+
callback(null, result)
50+
);
51+
for (const gotHandler of gotHandlers) {
52+
gotHandler(result, innerCallback);
4053
}
54+
} else if (gotHandlers.length === 1) {
55+
gotHandlers[0](result, () => callback(null, result));
56+
} else {
4157
callback(null, result);
42-
});
58+
}
4359
});
4460
}
4561

‎lib/Compilation.js

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -543,12 +543,12 @@ class Compilation {
543543
this._rebuildingModules = new Map();
544544
/** @type {Set<string>} */
545545
this.emittedAssets = new Set();
546-
/** @type {SortableSet<string>} */
547-
this.fileDependencies = new SortableSet();
548-
/** @type {SortableSet<string>} */
549-
this.contextDependencies = new SortableSet();
550-
/** @type {SortableSet<string>} */
551-
this.missingDependencies = new SortableSet();
546+
/** @type {Set<string>} */
547+
this.fileDependencies = new Set();
548+
/** @type {Set<string>} */
549+
this.contextDependencies = new Set();
550+
/** @type {Set<string>} */
551+
this.missingDependencies = new Set();
552552
}
553553

554554
getStats() {
@@ -2245,10 +2245,6 @@ class Compilation {
22452245
addAllToSet(this.missingDependencies, error.missing);
22462246
}
22472247
}
2248-
2249-
this.fileDependencies.sort();
2250-
this.contextDependencies.sort();
2251-
this.missingDependencies.sort();
22522248
}
22532249

22542250
createModuleHashes() {

‎lib/ContextModule.js

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -213,15 +213,23 @@ class ContextModule extends Module {
213213
* @returns {string | null} an identifier for library inclusion
214214
*/
215215
libIdent(options) {
216-
let identifier = contextify(options.context, this.context);
216+
let identifier = contextify(
217+
options.context,
218+
this.context,
219+
options.associatedObjectForCache
220+
);
217221
if (this.options.mode) {
218222
identifier += ` ${this.options.mode}`;
219223
}
220224
if (this.options.recursive) {
221225
identifier += " recursive";
222226
}
223227
if (this.options.addon) {
224-
identifier += ` ${contextify(options.context, this.options.addon)}`;
228+
identifier += ` ${contextify(
229+
options.context,
230+
this.options.addon,
231+
options.associatedObjectForCache
232+
)}`;
225233
}
226234
if (this.options.regExp) {
227235
identifier += ` ${this.prettyRegExp(this.options.regExp + "")}`;

‎lib/DelegatedModuleFactoryPlugin.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ const DelegatedModule = require("./DelegatedModule");
1212
// options.context
1313
// options.scope
1414
// options.content
15+
// options.associatedObjectForCache
1516
class DelegatedModuleFactoryPlugin {
1617
constructor(options) {
1718
this.options = options;

‎lib/DelegatedPlugin.js

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,14 @@ class DelegatedPlugin {
3838
);
3939

4040
compiler.hooks.compile.tap("DelegatedPlugin", ({ normalModuleFactory }) => {
41-
new DelegatedModuleFactoryPlugin(this.options).apply(normalModuleFactory);
41+
new DelegatedModuleFactoryPlugin(
42+
Object.assign(
43+
{
44+
associatedObjectForCache: compiler.root
45+
},
46+
this.options
47+
)
48+
).apply(normalModuleFactory);
4249
});
4350
}
4451
}

‎lib/DllReferencePlugin.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,8 @@ class DllReferencePlugin {
122122
scope: this.options.scope,
123123
context: this.options.context || compiler.options.context,
124124
content,
125-
extensions: this.options.extensions
125+
extensions: this.options.extensions,
126+
associatedObjectForCache: compiler.root
126127
}).apply(normalModuleFactory);
127128
});
128129

‎lib/LibManifestPlugin.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,8 @@ class LibManifestPlugin {
6767
return;
6868
}
6969
const ident = module.libIdent({
70-
context: this.options.context || compiler.options.context
70+
context: this.options.context || compiler.options.context,
71+
associatedObjectForCache: compiler.root
7172
});
7273
if (ident) {
7374
const providedExports = moduleGraph.getProvidedExports(

‎lib/Module.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ const makeSerializable = require("./util/makeSerializable");
3737
/**
3838
* @typedef {Object} LibIdentOptions
3939
* @property {string} context absolute context path to which lib ident is relative to
40+
* @property {Object=} associatedObjectForCache object for caching
4041
*/
4142

4243
/**

‎lib/cache/FileCachePlugin.js

Lines changed: 54 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
const path = require("path");
99
const createHash = require("../util/createHash");
1010
const makeSerializable = require("../util/makeSerializable");
11-
const { serializer } = require("../util/serialization");
11+
const { serializer, NOT_SERIALIZABLE } = require("../util/serialization");
1212

1313
/** @typedef {import("webpack-sources").Source} Source */
1414
/** @typedef {import("../../declarations/WebpackOptions").FileCacheOptions} FileCacheOptions */
@@ -23,6 +23,7 @@ class Pack {
2323
this.unserializable = new Set();
2424
this.used = new Set();
2525
this.invalid = false;
26+
this.log = 0;
2627
}
2728

2829
get(identifier) {
@@ -48,6 +49,10 @@ class Pack {
4849
}
4950
}
5051

52+
setLogLevel(log) {
53+
this.log = log;
54+
}
55+
5156
_updateLastAccess() {
5257
const now = Date.now();
5358
for (const identifier of this.used) {
@@ -65,6 +70,13 @@ class Pack {
6570
write(identifier);
6671
write(data);
6772
} catch (err) {
73+
if (this.log >= 1 && err !== NOT_SERIALIZABLE) {
74+
console.warn(
75+
`Caching failed for ${identifier}: ${
76+
this.log >= 4 ? err.stack : err
77+
}\nWe will not try to cache this entry again until the cache file is deleted.`
78+
);
79+
}
6880
rollback(s);
6981
this.unserializable.add(identifier);
7082
continue;
@@ -96,7 +108,6 @@ class FileCachePlugin {
96108
*/
97109
constructor(options) {
98110
this.options = options;
99-
this.missingEntries = new Set();
100111
}
101112

102113
/**
@@ -114,6 +125,11 @@ class FileCachePlugin {
114125
? { debug: 4, verbose: 3, info: 2, warning: 1 }[this.options.loglevel]
115126
: 0;
116127
const store = this.options.store || "pack";
128+
const idleTimeout = this.options.idleTimeout || 10000;
129+
const idleTimeoutForInitialStore = Math.min(
130+
idleTimeout,
131+
this.options.idleTimeoutForInitialStore || 0
132+
);
117133

118134
const resolvedPromise = Promise.resolve();
119135

@@ -162,7 +178,6 @@ class FileCachePlugin {
162178
});
163179
}
164180
const storeEntry = (identifier, etag, data) => {
165-
this.missingEntries.delete(identifier);
166181
const entry = {
167182
identifier,
168183
data: etag ? () => data : data,
@@ -210,20 +225,10 @@ class FileCachePlugin {
210225
}
211226
};
212227
compiler.cache.hooks.store.tapPromise("FileCachePlugin", storeEntry);
213-
compiler.cache.hooks.got.tapPromise(
214-
"FileCachePlugin",
215-
(identifier, etag, result) => {
216-
if (result !== undefined && this.missingEntries.has(identifier)) {
217-
return storeEntry(identifier, etag, result);
218-
} else {
219-
return resolvedPromise;
220-
}
221-
}
222-
);
223228

224229
compiler.cache.hooks.get.tapPromise(
225230
"FileCachePlugin",
226-
(identifier, etag) => {
231+
(identifier, etag, gotHandlers) => {
227232
let logMessage;
228233
let cacheEntryPromise;
229234
if (store === "pack") {
@@ -235,38 +240,43 @@ class FileCachePlugin {
235240
logMessage = filename;
236241
cacheEntryPromise = serializer.deserializeFromFile(filename);
237242
}
243+
const registerGot = () => {
244+
gotHandlers.push((result, callback) => {
245+
if (result !== undefined) {
246+
storeEntry(identifier, etag, result).then(callback, callback);
247+
} else {
248+
callback();
249+
}
250+
});
251+
};
238252
return cacheEntryPromise.then(
239253
cacheEntry => {
240254
if (cacheEntry === undefined) {
241-
this.missingEntries.add(identifier);
242-
return;
255+
return registerGot();
243256
}
244257
if (cacheEntry.identifier !== identifier) {
245258
if (log >= 3) {
246259
console.warn(
247260
`Restored ${identifier} from ${logMessage}, but identifier doesn't match.`
248261
);
249262
}
250-
this.missingEntries.add(identifier);
251-
return;
263+
return registerGot();
252264
}
253265
if (cacheEntry.etag !== etag) {
254266
if (log >= 3) {
255267
console.warn(
256268
`Restored ${identifier} from ${logMessage}, but etag doesn't match.`
257269
);
258270
}
259-
this.missingEntries.add(identifier);
260-
return;
271+
return registerGot();
261272
}
262273
if (cacheEntry.version !== version) {
263274
if (log >= 3) {
264275
console.warn(
265276
`Restored ${identifier} from ${logMessage}, but version doesn't match.`
266277
);
267278
}
268-
this.missingEntries.add(identifier);
269-
return;
279+
return registerGot();
270280
}
271281
if (log >= 3) {
272282
console.warn(`Restored ${identifier} from ${logMessage}.`);
@@ -275,14 +285,14 @@ class FileCachePlugin {
275285
return cacheEntry.data;
276286
},
277287
err => {
278-
this.missingEntries.add(identifier);
279288
if (log >= 1 && err && err.code !== "ENOENT") {
280289
console.warn(
281290
`Restoring failed for ${identifier} from ${logMessage}: ${
282291
log >= 4 ? err.stack : err
283292
}`
284293
);
285294
}
295+
registerGot();
286296
}
287297
);
288298
}
@@ -294,6 +304,7 @@ class FileCachePlugin {
294304
console.warn(`Storing pack...`);
295305
}
296306
pack.collectGarbage(1000 * 60 * 60 * 24 * 2);
307+
pack.setLogLevel(log);
297308
// You might think this breaks all access to the existing pack
298309
// which are still referenced, but serializing the pack memorizes
299310
// all data in the pack and makes it no longer need the backing file
@@ -331,6 +342,7 @@ class FileCachePlugin {
331342

332343
let currentIdlePromise;
333344
let isIdle = false;
345+
let isInitialStore = true;
334346
const processIdleTasks = () => {
335347
if (isIdle) {
336348
if (pendingPromiseFactories.size > 0) {
@@ -345,17 +357,32 @@ class FileCachePlugin {
345357
currentIdlePromise = Promise.all(promises).then(() => {
346358
currentIdlePromise = undefined;
347359
});
348-
currentIdlePromise.then(processIdleTasks);
349-
} else if (store === "pack") {
360+
currentIdlePromise.then(() => {
361+
// Allow to exit the process inbetween
362+
setTimeout(processIdleTasks, 0).unref();
363+
});
364+
return;
365+
}
366+
if (store === "pack") {
350367
currentIdlePromise = serializePack();
351368
}
369+
isInitialStore = false;
352370
}
353371
};
372+
let idleTimer = undefined;
354373
compiler.cache.hooks.beginIdle.tap("FileCachePlugin", () => {
355-
isIdle = true;
356-
resolvedPromise.then(processIdleTasks);
374+
idleTimer = setTimeout(() => {
375+
idleTimer = undefined;
376+
isIdle = true;
377+
resolvedPromise.then(processIdleTasks);
378+
}, isInitialStore ? idleTimeoutForInitialStore : idleTimeout);
379+
idleTimer.unref();
357380
});
358381
compiler.cache.hooks.endIdle.tap("FileCachePlugin", () => {
382+
if (idleTimer) {
383+
clearTimeout(idleTimer);
384+
idleTimer = undefined;
385+
}
359386
isIdle = false;
360387
});
361388
}

0 commit comments

Comments
 (0)