Merge pull request webpack#8477 from webpack/feature/better-chunk-ids

improve ids for Long Term Caching

Author: sokra

declarations/WebpackOptions.d.ts

@@ -805,9 +805,15 @@ export interface OptimizationOptions {
 	 */
 	checkWasmTypes?: boolean;
 	/**
-	 * Define the algorithm to choose chunk ids (named: readable ids for better debugging, size: numeric ids focused on minimal initial download size, total-size: numeric ids focused on minimal total download size, false: no algorithm used, as custom one can be provided via plugin)
+	 * Define the algorithm to choose chunk ids (named: readable ids for better debugging, deterministic: numeric hash ids for better long term caching, size: numeric ids focused on minimal initial download size, total-size: numeric ids focused on minimal total download size, false: no algorithm used, as custom one can be provided via plugin)
 	 */
-	chunkIds?: "natural" | "named" | "size" | "total-size" | false;
+	chunkIds?:
+		| "natural"
+		| "named"
+		| "deterministic"
+		| "size"
+		| "total-size"
+		| false;
 	/**
 	 * Concatenate modules when possible to generate less modules, more efficient code and enable more optimizations by the minimizer
 	 */
@@ -911,10 +917,6 @@ export interface OptimizationSplitChunksOptions {
 					 * Sets the name delimiter for created chunks
 					 */
 					automaticNameDelimiter?: string;
-					/**
-					 * Sets the name prefix for created chunks
-					 */
-					automaticNamePrefix?: string;
 					/**
 					 * Select chunks for determining cache group content (defaults to "initial", "initial" and "all" requires adding these chunks to the HTML)
 					 */
@@ -927,6 +929,10 @@ export interface OptimizationSplitChunksOptions {
 					 * Sets the template for the filename for created chunks (Only works for initial chunks)
 					 */
 					filename?: string;
+					/**
+					 * Sets the hint for chunk id
+					 */
+					idHint?: string;
 					/**
 					 * Maximum number of requests which are accepted for on-demand loading
 					 */
@@ -950,7 +956,7 @@ export interface OptimizationSplitChunksOptions {
 					/**
 					 * Give chunks for this cache group a name (chunks with equal name are merged)
 					 */
-					name?: boolean | Function | string;
+					name?: false | Function | string;
 					/**
 					 * Priority of this cache group
 					 */
@@ -1021,7 +1027,7 @@ export interface OptimizationSplitChunksOptions {
 	/**
 	 * Give chunks created a name (chunks with equal name are merged)
 	 */
-	name?: boolean | Function | string;
+	name?: false | Function | string;
 }
 /**
  * This interface was referenced by `WebpackOptions`'s JSON-Schema

lib/Chunk.js

@@ -66,6 +66,8 @@ class Chunk {
 		this.debugId = debugId++;
 		/** @type {string} */
 		this.name = name;
+		/** @type {SortableSet<string>} */
+		this.idNameHints = new SortableSet();
 		/** @type {boolean} */
 		this.preventIntegration = false;
 		/** @type {string?} */
@@ -413,6 +415,9 @@ class Chunk {
 			chunkGroup.insertChunk(newChunk, this);
 			newChunk.addGroup(chunkGroup);
 		}
+		for (const idHint of this.idNameHints) {
+			newChunk.idNameHints.add(idHint);
+		}
 	}
 
 	/**

lib/ChunkGraph.js

@@ -690,6 +690,11 @@ class ChunkGraph {
 			chunkA.name = chunkB.name;
 		}
 
+		// Merge id name hints
+		for (const hint of chunkB.idNameHints) {
+			chunkA.idNameHints.add(hint);
+		}
+
 		// getChunkModules is used here to create a clone, because disconnectChunkAndModule modifies
 		for (const module of this.getChunkModules(chunkB)) {
 			this.disconnectChunkAndModule(chunkB, module);

lib/ContextModule.js

@@ -125,35 +125,35 @@ class ContextModule extends Module {
 	_createIdentifier() {
 		let identifier = this.context;
 		if (this.options.resourceQuery) {
-			identifier += ` ${this.options.resourceQuery}`;
+			identifier += `|${this.options.resourceQuery}`;
 		}
 		if (this.options.mode) {
-			identifier += ` ${this.options.mode}`;
+			identifier += `|${this.options.mode}`;
 		}
 		if (!this.options.recursive) {
-			identifier += " nonrecursive";
+			identifier += "|nonrecursive";
 		}
 		if (this.options.addon) {
-			identifier += ` ${this.options.addon}`;
+			identifier += `|${this.options.addon}`;
 		}
 		if (this.options.regExp) {
-			identifier += ` ${this.options.regExp}`;
+			identifier += `|${this.options.regExp}`;
 		}
 		if (this.options.include) {
-			identifier += ` include: ${this.options.include}`;
+			identifier += `|include: ${this.options.include}`;
 		}
 		if (this.options.exclude) {
-			identifier += ` exclude: ${this.options.exclude}`;
+			identifier += `|exclude: ${this.options.exclude}`;
 		}
 		if (this.options.groupOptions) {
-			identifier += ` groupOptions: ${JSON.stringify(
+			identifier += `|groupOptions: ${JSON.stringify(
 				this.options.groupOptions
 			)}`;
 		}
 		if (this.options.namespaceObject === "strict") {
-			identifier += " strict namespace object";
+			identifier += "|strict namespace object";
 		} else if (this.options.namespaceObject) {
-			identifier += " namespace object";
+			identifier += "|namespace object";
 		}
 
 		return identifier;

lib/NormalModuleFactory.js

@@ -211,7 +211,7 @@ class NormalModuleFactory {
 							null,
 							new RawModule(
 								"/* (ignored) */",
-								`ignored ${context} ${request}`,
+								`ignored|${request}`,
 								`${request} (ignored)`
 							)
 						);

lib/Stats.js

@@ -1552,7 +1552,7 @@ class Stats {
 				if (childString) {
 					if (child.name) {
 						colors.normal("Child ");
-						colors.bold(child.name);
+						colors.bold(child.name.replace(/\|/g, " "));
 						colors.normal(":");
 					} else {
 						colors.normal("Child");

lib/WebpackOptionsApply.js

@@ -47,6 +47,7 @@ const SystemPlugin = require("./dependencies/SystemPlugin");
 const WarnDeprecatedOptionPlugin = require("./WarnDeprecatedOptionPlugin");
 const WarnNoModeSetPlugin = require("./WarnNoModeSetPlugin");
 
+const DeterministicChunkIdsPlugin = require("./ids/DeterministicChunkIdsPlugin");
 const DeterministicModuleIdsPlugin = require("./ids/DeterministicModuleIdsPlugin");
 const HashedModuleIdsPlugin = require("./ids/HashedModuleIdsPlugin");
 const NamedChunkIdsPlugin = require("./ids/NamedChunkIdsPlugin");
@@ -395,9 +396,7 @@ class WebpackOptionsApply extends OptionsApply {
 					new HashedModuleIdsPlugin().apply(compiler);
 					break;
 				case "deterministic":
-					new DeterministicModuleIdsPlugin({
-						maxLength: 3
-					}).apply(compiler);
+					new DeterministicModuleIdsPlugin().apply(compiler);
 					break;
 				case "size":
 					new OccurrenceModuleIdsPlugin({
@@ -419,6 +418,9 @@ class WebpackOptionsApply extends OptionsApply {
 				case "named":
 					new NamedChunkIdsPlugin().apply(compiler);
 					break;
+				case "deterministic":
+					new DeterministicChunkIdsPlugin().apply(compiler);
+					break;
 				case "size":
 					new OccurrenceChunkIdsPlugin({
 						prioritiseInitial: true

lib/WebpackOptionsDefaulter.js

@@ -134,8 +134,9 @@ class WebpackOptionsDefaulter extends OptionsDefaulter {
 				const hasName = filename.includes("[name]");
 				const hasId = filename.includes("[id]");
 				const hasChunkHash = filename.includes("[chunkhash]");
+				const hasContentHash = filename.includes("[contenthash]");
 				// Anything changing depending on chunk is fine
-				if (hasChunkHash || hasName || hasId) return filename;
+				if (hasChunkHash || hasContentHash || hasName || hasId) return filename;
 				// Elsewise prefix "[id]." in front of the basename to make it changing
 				return filename.replace(/(^|\/)([^/]*(?:\?|$))/, "$1[id].$2");
 			}
@@ -240,7 +241,7 @@ class WebpackOptionsDefaulter extends OptionsDefaulter {
 			return "natural";
 		});
 		this.set("optimization.chunkIds", "make", options => {
-			if (isProductionLikeMode(options)) return "total-size";
+			if (isProductionLikeMode(options)) return "deterministic";
 			if (options.mode === "development") return "named";
 			return "natural";
 		});
@@ -266,20 +267,19 @@ class WebpackOptionsDefaulter extends OptionsDefaulter {
 		this.set("optimization.splitChunks.maxAsyncRequests", "make", options => {
 			return isProductionLikeMode(options) ? 6 : Infinity;
 		});
-		this.set("optimization.splitChunks.automaticNameDelimiter", "~");
+		this.set("optimization.splitChunks.automaticNameDelimiter", "-");
 		this.set("optimization.splitChunks.maxInitialRequests", "make", options => {
 			return isProductionLikeMode(options) ? 4 : Infinity;
 		});
-		this.set("optimization.splitChunks.name", true);
 		this.set("optimization.splitChunks.cacheGroups", {});
 		this.set("optimization.splitChunks.cacheGroups.default", {
-			automaticNamePrefix: "",
+			idHint: "",
 			reuseExistingChunk: true,
 			minChunks: 2,
 			priority: -20
 		});
 		this.set("optimization.splitChunks.cacheGroups.defaultVendors", {
-			automaticNamePrefix: "vendors",
+			idHint: "vendors",
 			test: NODE_MODULES_REGEXP,
 			priority: -10
 		});

lib/ids/DeterministicChunkIdsPlugin.js

@@ -0,0 +1,62 @@
+/*
+	MIT License http://www.opensource.org/licenses/mit-license.php
+	Author Florent Cailhol @ooflorent
+*/
+
+"use strict";
+
+const { compareChunksNatural } = require("../util/comparators");
+const {
+	getFullChunkName,
+	getUsedChunkIds,
+	assignDeterministicIds
+} = require("./IdHelpers");
+
+/** @typedef {import("../Compiler")} Compiler */
+/** @typedef {import("../Module")} Module */
+
+class DeterministicChunkIdsPlugin {
+	constructor(options) {
+		this.options = options || {};
+	}
+
+	/**
+	 * @param {Compiler} compiler the compiler instance
+	 * @returns {void}
+	 */
+	apply(compiler) {
+		compiler.hooks.compilation.tap(
+			"DeterministicChunkIdsPlugin",
+			compilation => {
+				compilation.hooks.chunkIds.tap(
+					"DeterministicChunkIdsPlugin",
+					chunks => {
+						const chunkGraph = compilation.chunkGraph;
+						const context = this.options.context
+							? this.options.context
+							: compiler.context;
+
+						const compareNatural = compareChunksNatural(chunkGraph);
+
+						assignDeterministicIds(
+							Array.from(chunks).filter(chunk => {
+								return chunk.id === null;
+							}),
+							chunk =>
+								getFullChunkName(chunk, chunkGraph, context, compiler.root),
+							compareNatural,
+							this.options.maxLength || 3,
+							getUsedChunkIds(compilation),
+							(chunk, id) => {
+								chunk.id = id;
+								chunk.ids = [id];
+							}
+						);
+					}
+				);
+			}
+		);
+	}
+}
+
+module.exports = DeterministicChunkIdsPlugin;

lib/ids/DeterministicModuleIdsPlugin.js

@@ -8,29 +8,15 @@
 const {
 	compareModulesByPreOrderIndexOrIdentifier
 } = require("../util/comparators");
-const createHash = require("../util/createHash");
+const {
+	getUsedModuleIds,
+	getFullModuleName,
+	assignDeterministicIds
+} = require("./IdHelpers");
 
 /** @typedef {import("../Compiler")} Compiler */
 /** @typedef {import("../Module")} Module */
 
-/**
- * @param {string} str string to hash
- * @param {number} len max length of the number in decimal digests
- * @returns {number} hash as number in the range from i. e. for len = 3: 0 - 999
- */
-const getHashNumber = (str, len) => {
-	const hash = createHash("md4");
-	hash.update(str);
-	const digest = hash.digest("hex") + "10000000000";
-	if (len === 1) {
-		return +digest.match(/\d/)[0];
-	}
-	return +digest
-		.match(/\d/g)
-		.slice(0, len)
-		.join("");
-};
-
 class DeterministicModuleIdsPlugin {
 	constructor(options) {
 		this.options = options || {};
@@ -52,57 +38,22 @@ class DeterministicModuleIdsPlugin {
 							? this.options.context
 							: compiler.context;
 
-						const modulesInNaturalOrder = Array.from(modules)
-							.filter(m => chunkGraph.getNumberOfModuleChunks(m) > 0)
-							.sort(
-								compareModulesByPreOrderIndexOrIdentifier(
-									compilation.moduleGraph
-								)
-							);
-
-						// 80% fill rate
-						const optimalLength = Math.ceil(
-							Math.log10(modulesInNaturalOrder.length * 1.25 + 1)
-						);
-
-						// use the provided length or default to the optimal length
-						const maxLength = Math.max(
-							Math.min(
-								typeof this.options.maxLength === "number"
-									? this.options.maxLength
-									: 0,
-								15 // higher values will give numerical problems
+						assignDeterministicIds(
+							Array.from(modules).filter(module => {
+								if (chunkGraph.getNumberOfModuleChunks(module) === 0)
+									return false;
+								return chunkGraph.getModuleId(module) === null;
+							}),
+							module => getFullModuleName(module, context, compiler.root),
+							compareModulesByPreOrderIndexOrIdentifier(
+								compilation.moduleGraph
 							),
-							optimalLength
-						);
-
-						const usedIds = new Set();
-
-						if (compilation.usedModuleIds) {
-							for (const id of compilation.usedModuleIds) {
-								usedIds.add(id);
-							}
-						}
-
-						for (const module of modulesInNaturalOrder) {
-							const moduleId = chunkGraph.getModuleId(module);
-							if (moduleId !== null) {
-								usedIds.add(moduleId);
-							}
-						}
-
-						for (const module of modulesInNaturalOrder) {
-							if (chunkGraph.getModuleId(module) === null) {
-								const ident = module.libIdent({ context }) || "";
-								let id;
-								let i = 0;
-								do {
-									id = getHashNumber(ident + i++, maxLength);
-								} while (usedIds.has(id));
+							this.options.maxLength || 3,
+							getUsedModuleIds(compilation),
+							(module, id) => {
 								chunkGraph.setModuleId(module, id);
-								usedIds.add(id);
 							}
-						}
+						);
 					}
 				);
 			}