Merge pull request webpack#8650 from webpack/performance/webpack-sources

Caching of utf-8 conversion, memory improvement, refactoring

Author: sokra

declarations.d.ts

@@ -220,6 +220,101 @@ declare module "@webassemblyjs/ast" {
 	export function isFuncImportDescr(n: Node): boolean;
 }
 
+declare module "webpack-sources" {
+	type MapOptions = { columns?: boolean; module?: boolean };
+
+	export abstract class Source {
+		size(): number;
+
+		map(options?: MapOptions): Object;
+
+		sourceAndMap(
+			options?: MapOptions
+		): {
+			source: string | Buffer;
+			map: Object;
+		};
+
+		updateHash(hash: import("./lib/util/createHash").Hash): void;
+
+		source(): string | Buffer;
+	}
+
+	export class RawSource extends Source {
+		constructor(source: string | Buffer);
+
+		// TODO remove internals
+		_value: string | Buffer;
+	}
+
+	export class OriginalSource extends Source {
+		constructor(source: string | Buffer, name: string);
+
+		// TODO remove internals
+		_value: string | Buffer;
+		_name: string;
+	}
+
+	export class ReplaceSource extends Source {
+		constructor(source: Source, name?: string);
+
+		replace(start: number, end: number, newValue: string, name?: string): void;
+		insert(pos: number, newValue: string, name?: string): void;
+
+		// TODO remove internals
+		_name: string;
+		_source: string;
+		_replacements: {
+			start: number;
+			end: number;
+			content: string;
+			insertIndex: number;
+			name: string;
+		}[];
+	}
+
+	export class SourceMapSource extends Source {
+		constructor(
+			source: string,
+			name: string,
+			sourceMap: Object | string,
+			originalSource?: string,
+			innerSourceMap?: Object
+		);
+	}
+
+	export class ConcatSource extends Source {
+		constructor(...args: (string | Source)[]);
+
+		children: (string | Source)[];
+
+		add(item: string | Source): void;
+	}
+
+	export class PrefixSource extends Source {
+		constructor(prefix: string, source: string | Source);
+
+		_source: string | Source;
+		_prefix: string;
+	}
+
+	export class CachedSource extends Source {
+		constructor(source: Source);
+
+		_source: Source;
+		_cachedSource: string | Buffer;
+		_cachedBuffer: Buffer;
+		_cachedSize: number;
+		_cachedMaps: Object;
+	}
+
+	export class SizeOnlySource extends Source {
+		constructor(size: number);
+
+		_size: number;
+	}
+}
+
 // This "hack" is needed because typescript doesn't support recursive type definitions
 // It's referenced from "ruleSet-conditions" in schemas/WebpackOptions.json
 interface RuleSetConditionsRecursive

lib/Compilation.js

@@ -541,8 +541,8 @@ class Compilation {
 		this.builtModules = new WeakSet();
 		/** @private @type {Map<Module, Callback[]>} */
 		this._rebuildingModules = new Map();
-		/** @type {WeakSet<Source>} */
-		this.emittedAssets = new WeakSet();
+		/** @type {Set<string>} */
+		this.emittedAssets = new Set();
 		/** @type {SortableSet<string>} */
 		this.fileDependencies = new SortableSet();
 		/** @type {SortableSet<string>} */

lib/Compiler.js

@@ -15,6 +15,7 @@ const {
 	AsyncParallelHook,
 	AsyncSeriesHook
 } = require("tapable");
+const { SizeOnlySource } = require("webpack-sources");
 
 const Cache = require("./Cache");
 const Compilation = require("./Compilation");
@@ -27,6 +28,7 @@ const Stats = require("./Stats");
 const Watching = require("./Watching");
 const { makePathsRelative } = require("./util/identifier");
 
+/** @typedef {import("webpack-sources").Source} Source */
 /** @typedef {import("../declarations/WebpackOptions").Entry} Entry */
 /** @typedef {import("../declarations/WebpackOptions").OutputOptions} OutputOptions */
 /** @typedef {import("../declarations/WebpackOptions").WatchOptions} WatchOptions */
@@ -161,6 +163,11 @@ class Compiler {
 
 		/** @type {boolean} */
 		this.watchMode = false;
+
+		/** @private @type {WeakMap<Source, { sizeOnlySource: SizeOnlySource, writtenTo: Map<string, number> }>} */
+		this._assetEmittingSourceCache = new WeakMap();
+		/** @private @type {Map<string, number>} */
+		this._assetEmittingWrittenFiles = new Map();
 	}
 
 	/**
@@ -327,19 +334,70 @@ class Compiler {
 							outputPath,
 							targetFile
 						);
-						if (source.existsAt === targetPath) {
-							compilation.emittedAssets.delete(source);
-							return callback();
+
+						// check if the target file has already been written by this Compiler
+						const targetFileGeneration = this._assetEmittingWrittenFiles.get(
+							targetPath
+						);
+
+						// create an cache entry for this Source if not already existing
+						let cacheEntry = this._assetEmittingSourceCache.get(source);
+						if (cacheEntry === undefined) {
+							cacheEntry = {
+								sizeOnlySource: undefined,
+								writtenTo: new Map()
+							};
+							this._assetEmittingSourceCache.set(source, cacheEntry);
+						}
+
+						// if the target file has already been written
+						if (targetFileGeneration !== undefined) {
+							// check if the Source has been written to this target file
+							const writtenGeneration = cacheEntry.writtenTo.get(targetPath);
+							if (writtenGeneration === targetFileGeneration) {
+								// if yes, we skip writing the file
+								// as it's already there
+								// (we assume one doesn't remove files while the Compiler is running)
+								return callback();
+							}
 						}
-						let content = source.source();
 
-						if (!Buffer.isBuffer(content)) {
-							content = Buffer.from(content, "utf8");
+						// get the binary (Buffer) content from the Source
+						/** @type {Buffer} */
+						let content;
+						if (typeof source.buffer === "function") {
+							content = source.buffer();
+						} else {
+							const bufferOrString = source.source();
+							if (Buffer.isBuffer(bufferOrString)) {
+								content = bufferOrString;
+							} else {
+								content = Buffer.from(bufferOrString, "utf8");
+							}
 						}
 
-						source.existsAt = targetPath;
-						compilation.emittedAssets.add(source);
-						this.outputFileSystem.writeFile(targetPath, content, callback);
+						// Create a replacement resource which only allows to ask for size
+						// This allows to GC all memory allocated by the Source
+						// (expect when the Source is stored in any other cache)
+						cacheEntry.sizeOnlySource = new SizeOnlySource(content.length);
+						compilation.assets[file] = cacheEntry.sizeOnlySource;
+
+						// Write the file to output file system
+						this.outputFileSystem.writeFile(targetPath, content, err => {
+							if (err) return callback(err);
+
+							// information marker that the asset has been emitted
+							compilation.emittedAssets.add(file);
+
+							// cache the information that the Source has been written to that ___location
+							const newGeneration =
+								targetFileGeneration === undefined
+									? 1
+									: targetFileGeneration + 1;
+							cacheEntry.writtenTo.set(targetPath, newGeneration);
+							this._assetEmittingWrittenFiles.set(targetPath, newGeneration);
+							callback();
+						});
 					};
 
 					if (targetFile.match(/\/|\\/)) {

lib/NormalModule.js

@@ -574,7 +574,7 @@ class NormalModule extends Module {
 			}
 
 			const handleParseError = e => {
-				const source = this._source.source();
+				const source = /** @type {string} */ (this._source.source());
 				const error = new ModuleParseError(source, e);
 				this.markModuleAsErrored(error);
 				this._initBuildHash(compilation);

lib/stats/DefaultStatsFactoryPlugin.js

@@ -282,7 +282,7 @@ const SIMPLE_EXTRACTORS = {
 					return names;
 				}, /** @type {Set<string>} */ (new Set()))
 			).sort(compareIds);
-			object.emitted = compilation.emittedAssets.has(asset.source);
+			object.emitted = compilation.emittedAssets.has(asset.name);
 		},
 		performance: (object, asset) => {
 			object.isOverSizeLimit = SizeLimitsPlugin.isOverSizeLimit(asset.source);

lib/util/registerExternalSerializer.js

@@ -38,6 +38,7 @@ register(
 		 */
 		serialize(source, { write }) {
 			write(source._source);
+			write(source._cachedBuffer);
 			write(source._cachedSource);
 			write(source._cachedSize);
 			write(source._cachedMaps);
@@ -49,6 +50,7 @@ register(
 		 */
 		deserialize({ read }) {
 			const source = new CachedSource(read());
+			source._cachedBuffer = read();
 			source._cachedSource = read();
 			source._cachedSize = read();
 			source._cachedMaps = read();
@@ -148,9 +150,7 @@ register(
 		serialize(source, { write }) {
 			write(source._source);
 			write(source._name);
-			const replacements = /** @type {TODO} */ (Array.from(
-				source.replacements
-			));
+			const replacements = Array.from(source._replacements);
 			replacements.sort((a, b) => {
 				return a.insertIndex - b.insertIndex;
 			});
@@ -171,7 +171,6 @@ register(
 			const source = new ReplaceSource(read(), read());
 			const len = read();
 			for (let i = 0; i < len; i++) {
-				// @ts-ignore TODO signature is missing one argument in typings
 				source.replace(read(), read(), read(), read());
 			}
 			return source;

package.json

@@ -29,12 +29,11 @@
     "tapable": "^1.1.0",
     "terser-webpack-plugin": "^1.2.1",
     "watchpack": "2.0.0-beta.2",
-    "webpack-sources": "^1.3.0"
+    "webpack-sources": "2.0.0-beta.0"
   },
   "devDependencies": {
     "@types/node": "^10.12.3",
     "@types/tapable": "^1.0.1",
-    "@types/webpack-sources": "^0.1.4",
     "benchmark": "^2.1.1",
     "bundle-loader": "~0.5.0",
     "codacy-coverage": "^3.1.0",

test/HotModuleReplacementPlugin.test.js

@@ -99,32 +99,25 @@ describe("HotModuleReplacementPlugin", () => {
 	}, 120000);
 
 	it("should correct working when entry is Object and key is a number", done => {
+		const outputPath = path.join(__dirname, "js", "HotModuleReplacementPlugin");
 		const entryFile = path.join(
-			__dirname,
-			"js",
-			"HotModuleReplacementPlugin",
+			outputPath,
 			"entry.js"
 		);
 		const statsFile3 = path.join(
-			__dirname,
-			"js",
-			"HotModuleReplacementPlugin",
+			outputPath,
 			"HotModuleReplacementPlugin.test.stats3.txt"
 		);
 		const statsFile4 = path.join(
-			__dirname,
-			"js",
-			"HotModuleReplacementPlugin",
+			outputPath,
 			"HotModuleReplacementPlugin.test.stats4.txt"
 		);
 		const recordsFile = path.join(
-			__dirname,
-			"js",
-			"HotModuleReplacementPlugin",
+			outputPath,
 			"records.json"
 		);
 		try {
-			mkdirp.sync(path.join(__dirname, "js", "HotModuleReplacementPlugin"));
+			mkdirp.sync(outputPath);
 		} catch (e) {
 			// empty
 		}
@@ -141,7 +134,7 @@ describe("HotModuleReplacementPlugin", () => {
 			},
 			recordsPath: recordsFile,
 			output: {
-				path: path.join(__dirname, "js", "HotModuleReplacementPlugin")
+				path: outputPath
 			},
 			plugins: [new webpack.HotModuleReplacementPlugin()],
 			optimization: {
@@ -163,7 +156,7 @@ describe("HotModuleReplacementPlugin", () => {
 					if (err) throw err;
 					fs.writeFileSync(statsFile3, stats.toString());
 					const result = JSON.parse(
-						stats.compilation.assets[`${hash}.hot-update.json`].source()
+						fs.readFileSync(path.join(outputPath, `${hash}.hot-update.json`), "utf-8")
 					)["c"];
 					expect(result).toEqual([chunkName]);
 					done();