Merge pull request webpack#8622 from researchgate/perf/buffers-write

Improve memory consumption of file serialization middleware

Author: sokra

lib/serialization/FileMiddleware.js

@@ -255,6 +255,32 @@ const readSection = (filename, file, offset, size, callback) => {
 	});
 };
 
+const writeBuffers = (fileHandle, buffers, callback) => {
+	const stream = fs.createWriteStream(null, {
+		fd: fileHandle,
+		autoClose: false
+	});
+	let index = 0;
+
+	const doWrite = function() {
+		let canWriteMore = true;
+		while (canWriteMore && index < buffers.length) {
+			const chunk = buffers[index++];
+			canWriteMore = stream.write(chunk);
+		}
+
+		if (index < buffers.length) {
+			stream.once("drain", doWrite);
+		} else {
+			stream.end();
+		}
+	};
+
+	stream.on("error", err => callback(err));
+	stream.on("finish", () => callback(null));
+	doWrite();
+};
+
 class FileMiddleware extends SerializerMiddleware {
 	/**
 	 * @param {any[]} data data items
@@ -296,7 +322,7 @@ class FileMiddleware extends SerializerMiddleware {
 				mkdirp(path.dirname(filename), err => {
 					if (err) return reject(err);
 					fileManager.addJob(filename, true, (file, callback) => {
-						fs.writeFile(file, Buffer.concat(buffers), err => {
+						writeBuffers(file, buffers, err => {
 							if (err) return callback(err);
 							resolve(true);
 							callback();