From b02918883fffcb6bcd726e7cc6e60d4e3e8a7d73 Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Tue, 19 Mar 2024 11:10:14 +0100 Subject: [PATCH] fix: support recursive readdir in Asar files (#41582) --- lib/node/asar-fs-wrapper.ts | 272 +++++++++++++++++- spec/asar-spec.ts | 94 ++++++ spec/fixtures/recursive-asar/a.asar | Bin 0 -> 3458 bytes spec/fixtures/recursive-asar/nested/hello.txt | 1 + spec/fixtures/recursive-asar/test.txt | 1 + 5 files changed, 356 insertions(+), 12 deletions(-) create mode 100644 spec/fixtures/recursive-asar/a.asar create mode 100644 spec/fixtures/recursive-asar/nested/hello.txt create mode 100644 spec/fixtures/recursive-asar/test.txt diff --git a/lib/node/asar-fs-wrapper.ts b/lib/node/asar-fs-wrapper.ts index e3dc73e9c8..fda03e62e1 100644 --- a/lib/node/asar-fs-wrapper.ts +++ b/lib/node/asar-fs-wrapper.ts @@ -1,5 +1,5 @@ import { Buffer } from 'buffer'; -import { constants } from 'fs'; +import { Dirent, constants } from 'fs'; import * as path from 'path'; import * as util from 'util'; @@ -46,7 +46,17 @@ process._getOrCreateArchive = getOrCreateArchive; const asarRe = /\.asar/i; -const { getValidatedPath } = __non_webpack_require__('internal/fs/utils'); +const { + getValidatedPath, + getOptions, + getDirent +} = __non_webpack_require__('internal/fs/utils'); + +const { + validateBoolean, + validateFunction +} = __non_webpack_require__('internal/validators'); + // In the renderer node internals use the node global URL but we do not set that to be // the global URL instance. We need to do instanceof checks against the internal URL impl const { URL: NodeURL } = __non_webpack_require__('internal/url'); @@ -78,6 +88,22 @@ const gid = process.getgid?.() ?? 0; const fakeTime = new Date(); +function getDirents (p: string, { 0: names, 1: types }: any[][]): Dirent[] { + const info = splitPath(p); + const len = names.length; + for (let i = 0; i < len; i++) { + if (info.isAsar) { + const archive = getOrCreateArchive(info.asarPath); + const stats = archive!.stat(p); + if (!stats) continue; + names[i] = getDirent(p, names[i], stats.type); + } else { + names[i] = getDirent(p, names[i], types[i]); + } + } + return names; +} + enum AsarFileType { kFile = (constants as any).UV_DIRENT_FILE, kDirectory = (constants as any).UV_DIRENT_DIR, @@ -662,13 +688,27 @@ export const wrapFsWithAsar = (fs: Record) => { return (encoding) ? buffer.toString(encoding) : buffer; }; + type ReaddirOptions = { encoding: BufferEncoding | null; withFileTypes?: false, recursive?: false } | undefined | null; + type ReaddirCallback = (err: NodeJS.ErrnoException | null, files: string[]) => void; + const { readdir } = fs; - fs.readdir = function (pathArgument: string, options?: { encoding?: string | null; withFileTypes?: boolean } | null, callback?: Function) { - const pathInfo = splitPath(pathArgument); - if (typeof options === 'function') { - callback = options; - options = undefined; + fs.readdir = function (pathArgument: string, options: ReaddirOptions, callback: ReaddirCallback) { + callback = typeof options === 'function' ? options : callback; + validateFunction(callback, 'callback'); + + options = getOptions(options); + pathArgument = getValidatedPath(pathArgument); + + if (options?.recursive != null) { + validateBoolean(options?.recursive, 'options.recursive'); } + + if (options?.recursive) { + nextTick(callback!, [null, readdirSyncRecursive(pathArgument, options)]); + return; + } + + const pathInfo = splitPath(pathArgument); if (!pathInfo.isAsar) return readdir.apply(this, arguments); const { asarPath, filePath } = pathInfo; @@ -705,12 +745,62 @@ export const wrapFsWithAsar = (fs: Record) => { nextTick(callback!, [null, files]); }; - fs.promises.readdir = util.promisify(fs.readdir); + const { readdir: readdirPromise } = require('fs').promises; + fs.promises.readdir = async function (pathArgument: string, options: ReaddirOptions) { + options = getOptions(options); + pathArgument = getValidatedPath(pathArgument); - type ReaddirSyncOptions = { encoding: BufferEncoding | null; withFileTypes?: false }; + if (options?.recursive != null) { + validateBoolean(options?.recursive, 'options.recursive'); + } + + if (options?.recursive) { + return readdirRecursive(pathArgument, options); + } + + const pathInfo = splitPath(pathArgument); + if (!pathInfo.isAsar) return readdirPromise(pathArgument, options); + const { asarPath, filePath } = pathInfo; + + const archive = getOrCreateArchive(asarPath); + if (!archive) { + return Promise.reject(createError(AsarError.INVALID_ARCHIVE, { asarPath })); + } + + const files = archive.readdir(filePath); + if (!files) { + return Promise.reject(createError(AsarError.NOT_FOUND, { asarPath, filePath })); + } + + if (options?.withFileTypes) { + const dirents = []; + for (const file of files) { + const childPath = path.join(filePath, file); + const stats = archive.stat(childPath); + if (!stats) { + throw createError(AsarError.NOT_FOUND, { asarPath, filePath: childPath }); + } + dirents.push(new fs.Dirent(file, stats.type)); + } + return Promise.resolve(dirents); + } + + return Promise.resolve(files); + }; const { readdirSync } = fs; - fs.readdirSync = function (pathArgument: string, options: ReaddirSyncOptions | BufferEncoding | null) { + fs.readdirSync = function (pathArgument: string, options: ReaddirOptions) { + options = getOptions(options); + pathArgument = getValidatedPath(pathArgument); + + if (options?.recursive != null) { + validateBoolean(options?.recursive, 'options.recursive'); + } + + if (options?.recursive) { + return readdirSyncRecursive(pathArgument, options); + } + const pathInfo = splitPath(pathArgument); if (!pathInfo.isAsar) return readdirSync.apply(this, arguments); const { asarPath, filePath } = pathInfo; @@ -725,7 +815,7 @@ export const wrapFsWithAsar = (fs: Record) => { throw createError(AsarError.NOT_FOUND, { asarPath, filePath }); } - if (options && (options as ReaddirSyncOptions).withFileTypes) { + if (options?.withFileTypes) { const dirents = []; for (const file of files) { const childPath = path.join(filePath, file); @@ -741,7 +831,8 @@ export const wrapFsWithAsar = (fs: Record) => { return files; }; - const { internalModuleReadJSON } = internalBinding('fs'); + const binding = internalBinding('fs'); + const { internalModuleReadJSON, kUsePromises } = binding; internalBinding('fs').internalModuleReadJSON = (pathArgument: string) => { const pathInfo = splitPath(pathArgument); if (!pathInfo.isAsar) return internalModuleReadJSON(pathArgument); @@ -787,6 +878,163 @@ export const wrapFsWithAsar = (fs: Record) => { return (stats.type === AsarFileType.kDirectory) ? 1 : 0; }; + async function readdirRecursive (originalPath: string, options: ReaddirOptions) { + const result: any[] = []; + + const pathInfo = splitPath(originalPath); + let queue: [string, string[]][] = []; + + let initialItem = []; + if (pathInfo.isAsar) { + const archive = getOrCreateArchive(pathInfo.asarPath); + if (!archive) return result; + const files = archive.readdir(pathInfo.filePath); + if (!files) return result; + initialItem = files; + } else { + initialItem = await binding.readdir( + path.toNamespacedPath(originalPath), + options!.encoding, + !!options!.withFileTypes, + kUsePromises + ); + } + + queue = [[originalPath, initialItem]]; + + if (options?.withFileTypes) { + while (queue.length > 0) { + // @ts-expect-error this is a valid array destructure assignment. + const { 0: pathArg, 1: readDir } = queue.pop(); + for (const dirent of getDirents(pathArg, readDir)) { + result.push(dirent); + if (dirent.isDirectory()) { + const direntPath = path.join(pathArg, dirent.name); + const info = splitPath(direntPath); + let archive; + let readdirResult; + if (info.isAsar) { + archive = getOrCreateArchive(info.asarPath); + if (!archive) continue; + readdirResult = archive.readdir(info.filePath); + } else { + readdirResult = await binding.readdir( + direntPath, + options.encoding, + true, + kUsePromises + ); + } + queue.push([direntPath, readdirResult]); + } + } + } + } else { + while (queue.length > 0) { + // @ts-expect-error this is a valid array destructure assignment. + const { 0: pathArg, 1: readDir } = queue.pop(); + for (const ent of readDir) { + const direntPath = path.join(pathArg, ent); + const stat = internalBinding('fs').internalModuleStat(direntPath); + result.push(path.relative(originalPath, direntPath)); + + if (stat === 1) { + const subPathInfo = splitPath(direntPath); + let item = []; + if (subPathInfo.isAsar) { + const archive = getOrCreateArchive(subPathInfo.asarPath); + if (!archive) return; + const files = archive.readdir(subPathInfo.filePath); + if (!files) return result; + item = files; + } else { + item = await binding.readdir( + path.toNamespacedPath(direntPath), + options!.encoding, + false, + kUsePromises + ); + } + queue.push([direntPath, item]); + } + } + } + } + + return result; + } + + function readdirSyncRecursive (basePath: string, options: ReaddirOptions) { + const withFileTypes = Boolean(options!.withFileTypes); + const encoding = options!.encoding; + + const readdirResults: string[] = []; + const pathsQueue = [basePath]; + + function read (pathArg: string) { + let readdirResult; + const pathInfo = splitPath(pathArg); + + let archive; + if (pathInfo.isAsar) { + const { asarPath, filePath } = pathInfo; + archive = getOrCreateArchive(asarPath); + if (!archive) return; + + readdirResult = archive.readdir(filePath); + } else { + readdirResult = binding.readdir( + path.toNamespacedPath(pathArg), + encoding, + withFileTypes + ); + } + + if (readdirResult === undefined) return; + + if (withFileTypes) { + // Calling `readdir` with `withFileTypes=true`, the result is an array of arrays. + // The first array is the names, and the second array is the types. + // They are guaranteed to be the same length; hence, setting `length` to the length + // of the first array within the result. + const length = readdirResult[0].length; + for (let i = 0; i < length; i++) { + let dirent; + if (pathInfo.isAsar) { + const stats = archive!.stat(pathArg); + if (!stats) continue; + dirent = getDirent(pathArg, readdirResult[0][i], stats.type); + } else { + dirent = getDirent(pathArg, readdirResult[0][i], readdirResult[1][i]); + } + + readdirResults.push(dirent); + if (dirent.isDirectory()) { + pathsQueue.push(path.join(dirent.path, dirent.name)); + } + } + } else { + for (let i = 0; i < readdirResult.length; i++) { + const resultPath = path.join(pathArg, readdirResult[i]); + const relativeResultPath = path.relative(basePath, resultPath); + const stat = internalBinding('fs').internalModuleStat(resultPath); + readdirResults.push(relativeResultPath); + + // 1 indicates directory + if (stat === 1) { + pathsQueue.push(resultPath); + } + } + } + } + + for (let i = 0; i < pathsQueue.length; i++) { + read(pathsQueue[i]); + } + + return readdirResults; + } + // Calling mkdir for directory inside asar archive should throw ENOTDIR // error, but on Windows it throws ENOENT. if (process.platform === 'win32') { diff --git a/spec/asar-spec.ts b/spec/asar-spec.ts index 48fea506f9..beb0644933 100644 --- a/spec/asar-spec.ts +++ b/spec/asar-spec.ts @@ -161,6 +161,7 @@ describe('asar package', function () { fs = require('node:fs') path = require('node:path') + fixtures = ${JSON.stringify(fixtures)} asarDir = ${JSON.stringify(asarDir)} // This is used instead of util.promisify for some tests to dodge the @@ -897,6 +898,37 @@ describe('asar package', function () { expect(dirs).to.deep.equal(['dir1', 'dir2', 'dir3', 'file1', 'file2', 'file3', 'link1', 'link2', 'ping.js']); }); + itremote('supports recursive readdirSync', async () => { + const dir = path.join(fixtures, 'recursive-asar'); + const files = await fs.readdirSync(dir, { recursive: true }); + expect(files).to.have.members([ + 'a.asar', + 'nested', + 'test.txt', + path.join('a.asar', 'dir1'), + path.join('a.asar', 'dir2'), + path.join('a.asar', 'dir3'), + path.join('a.asar', 'file1'), + path.join('a.asar', 'file2'), + path.join('a.asar', 'file3'), + path.join('a.asar', 'link1'), + path.join('a.asar', 'link2'), + path.join('a.asar', 'ping.js'), + path.join('nested', 'hello.txt'), + path.join('a.asar', 'dir1', 'file1'), + path.join('a.asar', 'dir1', 'file2'), + path.join('a.asar', 'dir1', 'file3'), + path.join('a.asar', 'dir1', 'link1'), + path.join('a.asar', 'dir1', 'link2'), + path.join('a.asar', 'dir2', 'file1'), + path.join('a.asar', 'dir2', 'file2'), + path.join('a.asar', 'dir2', 'file3'), + path.join('a.asar', 'dir3', 'file1'), + path.join('a.asar', 'dir3', 'file2'), + path.join('a.asar', 'dir3', 'file3') + ]); + }); + itremote('reads dirs from a normal dir', function () { const p = path.join(asarDir, 'a.asar', 'dir1'); const dirs = fs.readdirSync(p); @@ -944,6 +976,37 @@ describe('asar package', function () { expect(dirs).to.deep.equal(['dir1', 'dir2', 'dir3', 'file1', 'file2', 'file3', 'link1', 'link2', 'ping.js']); }); + itremote('supports recursive readdirSync', async () => { + const dir = path.join(fixtures, 'recursive-asar'); + const files = await promisify(fs.readdir)(dir, { recursive: true }); + expect(files).to.have.members([ + 'a.asar', + 'nested', + 'test.txt', + path.join('a.asar', 'dir1'), + path.join('a.asar', 'dir2'), + path.join('a.asar', 'dir3'), + path.join('a.asar', 'file1'), + path.join('a.asar', 'file2'), + path.join('a.asar', 'file3'), + path.join('a.asar', 'link1'), + path.join('a.asar', 'link2'), + path.join('a.asar', 'ping.js'), + path.join('nested', 'hello.txt'), + path.join('a.asar', 'dir1', 'file1'), + path.join('a.asar', 'dir1', 'file2'), + path.join('a.asar', 'dir1', 'file3'), + path.join('a.asar', 'dir1', 'link1'), + path.join('a.asar', 'dir1', 'link2'), + path.join('a.asar', 'dir2', 'file1'), + path.join('a.asar', 'dir2', 'file2'), + path.join('a.asar', 'dir2', 'file3'), + path.join('a.asar', 'dir3', 'file1'), + path.join('a.asar', 'dir3', 'file2'), + path.join('a.asar', 'dir3', 'file3') + ]); + }); + itremote('supports withFileTypes', async () => { const p = path.join(asarDir, 'a.asar'); @@ -1008,6 +1071,37 @@ describe('asar package', function () { expect(dirs).to.deep.equal(['dir1', 'dir2', 'dir3', 'file1', 'file2', 'file3', 'link1', 'link2', 'ping.js']); }); + itremote('supports recursive readdir', async () => { + const dir = path.join(fixtures, 'recursive-asar'); + const files = await fs.promises.readdir(dir, { recursive: true }); + expect(files).to.have.members([ + 'a.asar', + 'nested', + 'test.txt', + path.join('a.asar', 'dir1'), + path.join('a.asar', 'dir2'), + path.join('a.asar', 'dir3'), + path.join('a.asar', 'file1'), + path.join('a.asar', 'file2'), + path.join('a.asar', 'file3'), + path.join('a.asar', 'link1'), + path.join('a.asar', 'link2'), + path.join('a.asar', 'ping.js'), + path.join('nested', 'hello.txt'), + path.join('a.asar', 'dir1', 'file1'), + path.join('a.asar', 'dir1', 'file2'), + path.join('a.asar', 'dir1', 'file3'), + path.join('a.asar', 'dir1', 'link1'), + path.join('a.asar', 'dir1', 'link2'), + path.join('a.asar', 'dir2', 'file1'), + path.join('a.asar', 'dir2', 'file2'), + path.join('a.asar', 'dir2', 'file3'), + path.join('a.asar', 'dir3', 'file1'), + path.join('a.asar', 'dir3', 'file2'), + path.join('a.asar', 'dir3', 'file3') + ]); + }); + itremote('supports withFileTypes', async function () { const p = path.join(asarDir, 'a.asar'); const dirs = await fs.promises.readdir(p, { withFileTypes: true }); diff --git a/spec/fixtures/recursive-asar/a.asar b/spec/fixtures/recursive-asar/a.asar new file mode 100644 index 0000000000000000000000000000000000000000..852f460b6d5482b341374992aa3c0a76971904de GIT binary patch literal 3458 zcmdT`J#X7E5Y^Br=#Mb8Bms4*WRUYHT^MT%fJt-`S-@e@ZFF4FaSn zEpYNc>Ufm&_`r7$_pWK0-|w5|R~kRlIJB`_cAZLc|2eu0}Z?`cH zxNjFNY3FU%?{T^7_TR^k?0UIP@oJMcA3lCyQl&Rmc37n~`rxJHDgt-GqSjPJf>sb< z3Zq2hG$p}MAq*!V4dKoiq8JYb91S}C$*s5k@gW~WP$M`Ir(8Pur+R?T$KyQnW+Ruk zbVo90(^Y^d1!%N3IKqXq(ikDo830I(gz;8cs{@xHq=v{9I`0sjk`V|N=~e^0(B=Q8 zOW72@Mnr{DyFi2$l5(dxp$IYxA{qDyiW09>phgkLR8-b81i_J#^pW*NzSRJk@4D+B zvzo^ssq$If>E6e6zW38cu7l&Kh!o|Gb1H(4GCf-(mtPW0OyOHC*^&iR1C(llPqMsf z$(Hz510=pDiDwfyN8)^v4PL&)#UvZ7R$%GMs{u+97n3ZkT7f0L)c_}n|EG)skN@1-YWAv?he<9M2nQc5 zxCquLj}VB`7(GNM7?V;*2*{kXTw6n(3lSJG1kg#sjgG};uNoj%eUpp+YMfb~d7kOy z%+qe`aTxBl{mu0z1$K$o^I81b`+b*kn_+o7J6z3Xf8QIh4>^1PDv95^{SCRjzdEMz E2VV&4O#lD@ literal 0 HcmV?d00001 diff --git a/spec/fixtures/recursive-asar/nested/hello.txt b/spec/fixtures/recursive-asar/nested/hello.txt new file mode 100644 index 0000000000..ec68ffb2b6 --- /dev/null +++ b/spec/fixtures/recursive-asar/nested/hello.txt @@ -0,0 +1 @@ +goodbye! \ No newline at end of file diff --git a/spec/fixtures/recursive-asar/test.txt b/spec/fixtures/recursive-asar/test.txt new file mode 100644 index 0000000000..05a682bd4e --- /dev/null +++ b/spec/fixtures/recursive-asar/test.txt @@ -0,0 +1 @@ +Hello! \ No newline at end of file