diff --git a/tools/isobuild/compiler-plugin.js b/tools/isobuild/compiler-plugin.js index 4c6e2f70f3..e5c2caffda 100644 --- a/tools/isobuild/compiler-plugin.js +++ b/tools/isobuild/compiler-plugin.js @@ -597,6 +597,7 @@ class PackageSourceBatch { const linkerOptions = { useGlobalNamespace: isApp, sourceRoot: self.sourceRoot, + nodeModulesPath: self.unibuild.nodeModulesPath, // I was confused about this, so I am leaving a comment -- the // combinedServePath is either [pkgname].js or [pluginName]:plugin.js. // XXX: If we change this, we can get rid of source arch names! @@ -606,6 +607,7 @@ class PackageSourceBatch { (self.unibuild.kind === "main" ? "" : (":" + self.unibuild.kind)) + ".js"), name: self.unibuild.pkg.name || null, + bundleArch, declaredExports: _.pluck(self.unibuild.declaredExports, 'name'), imports: self.importedSymbolToPackageName, usedPackageNames: self.usedPackageNames, diff --git a/tools/isobuild/compiler.js b/tools/isobuild/compiler.js index 1185327295..b2bfd40545 100644 --- a/tools/isobuild/compiler.js +++ b/tools/isobuild/compiler.js @@ -589,8 +589,8 @@ api.addAssets('${relPath}', 'client').`); } let nodeModulesPathOrUndefined = nodeModulesPath; - if (! archinfo.matches(arch, "os")) { - // npm modules only work on server architectures + if (! archinfo.matches(arch, "os") && ! isPortable) { + // non-portable npm modules only work on server architectures nodeModulesPathOrUndefined = undefined; } diff --git a/tools/isobuild/import-scanner.js b/tools/isobuild/import-scanner.js index 757776e2cb..852914001b 100644 --- a/tools/isobuild/import-scanner.js +++ b/tools/isobuild/import-scanner.js @@ -1,37 +1,39 @@ import assert from "assert"; import {isString, has, keys, each, without} from "underscore"; import {sha1} from "../fs/watch.js"; +import {matches as archMatches} from "../utils/archinfo.js"; import {findImportedModuleIdentifiers} from "./js-analyze.js"; import { pathJoin, pathRelative, pathNormalize, pathDirname, + pathBasename, + pathExtname, statOrNull, readFile, + convertToPosixPath, } from "../fs/files.js"; export default class ImportScanner { constructor({ name, + bundleArch, sourceRoot, - extensions, + extensions = [".js", ".json"], usedPackageNames = {}, + nodeModulesPath, }) { assert.ok(isString(sourceRoot)); this.name = name; + this.bundleArch = bundleArch; this.sourceRoot = sourceRoot; this.usedPackageNames = usedPackageNames; + this.nodeModulesPath = nodeModulesPath; this.absPathToOutputIndex = {}; this.outputFiles = []; - - if (extensions) { - this.extensions = without(extensions, ""); - this.extensions.unshift(""); - } else { - this.extensions = ["", ".js", ".json"]; - } + this.extensions = extensions; } addInputFiles(files) { @@ -42,6 +44,12 @@ export default class ImportScanner { // in the bundle if they are actually imported. file.lazy = this._isFileLazy(file); + // Files that are eagerly evaluated are effectively "imported" as + // entry points. + file.imported = ! file.lazy; + + file.installPath = this._getInstallInfo(absPath).path; + if (has(this.absPathToOutputIndex, absPath)) { const index = this.absPathToOutputIndex[absPath]; this.outputFiles[index] = file; @@ -55,7 +63,11 @@ export default class ImportScanner { } getOutputFiles() { - this.outputFiles.forEach(this._scanFile, this); + this.outputFiles.forEach(file => { + const absPath = pathJoin(this.sourceRoot, file.sourcePath); + file.deps = this._scanDeps(absPath, file.data) + }); + return this.outputFiles; } @@ -77,47 +89,177 @@ export default class ImportScanner { ).indexOf("imports") >= 0; } - _scanFile(file) { - if (file.deps) { - return; - } + _scanDeps(absPath, data) { + const deps = keys(findImportedModuleIdentifiers(data.toString("utf8"))); - const data = file.data.toString("utf8"); - const absFilePath = pathJoin(this.sourceRoot, file.sourcePath); + each(deps, id => { + const absImportedPath = this._tryToResolveImportedPath(id, absPath); + if (! absImportedPath) { + return; + } - file.deps = keys(findImportedModuleIdentifiers(data)); + if (has(this.absPathToOutputIndex, absImportedPath)) { + // Avoid scanning files that we've scanned before, but mark them + // as imported so we know to include them in the bundle if they + // are lazy. + const index = this.absPathToOutputIndex[absImportedPath]; + this.outputFiles[index].imported = true; + return; + } - each(file.deps, id => { - const absImportedPath = this._tryToResolveImportedPath(id, absFilePath); - if (! absImportedPath || - // Avoid scanning files that we've scanned before. - has(this.absPathToOutputIndex, absImportedPath)) { + const installInfo = this._getInstallInfo(absImportedPath); + if (! installInfo) { + // The given path cannot be installed on this architecture. + return; + } + + if (! installInfo.inNodeModules) { + // At this point, if the file is not in a node_modules directory, + // and it was not part of the set of input files, then we can + // conclude there was no JS output for it on this architecture, so + // we should not try to add it to the bundle. return; } var relImportedPath = pathRelative(this.sourceRoot, absImportedPath); - // TODO Disallow files outside this.sourceRoot in a way that works - // between isopacket directories and package source directories. - - // TODO If the dependency is eagerly evaluated on a different - // architecture, but not on this architecture, then ignore it and - // warn the developer. - - const depFile = Object.create(Object.getPrototypeOf(file)); - depFile.type = "js"; // TODO Is this correct? - depFile.data = readFile(absImportedPath); - depFile.sourcePath = relImportedPath; - depFile.servePath = relImportedPath; - depFile.hash = sha1(depFile.data); - depFile.lazy = true; + const depData = readFile(absImportedPath); + const depFile = { + type: "js", // TODO Is this correct? + data: depData, + sourcePath: relImportedPath, + installPath: installInfo.path, + servePath: installInfo.path, + hash: sha1(depData), + lazy: true, + imported: true, + }; // Append this file to the output array and record its index. this.absPathToOutputIndex[absImportedPath] = this.outputFiles.push(depFile) - 1; - this._scanFile(depFile); + depFile.deps = this._scanDeps(absImportedPath, depFile.data); }); + + return deps; + } + + // Returns a { installPath, inNodeModules } record indicating where to + // install the given file via meteorInstall, and whether it resides in a + // node_modules directory. May return undefined if the file should not + // be installed on the current architecture. + _getInstallInfo(absPath) { + const info = + this._getNodeModulesInstallInfo(absPath) || + this._getSourceRootInstallInfo(absPath); + + if (! info) { + return; + } + + if (this.name) { + // If we're bundling a package, prefix info.path with + // node_modules//. + info.path = pathJoin("node_modules", this.name, info.path); + } else { + // If we're bundling an app, prefix info.path with app/. + info.path = pathJoin("app", info.path); + } + + // Note that info.inNodeModules may be false even if info.path now + // contains a node_modules directory. + return info; + } + + _getNodeModulesInstallInfo(absPath) { + if (this.nodeModulesPath) { + const relPathWithinNodeModules = + pathRelative(this.nodeModulesPath, absPath); + + if (relPathWithinNodeModules.startsWith("..")) { + // absPath is not a subdirectory of this.nodeModulesPath. + return; + } + + if (! this._hasKnownExtension(relPathWithinNodeModules)) { + // Only accept files within node_modules directories if they + // have one of the known extensions. + return; + } + + return { + // Install the module into the local node_modules directory within + // this app or package. + path: pathJoin("node_modules", relPathWithinNodeModules), + // The original path was contained by a node_modules directory. + inNodeModules: true, + }; + } + } + + _getSourceRootInstallInfo(absPath) { + const installPath = pathRelative(this.sourceRoot, absPath); + + if (installPath.startsWith("..")) { + // absPath is not a subdirectory of this.sourceRoot. + return; + } + + const dirs = this._splitPath(pathDirname(installPath)); + const bundlingClientApp = + ! this.name && // Indicates we are bundling an app. + archMatches(this.bundleArch, "web"); + + for (let dir of dirs) { + if (dir.charAt(0) === "." || + dir === "packages" || + dir === "programs" || + dir === "cordova-build-override") { + // These directories are never loaded as part of an app. + return; + } + + if (bundlingClientApp && (dir === "server" || + dir === "private")) { + // If we're bundling an app for a client architecture, any files + // contained by a server-only directory that is not contained by + // a node_modules directory must be ignored. + return; + } + + if (dir === "node_modules") { + if (! this._hasKnownExtension(installPath)) { + // Reject any files within node_modules directories that do + // not have one of the known extensions. + return; + } + + // Accept any file within a node_modules directory if it has a + // known file extension. + return { + path: installPath, + inNodeModules: true, + }; + } + } + + return { + path: installPath, + inNodeModules: false, + }; + } + + _hasKnownExtension(path) { + return this.extensions.indexOf(pathExtname(path)) >= 0; + } + + _splitPath(path) { + const partsInReverse = []; + for (let dir; (dir = pathDirname(path)) !== path; path = dir) { + partsInReverse.push(pathBasename(path)); + } + return partsInReverse.reverse(); } _tryToResolveImportedPath(id, path) { @@ -136,6 +278,12 @@ export default class ImportScanner { _joinAndStat(...joinArgs) { const path = pathNormalize(pathJoin(...joinArgs)); + const exactStat = statOrNull(path); + const exactResult = exactStat && { path, stat: exactStat }; + if (exactResult && exactStat.isFile()) { + return exactResult; + } + for (let ext of this.extensions) { const pathWithExt = path + ext; const stat = statOrNull(pathWithExt); @@ -143,6 +291,13 @@ export default class ImportScanner { return { path: pathWithExt, stat }; } } + + if (exactResult && exactStat.isDirectory()) { + // After trying all available file extensions, fall back to the + // original result if it was a directory. + return exactResult; + } + return null; } @@ -168,6 +323,12 @@ export default class ImportScanner { dir = pathDirname(dir); resolved = this._joinAndStat(dir, "node_modules", id); } while (! resolved && dir !== this.sourceRoot); + + if (! resolved && this.nodeModulesPath) { + // After checking any local node_modules directories, fall back to + // the package NPM directory, if one was specified. + resolved = this._joinAndStat(this.nodeModulesPath, id); + } } return resolved; @@ -217,9 +378,11 @@ export default class ImportScanner { data, deps: [], // Avoid accidentally re-scanning this file. sourcePath: relPkgJsonPath, + installPath: this._getInstallInfo(pkgJsonPath).path, servePath: relPkgJsonPath, hash: sha1(data), lazy: true, + imported: true, }; this.absPathToOutputIndex[pkgJsonPath] = @@ -255,6 +418,7 @@ export default class ImportScanner { installPath: relPkgPath, servePath: relPkgPath, lazy: true, + imported: true, }; this.absPathToOutputIndex[absPkgPath] = diff --git a/tools/isobuild/linker.js b/tools/isobuild/linker.js index 3b75a1590e..e1af8cfd02 100644 --- a/tools/isobuild/linker.js +++ b/tools/isobuild/linker.js @@ -166,6 +166,13 @@ _.extend(Module.prototype, { return; } + if (file.lazy && ! file.imported) { + // If the file is not eagerly evaluated, and no other files + // import or require it, then it need not be included in the + // bundle. + return; + } + const parts = file.installPath.split("/"); let t = tree; _.each(parts, function (part, i) { @@ -339,12 +346,9 @@ var File = function (inputFile, module) { self.sourcePath = inputFile.sourcePath; // Absolute module identifier to use when installing this file via - // meteorInstall. - self.installPath = files.convertToPosixPath( - module.name - ? files.pathJoin("node_modules", module.name, inputFile.sourcePath) - : inputFile.sourcePath - ); + // meteorInstall. If the inputFile has no .installPath, then this file + // cannot be installed as a module. + self.installPath = inputFile.installPath || null; // the path where this file would prefer to be served if possible self.servePath = inputFile.servePath; @@ -355,6 +359,10 @@ var File = function (inputFile, module) { // True if the input file should not be evaluated eagerly. self.lazy = !!inputFile.lazy; + // True if the file is an eagerly evaluated entry point, or if some + // other file imports or requires it. + self.imported = !!inputFile.imported; + // If true, don't wrap this individual file in a closure. self.bare = !!inputFile.bare; @@ -849,6 +857,8 @@ export var fullLink = Profile("linker.fullLink", function (inputFiles, { // imports of other modules); null if the module has no name (in that // case exports will not work properly) name, + // The architecture for which this bundle is to be linked. + bundleArch, // An array of symbols that the module exports. Symbols are // {name,testOnly} pairs. declaredExports, @@ -867,7 +877,11 @@ export var fullLink = Profile("linker.fullLink", function (inputFiles, { // Absolute path of the package or application root directory. Can be // joined with the .path of an input file to determine the absolute path // of the file. - sourceRoot + sourceRoot, + // Absolute path to the node_modules directory to use at runtime to + // resolve require() calls for this package, or null if we're not + // linking a package, or if this package has no node_modules. + nodeModulesPath, }) { buildmessage.assertInJob(); @@ -886,8 +900,10 @@ export var fullLink = Profile("linker.fullLink", function (inputFiles, { if (useMeteorInstall) { inputFiles = new ImportScanner({ name, + bundleArch, sourceRoot, usedPackageNames, + nodeModulesPath, }).addInputFiles(inputFiles) .getOutputFiles(); }