From d01d7d5f8d34243500fca9078b9f74288becb1dc Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Wed, 22 Jul 2015 10:28:45 -0700 Subject: [PATCH 01/10] Replace PowerShell operations by cmd to avoid long directory name errors --- scripts/generate-dev-bundle.ps1 | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/scripts/generate-dev-bundle.ps1 b/scripts/generate-dev-bundle.ps1 index 963cfa9b75..cace2b5721 100644 --- a/scripts/generate-dev-bundle.ps1 +++ b/scripts/generate-dev-bundle.ps1 @@ -20,10 +20,7 @@ $BUNDLE_VERSION = $BUNDLE_VERSION.Trim() $DIR = $script_path + "\gdbXXX" echo $DIR -# removing folders isn't easy on Windows, try both commands -rm -Recurse -Force "${DIR}" -cmd /C "rmdir /S /Q ${DIR}" - +cmd /c rmdir "$DIR" /s /q mkdir "$DIR" cd "$DIR" @@ -40,7 +37,7 @@ npm install npm shrinkwrap mkdir -Force "${DIR}\server-lib\node_modules" -cp -R "${DIR}\b\t\node_modules\*" "${DIR}\server-lib\node_modules\" +cmd /c robocopy "${DIR}\b\t\node_modules" "${DIR}\server-lib\node_modules" /e /nfl /ndl mkdir -Force "${DIR}\etc" Move-Item package.json "${DIR}\etc\" @@ -54,12 +51,9 @@ npm dedupe # install the latest flatten-packages npm install -g flatten-packages flatten-packages . -cp -R "${DIR}\b\p\node_modules\" "${DIR}\lib\node_modules\" +cmd /c robocopy "${DIR}\b\p\node_modules" "${DIR}\lib\node_modules" /e /nfl /ndl cd "$DIR" - -# deleting folders is hard so we try twice -rm -Recurse -Force "${DIR}\b" -cmd /C "rmdir /s /q $DIR\b" +cmd /c rmdir "${DIR}\b" /s /q cd "$DIR" mkdir "$DIR\mongodb" @@ -122,14 +116,11 @@ echo "${BUNDLE_VERSION}" | Out-File .bundle_version.txt -Encoding ascii cd "$DIR\.." # rename and move the folder with the devbundle -# XXX this can generate a path that is too long -Move-Item "$DIR" "dev_bundle_${PLATFORM}_${BUNDLE_VERSION}" +cmd /c robocopy "$DIR" "dev_bundle_${PLATFORM}_${BUNDLE_VERSION}" /e /move /nfl /ndl cmd /c 7z.exe a -ttar dev_bundle.tar "dev_bundle_${PLATFORM}_${BUNDLE_VERSION}" cmd /c 7z.exe a -tgzip "${CHECKOUT_DIR}\dev_bundle_${PLATFORM}_${BUNDLE_VERSION}.tar.gz" dev_bundle.tar del dev_bundle.tar -rm -Recurse -Force "dev_bundle_${PLATFORM}_${BUNDLE_VERSION}" -cmd /C "rmdir /s /q dev_bundle_${PLATFORM}_${BUNDLE_VERSION}" +cmd /c rmdir "dev_bundle_${PLATFORM}_${BUNDLE_VERSION}" /s /q echo "Done building Dev Bundle!" - From 10fe112959758298e40d2ad50c3a9485571910d2 Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Wed, 22 Jul 2015 11:19:19 -0700 Subject: [PATCH 02/10] To be removed path no longer exists --- scripts/generate-dev-bundle.ps1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/generate-dev-bundle.ps1 b/scripts/generate-dev-bundle.ps1 index cace2b5721..bae948aced 100644 --- a/scripts/generate-dev-bundle.ps1 +++ b/scripts/generate-dev-bundle.ps1 @@ -101,9 +101,6 @@ flatten-packages . cd node_modules\npm npm install node-gyp -# this path is too long -rm -Recurse -Force "node_modules\node-gyp\node_modules\request\node_modules\combined-stream\node_modules\delayed-stream\test" - cd ..\.. cp node_modules\npm\bin\npm.cmd . From 1d862f1437b60c9471c05242ac455bdd86b75dd4 Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Wed, 22 Jul 2015 11:28:25 -0700 Subject: [PATCH 03/10] Fix check for PLATFORM env variable --- scripts/generate-dev-bundle.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate-dev-bundle.ps1 b/scripts/generate-dev-bundle.ps1 index bae948aced..8c4b0e3fc7 100644 --- a/scripts/generate-dev-bundle.ps1 +++ b/scripts/generate-dev-bundle.ps1 @@ -5,7 +5,7 @@ $MONGO_VERSION = "2.6.7" $NODE_VERSION = "0.10.40" # take it form the environment if exists -if (Test-Path variable:global:PLATFORM) { +if (Test-Path env:PLATFORM) { $PLATFORM = (Get-Item env:PLATFORM).Value } From b6cb554364406a402f771ed52f75ce4f2426d207 Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Wed, 22 Jul 2015 19:25:23 -0700 Subject: [PATCH 04/10] Install npm 3 to avoid having to flatten packages manually --- scripts/generate-dev-bundle.ps1 | 59 ++++++++++++++++----------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/scripts/generate-dev-bundle.ps1 b/scripts/generate-dev-bundle.ps1 index 8c4b0e3fc7..4c7826c49f 100644 --- a/scripts/generate-dev-bundle.ps1 +++ b/scripts/generate-dev-bundle.ps1 @@ -24,12 +24,36 @@ cmd /c rmdir "$DIR" /s /q mkdir "$DIR" cd "$DIR" +mkdir bin +cd bin + +$webclient = New-Object System.Net.WebClient + +# download node +# same node on 32bit vs 64bit? +$node_link = "http://nodejs.org/dist/v${NODE_VERSION}/node.exe" +$webclient.DownloadFile($node_link, "$DIR\bin\node.exe") + +# install npm +echo "{}" | Out-File package.json -Encoding ascii # otherwise it doesn't install in local dir +npm install npm@3.0-latest --save + +# npm depends on a hardcoded file path to node-gyp, so we need this to be +# un-flattened +cd node_modules\npm +npm install node-gyp +cd ..\.. + +cp node_modules\npm\bin\npm.cmd . + +# add bin to the front of the path so we can use our own node for building +$env:PATH = "${DIR}\bin;${env:PATH}" + # install dev-bundle-package.json # use short folder names -mkdir b # for build -cd b -mkdir t -cd t +# b for build +mkdir "$DIR\b\t" +cd "$DIR\b\t" npm config set loglevel error node "${CHECKOUT_DIR}\scripts\dev-bundle-server-package.js" | Out-File -FilePath package.json -Encoding ascii @@ -47,10 +71,6 @@ mkdir -Force "${DIR}\b\p" cd "${DIR}\b\p" node "${CHECKOUT_DIR}\scripts\dev-bundle-tool-package.js" | Out-File -FilePath package.json -Encoding ascii npm install -npm dedupe -# install the latest flatten-packages -npm install -g flatten-packages -flatten-packages . cmd /c robocopy "${DIR}\b\p\node_modules" "${DIR}\lib\node_modules" /e /nfl /ndl cd "$DIR" cmd /c rmdir "${DIR}\b" /s /q @@ -59,8 +79,6 @@ cd "$DIR" mkdir "$DIR\mongodb" mkdir "$DIR\mongodb\bin" -$webclient = New-Object System.Net.WebClient - # download Mongo $mongo_name = "mongodb-win32-i386-${MONGO_VERSION}" If ($PLATFORM -eq 'windows_x86_64') { @@ -84,27 +102,6 @@ cp "$DIR\mongodb\$mongo_name\bin\mongo.exe" $DIR\mongodb\bin rm -Recurse -Force $mongo_zip rm -Recurse -Force "$DIR\mongodb\$mongo_name" -mkdir bin -cd bin - -# download node -# same node on 32bit vs 64bit? -$node_link = "http://nodejs.org/dist/v${NODE_VERSION}/node.exe" -$webclient.DownloadFile($node_link, "$DIR\bin\node.exe") -# install npm -echo "{}" | Out-File package.json -Encoding ascii # otherwise it doesn't install in local dir -npm install npm --save -flatten-packages . - -# npm depends on a hardcoded file path to node-gyp, so we need this to be -# un-flattened -cd node_modules\npm -npm install node-gyp - -cd ..\.. - -cp node_modules\npm\bin\npm.cmd . - cd $DIR # mark the version From cb9111a97859213f7a3f7670ca7e08c31a9c7832 Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Wed, 22 Jul 2015 21:26:30 -0700 Subject: [PATCH 05/10] Don't put temp directory for generating dev bundle under scripts, but one level up --- scripts/generate-dev-bundle.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/generate-dev-bundle.ps1 b/scripts/generate-dev-bundle.ps1 index 4c7826c49f..2743ea70d0 100644 --- a/scripts/generate-dev-bundle.ps1 +++ b/scripts/generate-dev-bundle.ps1 @@ -17,7 +17,8 @@ $BUNDLE_VERSION = Select-String -Path ($CHECKOUT_DIR + "\meteor") -Pattern 'BUND $BUNDLE_VERSION = $BUNDLE_VERSION.Trim() # generate-dev-bundle-xxxxxxxx shortly -$DIR = $script_path + "\gdbXXX" +# convert relative path to absolute path because not all commands know how to deal with this themselves +$DIR = $executionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("${script_path}\..\gdbXXX") echo $DIR cmd /c rmdir "$DIR" /s /q From 01afd89a595a9f2c0787a5de7367707bc95fa69f Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Thu, 23 Jul 2015 10:24:50 -0700 Subject: [PATCH 06/10] Install initial version of npm from dist and only use npm for building dev bundle --- scripts/generate-dev-bundle.ps1 | 35 +++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/scripts/generate-dev-bundle.ps1 b/scripts/generate-dev-bundle.ps1 index 2743ea70d0..0bb5daa78d 100644 --- a/scripts/generate-dev-bundle.ps1 +++ b/scripts/generate-dev-bundle.ps1 @@ -3,6 +3,7 @@ $PLATFORM = "windows_x86" $MONGO_VERSION = "2.6.7" $NODE_VERSION = "0.10.40" +$NPM_VERSION = "1.4.9" # take it form the environment if exists if (Test-Path env:PLATFORM) { @@ -29,26 +30,42 @@ mkdir bin cd bin $webclient = New-Object System.Net.WebClient +$shell = New-Object -com shell.application # download node # same node on 32bit vs 64bit? $node_link = "http://nodejs.org/dist/v${NODE_VERSION}/node.exe" $webclient.DownloadFile($node_link, "$DIR\bin\node.exe") -# install npm +# download initial version of npm +$npm_zip = "$DIR\bin\npm.zip" +$npm_link = "https://nodejs.org/dist/npm/npm-${NPM_VERSION}.zip" +$webclient.DownloadFile($npm_link, $npm_zip) + +$zip = $shell.NameSpace($npm_zip) +foreach($item in $zip.items()) { + $shell.Namespace("$DIR\bin").copyhere($item, 0x14) # 0x10 - overwrite, 0x4 - no dialog +} + +rm -Recurse -Force $npm_zip + +# add bin to the front of the path so we can use our own node for building +$env:PATH = "${DIR}\bin;${env:PATH}" + +mkdir "${DIR}\bin\npm3" +cd "${DIR}\bin\npm3" echo "{}" | Out-File package.json -Encoding ascii # otherwise it doesn't install in local dir -npm install npm@3.0-latest --save +npm install npm@3.0-latest + +# add bin\npm3 to the front of the path so we can use npm 3 for building +$env:PATH = "${DIR}\bin\npm3;${env:PATH}" # npm depends on a hardcoded file path to node-gyp, so we need this to be # un-flattened cd node_modules\npm npm install node-gyp cd ..\.. - -cp node_modules\npm\bin\npm.cmd . - -# add bin to the front of the path so we can use our own node for building -$env:PATH = "${DIR}\bin;${env:PATH}" +cp node_modules\npm\bin\npm.cmd # install dev-bundle-package.json # use short folder names @@ -91,7 +108,6 @@ $mongo_zip = "$DIR\mongodb\mongo.zip" $webclient.DownloadFile($mongo_link, $mongo_zip) -$shell = New-Object -com shell.application $zip = $shell.NameSpace($mongo_zip) foreach($item in $zip.items()) { $shell.Namespace("$DIR\mongodb").copyhere($item, 0x14) # 0x10 - overwrite, 0x4 - no dialog @@ -103,6 +119,9 @@ cp "$DIR\mongodb\$mongo_name\bin\mongo.exe" $DIR\mongodb\bin rm -Recurse -Force $mongo_zip rm -Recurse -Force "$DIR\mongodb\$mongo_name" +# Remove npm 3 before we package the dev bundle +rm -Recurse -Force "${DIR}\bin\npm3" + cd $DIR # mark the version From 4d6a5fe795bb0e2253d2f6d7f6a7b6affe8e112b Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Thu, 23 Jul 2015 11:02:18 -0700 Subject: [PATCH 07/10] Use console.log instead of Console.debug when cleaning up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For some reason, Console was undefined here… --- tools/files.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/files.js b/tools/files.js index d2a63c56b6..d7f6a58bd4 100644 --- a/tools/files.js +++ b/tools/files.js @@ -23,7 +23,6 @@ var buildmessage = require('./buildmessage.js'); var watch = require('./watch.js'); var fiberHelpers = require('./fiber-helpers.js'); var colonConverter = require("./colon-converter.js"); -var Console = require("./console.js").Console; var miniFiles = require("./server/mini-files.js"); @@ -643,7 +642,7 @@ files.freeTempDir = function (tempDir) { // Don't crash and print a stack trace because we failed to delete a temp // directory. This happens sometimes on Windows and seems to be // unavoidable. - Console.debug(err); + console.log(err); } tempDirs = _.without(tempDirs, tempDir); @@ -659,7 +658,7 @@ if (! process.env.METEOR_SAVE_TMPDIRS) { // Don't crash and print a stack trace because we failed to delete a temp // directory. This happens sometimes on Windows and seems to be // unavoidable. - Console.debug(err); + console.log(err); } }); From 6038c8dc00bc0fa4ad50138274481d3375f04e5a Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Thu, 23 Jul 2015 12:16:31 -0700 Subject: [PATCH 08/10] Use npm 3 for building dev bundle on Unix --- scripts/generate-dev-bundle.sh | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/scripts/generate-dev-bundle.sh b/scripts/generate-dev-bundle.sh index fd3d67f168..eb827cfb9b 100755 --- a/scripts/generate-dev-bundle.sh +++ b/scripts/generate-dev-bundle.sh @@ -67,10 +67,19 @@ else curl "${MONGO_URL}" | tar zx fi -cd "$DIR/build" - -# export path so we use our new node for later builds +# export path so we use the downloaded node and npm export PATH="$DIR/bin:$PATH" + +# install npm 3 in a temporary directory +mkdir "$DIR/bin/npm3" +cd "$DIR/bin/npm3" +npm install npm@3.0-latest +cp node_modules/npm/bin/npm . + +# export path again with our temporary npm3 directory first, +# so we can use npm 3 during builds +export PATH="$DIR/bin/npm3:$PATH" + which node which npm @@ -94,7 +103,6 @@ mkdir -p "${DIR}/server-lib/node_modules" # This ignores the stuff in node_modules/.bin, but that's OK. cp -R node_modules/* "${DIR}/server-lib/node_modules/" -mkdir "${DIR}/etc" mv package.json npm-shrinkwrap.json "${DIR}/etc/" # Fibers ships with compiled versions of its C code for a dozen platforms. This @@ -116,8 +124,6 @@ mkdir "${DIR}/build/npm-tool-install" cd "${DIR}/build/npm-tool-install" node "${CHECKOUT_DIR}/scripts/dev-bundle-tool-package.js" >package.json npm install -# Refactor node modules to top level and remove unnecessary duplicates. -npm dedupe cp -R node_modules/* "${DIR}/lib/node_modules/" cd "${DIR}/lib" @@ -143,19 +149,6 @@ delete sqlite3/deps delete wordwrap/test delete moment/min -# dedupe isn't good enough to eliminate 3 copies of esprima, sigh. -find . -path '*/esprima/test' | xargs rm -rf -find . -path '*/esprima-fb/test' | xargs rm -rf - -# dedupe isn't good enough to eliminate 4 copies of JSONstream, sigh. -find . -path '*/JSONStream/test/fixtures' | xargs rm -rf - -# Not sure why dedupe doesn't lift these to the top. -pushd cordova/node_modules/cordova-lib/node_modules/cordova-js/node_modules/browserify/node_modules -delete crypto-browserify/test -delete umd/node_modules/ruglify/test -popd - cd "$DIR/lib/node_modules/fibers/bin" shrink_fibers @@ -168,6 +161,9 @@ gunzip BrowserStackLocal* mv BrowserStackLocal* BrowserStackLocal mv BrowserStackLocal "$DIR/bin/" +# remove our temporary npm3 directory +rm -rf "$DIR/bin/npm3" + echo BUNDLING cd "$DIR" From fa38facc838c287014ddef0148ce9be0b49e893f Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Thu, 23 Jul 2015 14:55:41 -0700 Subject: [PATCH 09/10] Specify explicit dependency on npm to avoid installing earlier version The cordova dependency pulled in npm 1.3.4, which apparently has a bug reading npm-shrinkwrap.json with tarball dependencies. --- scripts/dev-bundle-tool-package.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/dev-bundle-tool-package.js b/scripts/dev-bundle-tool-package.js index 7531621a34..1b4065a9e1 100644 --- a/scripts/dev-bundle-tool-package.js +++ b/scripts/dev-bundle-tool-package.js @@ -20,6 +20,7 @@ var packageJson = { // Version is not important but is needed to prevent warnings. version: "0.0.0", dependencies: { + npm: "1.4.28", fibers: fibersVersion, "meteor-babel": "0.4.4", "meteor-promise": "0.4.0", From e07143b2ca4adba2ce79cf88bb969be0c38e35c2 Mon Sep 17 00:00:00 2001 From: Martijn Walraven Date: Thu, 23 Jul 2015 16:08:36 -0700 Subject: [PATCH 10/10] Save some space by removing esprima tests (as we were doing before) --- scripts/generate-dev-bundle.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/generate-dev-bundle.sh b/scripts/generate-dev-bundle.sh index eb827cfb9b..9b8882ec31 100755 --- a/scripts/generate-dev-bundle.sh +++ b/scripts/generate-dev-bundle.sh @@ -149,6 +149,9 @@ delete sqlite3/deps delete wordwrap/test delete moment/min +# Remove esprima tests to reduce the size of the dev bundle +find . -path '*/esprima-fb/test' | xargs rm -rf + cd "$DIR/lib/node_modules/fibers/bin" shrink_fibers