Merge branch 'npm3-dev-bundle' into devel

scripts/dev-bundle-tool-package.js

@@ -20,6 +20,7 @@ var packageJson = {
   // Version is not important but is needed to prevent warnings.
   version: "0.0.0",
   dependencies: {
+    npm: "1.4.28",
     fibers: fibersVersion,
     "meteor-babel": "0.4.4",
     "meteor-promise": "0.4.0",

scripts/generate-dev-bundle.ps1

@@ -3,9 +3,10 @@
 $PLATFORM = "windows_x86"
 $MONGO_VERSION = "2.6.7"
 $NODE_VERSION = "0.10.40"
+$NPM_VERSION = "1.4.9"
 
 # take it form the environment if exists
-if (Test-Path variable:global:PLATFORM) {
+if (Test-Path env:PLATFORM) {
   $PLATFORM = (Get-Item env:PLATFORM).Value
 }
 
@@ -17,22 +18,60 @@ $BUNDLE_VERSION = Select-String -Path ($CHECKOUT_DIR + "\meteor") -Pattern 'BUND
 $BUNDLE_VERSION = $BUNDLE_VERSION.Trim()
 
 # generate-dev-bundle-xxxxxxxx shortly
-$DIR = $script_path + "\gdbXXX"
+# convert relative path to absolute path because not all commands know how to deal with this themselves
+$DIR = $executionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath("${script_path}\..\gdbXXX")
 echo $DIR
 
-# removing folders isn't easy on Windows, try both commands
-rm -Recurse -Force "${DIR}"
-cmd /C "rmdir /S /Q ${DIR}"
-
+cmd /c rmdir "$DIR" /s /q
 mkdir "$DIR"
 cd "$DIR"
 
+mkdir bin
+cd bin
+
+$webclient = New-Object System.Net.WebClient
+$shell = New-Object -com shell.application
+
+# download node
+# same node on 32bit vs 64bit?
+$node_link = "http://nodejs.org/dist/v${NODE_VERSION}/node.exe"
+$webclient.DownloadFile($node_link, "$DIR\bin\node.exe")
+
+# download initial version of npm
+$npm_zip = "$DIR\bin\npm.zip"
+$npm_link = "https://nodejs.org/dist/npm/npm-${NPM_VERSION}.zip"
+$webclient.DownloadFile($npm_link, $npm_zip)
+
+$zip = $shell.NameSpace($npm_zip)
+foreach($item in $zip.items()) {
+  $shell.Namespace("$DIR\bin").copyhere($item, 0x14) # 0x10 - overwrite, 0x4 - no dialog
+}
+
+rm -Recurse -Force $npm_zip
+
+# add bin to the front of the path so we can use our own node for building
+$env:PATH = "${DIR}\bin;${env:PATH}"
+
+mkdir "${DIR}\bin\npm3"
+cd "${DIR}\bin\npm3"
+echo "{}" | Out-File package.json -Encoding ascii # otherwise it doesn't install in local dir
+npm install npm@3.0-latest
+
+# add bin\npm3 to the front of the path so we can use npm 3 for building
+$env:PATH = "${DIR}\bin\npm3;${env:PATH}"
+
+# npm depends on a hardcoded file path to node-gyp, so we need this to be
+# un-flattened
+cd node_modules\npm
+npm install node-gyp
+cd ..\..
+cp node_modules\npm\bin\npm.cmd
+
 # install dev-bundle-package.json
 # use short folder names
-mkdir b # for build
-cd b
-mkdir t
-cd t
+# b for build
+mkdir "$DIR\b\t"
+cd "$DIR\b\t"
 
 npm config set loglevel error
 node "${CHECKOUT_DIR}\scripts\dev-bundle-server-package.js" | Out-File -FilePath package.json -Encoding ascii
@@ -40,7 +79,7 @@ npm install
 npm shrinkwrap
 
 mkdir -Force "${DIR}\server-lib\node_modules"
-cp -R "${DIR}\b\t\node_modules\*" "${DIR}\server-lib\node_modules\"
+cmd /c robocopy "${DIR}\b\t\node_modules" "${DIR}\server-lib\node_modules" /e /nfl /ndl
 
 mkdir -Force "${DIR}\etc"
 Move-Item package.json "${DIR}\etc\"
@@ -50,23 +89,14 @@ mkdir -Force "${DIR}\b\p"
 cd "${DIR}\b\p"
 node "${CHECKOUT_DIR}\scripts\dev-bundle-tool-package.js" | Out-File -FilePath package.json -Encoding ascii
 npm install
-npm dedupe
-# install the latest flatten-packages
-npm install -g flatten-packages
-flatten-packages .
-cp -R "${DIR}\b\p\node_modules\" "${DIR}\lib\node_modules\"
+cmd /c robocopy "${DIR}\b\p\node_modules" "${DIR}\lib\node_modules" /e /nfl /ndl
 cd "$DIR"
-
-# deleting folders is hard so we try twice
-rm -Recurse -Force "${DIR}\b"
-cmd /C "rmdir /s /q $DIR\b"
+cmd /c rmdir "${DIR}\b" /s /q
 
 cd "$DIR"
 mkdir "$DIR\mongodb"
 mkdir "$DIR\mongodb\bin"
 
-$webclient = New-Object System.Net.WebClient
-
 # download Mongo
 $mongo_name = "mongodb-win32-i386-${MONGO_VERSION}"
 If ($PLATFORM -eq 'windows_x86_64') {
@@ -78,7 +108,6 @@ $mongo_zip = "$DIR\mongodb\mongo.zip"
 
 $webclient.DownloadFile($mongo_link, $mongo_zip)
 
-$shell = New-Object -com shell.application
 $zip = $shell.NameSpace($mongo_zip)
 foreach($item in $zip.items()) {
   $shell.Namespace("$DIR\mongodb").copyhere($item, 0x14) # 0x10 - overwrite, 0x4 - no dialog
@@ -90,29 +119,8 @@ cp "$DIR\mongodb\$mongo_name\bin\mongo.exe" $DIR\mongodb\bin
 rm -Recurse -Force $mongo_zip
 rm -Recurse -Force "$DIR\mongodb\$mongo_name"
 
-mkdir bin
-cd bin
-
-# download node
-# same node on 32bit vs 64bit?
-$node_link = "http://nodejs.org/dist/v${NODE_VERSION}/node.exe"
-$webclient.DownloadFile($node_link, "$DIR\bin\node.exe")
-# install npm
-echo "{}" | Out-File package.json -Encoding ascii # otherwise it doesn't install in local dir
-npm install npm --save
-flatten-packages .
-
-# npm depends on a hardcoded file path to node-gyp, so we need this to be
-# un-flattened
-cd node_modules\npm
-npm install node-gyp
-
-# this path is too long
-rm -Recurse -Force "node_modules\node-gyp\node_modules\request\node_modules\combined-stream\node_modules\delayed-stream\test"
-
-cd ..\..
-
-cp node_modules\npm\bin\npm.cmd .
+# Remove npm 3 before we package the dev bundle
+rm -Recurse -Force "${DIR}\bin\npm3"
 
 cd $DIR
 
@@ -122,14 +130,11 @@ echo "${BUNDLE_VERSION}" | Out-File .bundle_version.txt -Encoding ascii
 cd "$DIR\.."
 
 # rename and move the folder with the devbundle
-# XXX this can generate a path that is too long
-Move-Item "$DIR" "dev_bundle_${PLATFORM}_${BUNDLE_VERSION}"
+cmd /c robocopy "$DIR" "dev_bundle_${PLATFORM}_${BUNDLE_VERSION}" /e /move /nfl /ndl
 
 cmd /c 7z.exe a -ttar dev_bundle.tar "dev_bundle_${PLATFORM}_${BUNDLE_VERSION}"
 cmd /c 7z.exe a -tgzip "${CHECKOUT_DIR}\dev_bundle_${PLATFORM}_${BUNDLE_VERSION}.tar.gz" dev_bundle.tar
 del dev_bundle.tar
-rm -Recurse -Force "dev_bundle_${PLATFORM}_${BUNDLE_VERSION}"
-cmd /C "rmdir /s /q dev_bundle_${PLATFORM}_${BUNDLE_VERSION}"
+cmd /c rmdir "dev_bundle_${PLATFORM}_${BUNDLE_VERSION}" /s /q
 
 echo "Done building Dev Bundle!"
-

scripts/generate-dev-bundle.sh

@@ -67,10 +67,19 @@ else
     curl "${MONGO_URL}" | tar zx
 fi
 
-cd "$DIR/build"
-
-# export path so we use our new node for later builds
+# export path so we use the downloaded node and npm
 export PATH="$DIR/bin:$PATH"
+
+# install npm 3 in a temporary directory
+mkdir "$DIR/bin/npm3"
+cd "$DIR/bin/npm3"
+npm install npm@3.0-latest
+cp node_modules/npm/bin/npm .
+
+# export path again with our temporary npm3 directory first,
+# so we can use npm 3 during builds
+export PATH="$DIR/bin/npm3:$PATH"
+
 which node
 which npm
 
@@ -94,7 +103,6 @@ mkdir -p "${DIR}/server-lib/node_modules"
 # This ignores the stuff in node_modules/.bin, but that's OK.
 cp -R node_modules/* "${DIR}/server-lib/node_modules/"
 
-mkdir "${DIR}/etc"
 mv package.json npm-shrinkwrap.json "${DIR}/etc/"
 
 # Fibers ships with compiled versions of its C code for a dozen platforms. This
@@ -116,8 +124,6 @@ mkdir "${DIR}/build/npm-tool-install"
 cd "${DIR}/build/npm-tool-install"
 node "${CHECKOUT_DIR}/scripts/dev-bundle-tool-package.js" >package.json
 npm install
-# Refactor node modules to top level and remove unnecessary duplicates.
-npm dedupe
 cp -R node_modules/* "${DIR}/lib/node_modules/"
 
 cd "${DIR}/lib"
@@ -143,19 +149,9 @@ delete sqlite3/deps
 delete wordwrap/test
 delete moment/min
 
-# dedupe isn't good enough to eliminate 3 copies of esprima, sigh.
-find . -path '*/esprima/test' | xargs rm -rf
+# Remove esprima tests to reduce the size of the dev bundle
 find . -path '*/esprima-fb/test' | xargs rm -rf
 
-# dedupe isn't good enough to eliminate 4 copies of JSONstream, sigh.
-find . -path '*/JSONStream/test/fixtures' | xargs rm -rf
-
-# Not sure why dedupe doesn't lift these to the top.
-pushd cordova/node_modules/cordova-lib/node_modules/cordova-js/node_modules/browserify/node_modules
-delete crypto-browserify/test
-delete umd/node_modules/ruglify/test
-popd
-
 cd "$DIR/lib/node_modules/fibers/bin"
 shrink_fibers
 
@@ -168,6 +164,9 @@ gunzip BrowserStackLocal*
 mv BrowserStackLocal* BrowserStackLocal
 mv BrowserStackLocal "$DIR/bin/"
 
+# remove our temporary npm3 directory
+rm -rf "$DIR/bin/npm3"
+
 echo BUNDLING
 
 cd "$DIR"

tools/files.js

@@ -23,7 +23,6 @@ var buildmessage = require('./buildmessage.js');
 var watch = require('./watch.js');
 var fiberHelpers = require('./fiber-helpers.js');
 var colonConverter = require("./colon-converter.js");
-var Console = require("./console.js").Console;
 
 var miniFiles = require("./server/mini-files.js");
 
@@ -643,7 +642,7 @@ files.freeTempDir = function (tempDir) {
       // Don't crash and print a stack trace because we failed to delete a temp
       // directory. This happens sometimes on Windows and seems to be
       // unavoidable.
-      Console.debug(err);
+      console.log(err);
     }
 
     tempDirs = _.without(tempDirs, tempDir);
@@ -659,7 +658,7 @@ if (! process.env.METEOR_SAVE_TMPDIRS) {
         // Don't crash and print a stack trace because we failed to delete a temp
         // directory. This happens sometimes on Windows and seems to be
         // unavoidable.
-        Console.debug(err);
+        console.log(err);
       }
     });