From 984548d671e9d44d5af97bab240c40064186107d Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 21 Aug 2017 12:06:54 -0400 Subject: [PATCH 01/11] Add standalone script for building Node on Jenkins. This commit revives the script that was removed last year by a4ff6b73d30d13d48df5d4c4d4d489a7da405e40, when we switched from building our own version of Node to downloading the prebuilt release. The new implementation comes from @abernix's work on this branch: https://github.com/meteor/meteor/tree/abernix/dev-bundle-from-hash --- scripts/build-node-for-dev-bundle.sh | 69 ++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100755 scripts/build-node-for-dev-bundle.sh diff --git a/scripts/build-node-for-dev-bundle.sh b/scripts/build-node-for-dev-bundle.sh new file mode 100755 index 0000000000..a4fd6b3e59 --- /dev/null +++ b/scripts/build-node-for-dev-bundle.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +set -e +set -u + +source "$(dirname $0)/build-dev-bundle-common.sh" +echo CHECKOUT DIR IS "$CHECKOUT_DIR" +echo BUILDING NODE "v$NODE_VERSION" IN "$DIR" + +cd "$DIR" + +if [ ! -z ${NODE_FROM_SRC+x} ] || [ ! -z ${NODE_COMMIT_HASH+x} ] +then + if [ ! -z ${NODE_COMMIT_HASH+x} ] + then + NODE_FROM_SRC=${NODE_FROM_SRC:=true} + echo "Building Node source from Git hash ${NODE_COMMIT_HASH}..."; + NODE_URL="https://github.com/meteor/node/archive/${NODE_COMMIT_HASH}.tar.gz" + else + echo "Building Node source from ${NODE_VERSION} src tarball..."; + NODE_URL="https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}.tar.gz" + fi +else + NODE_URL="https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TGZ}" +fi + +# Update these values after building the dev-bundle-node Jenkins project. +# Also make sure to update NODE_VERSION in generate-dev-bundle.ps1. +function downloadNode { + echo "Downloading Node from ${NODE_URL}" + curl -sL "${NODE_URL}" | tar zx --strip-components 1 +} + +if [ ! -z ${NODE_FROM_SRC+x} ] +then + mkdir node-src/ && cd node-src/ + downloadNode + if [ "${NODE_FROM_SRC:-}" = "debug" ] + then + ./configure --debug --prefix "${DIR}" + else + ./configure --prefix "${DIR}" + fi + make -j4 + # PORTABLE=1 is a node hack to make npm look relative to itself instead + # of hard coding the PREFIX. + make install PORTABLE=1 + export npm_config_nodedir="${DIR}/node-src" + cd "$DIR" +else + downloadNode +fi + +cd "$DIR" +stripBinary bin/node + +# export path so we use our new node for later builds +PATH="$DIR/bin:$PATH" +which node +which npm +npm version + +echo BUNDLING + +cd "$DIR" +rm -rf build +tar czvf "${CHECKOUT_DIR}/node_${PLATFORM}_v${NODE_VERSION}.tar.gz" . + +echo DONE From 77becc6bfea19cbd51ba392d7f918ec194f6a911 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 21 Aug 2017 13:35:46 -0400 Subject: [PATCH 02/11] Make sure the Node build directory gets deleted. --- scripts/build-node-for-dev-bundle.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/build-node-for-dev-bundle.sh b/scripts/build-node-for-dev-bundle.sh index a4fd6b3e59..56f0e5ae72 100755 --- a/scripts/build-node-for-dev-bundle.sh +++ b/scripts/build-node-for-dev-bundle.sh @@ -33,7 +33,7 @@ function downloadNode { if [ ! -z ${NODE_FROM_SRC+x} ] then - mkdir node-src/ && cd node-src/ + mkdir node-build && cd node-build downloadNode if [ "${NODE_FROM_SRC:-}" = "debug" ] then @@ -45,7 +45,6 @@ then # PORTABLE=1 is a node hack to make npm look relative to itself instead # of hard coding the PREFIX. make install PORTABLE=1 - export npm_config_nodedir="${DIR}/node-src" cd "$DIR" else downloadNode @@ -63,7 +62,7 @@ npm version echo BUNDLING cd "$DIR" -rm -rf build +rm -rf node-build tar czvf "${CHECKOUT_DIR}/node_${PLATFORM}_v${NODE_VERSION}.tar.gz" . echo DONE From 8e7815b43aba706ed31a69f1a562f7953b7bbf1e Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 21 Aug 2017 14:37:12 -0400 Subject: [PATCH 03/11] Try downloading Node from S3 if version exists. --- scripts/build-dev-bundle-common.sh | 3 +++ scripts/generate-dev-bundle.sh | 32 ++++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/scripts/build-dev-bundle-common.sh b/scripts/build-dev-bundle-common.sh index de6be8bece..b35a0d8ac6 100644 --- a/scripts/build-dev-bundle-common.sh +++ b/scripts/build-dev-bundle-common.sh @@ -9,6 +9,9 @@ MONGO_VERSION=3.2.15 NODE_VERSION=4.8.4 NPM_VERSION=4.6.1 +# If we built Node from source on Jenkins, this is the build number. +NODE_BUILD_NUMBER=27 + if [ "$UNAME" == "Linux" ] ; then if [ "$ARCH" != "i686" -a "$ARCH" != "x86_64" ] ; then echo "Unsupported architecture: $ARCH" diff --git a/scripts/generate-dev-bundle.sh b/scripts/generate-dev-bundle.sh index 2ef74cf726..6b5faae2a0 100755 --- a/scripts/generate-dev-bundle.sh +++ b/scripts/generate-dev-bundle.sh @@ -16,13 +16,33 @@ echo BUILDING DEV BUNDLE "$BUNDLE_VERSION" IN "$DIR" cd "$DIR" -S3_HOST="s3.amazonaws.com/com.meteor.jenkins" +extractNodeFromTarGz() { + LOCAL_TGZ="${CHECKOUT_DIR}/node_${PLATFORM}_v${NODE_VERSION}.tar.gz" + if [ -f "$LOCAL_TGZ" ] + then + echo "Skipping download and installing Node from $LOCAL_TGZ" >&2 + tar zxf "$LOCAL_TGZ" + return 0 + fi + return 1 +} -# Update these values after building the dev-bundle-node Jenkins project. -# Also make sure to update NODE_VERSION in generate-dev-bundle.ps1. -NODE_URL="https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TGZ}" -echo "Downloading Node from ${NODE_URL}" -curl "${NODE_URL}" | tar zx --strip-components 1 +downloadNodeFromS3() { + S3_HOST="s3.amazonaws.com/com.meteor.jenkins" + S3_TGZ="node_${UNAME}_${ARCH}_v${NODE_VERSION}.tar.gz" + NODE_URL="https://${S3_HOST}/dev-bundle-node-${NODE_BUILD_NUMBER}/${S3_TGZ}" + echo "Downloading Node from ${NODE_URL}" >&2 + curl "${NODE_URL}" | tar zx +} + +downloadOfficialNode() { + NODE_URL="https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TGZ}" + echo "Downloading Node from ${NODE_URL}" >&2 + curl "${NODE_URL}" | tar zx --strip-components 1 +} + +# Try each strategy in the following order: +extractNodeFromTarGz || downloadNodeFromS3 || downloadOfficialNode # Download Mongo from mongodb.com MONGO_NAME="mongodb-${OS}-${ARCH}-${MONGO_VERSION}" From 857841c20d0999524ed5181fde9c9dd5258c8020 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 21 Aug 2017 14:40:33 -0400 Subject: [PATCH 04/11] Bump $BUNDLE_VERSION to 4.8.32 before rebuilding dev bundle. --- meteor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meteor b/meteor index 76e415db96..447bd3fc39 100755 --- a/meteor +++ b/meteor @@ -1,6 +1,6 @@ #!/usr/bin/env bash -BUNDLE_VERSION=4.8.31 +BUNDLE_VERSION=4.8.32 # OS Check. Put here because here is where we download the precompiled # bundles that are arch specific. From 67b76abc7867eebd07df7dce2da2bae064a8c1ac Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 21 Aug 2017 15:10:25 -0400 Subject: [PATCH 05/11] Make node-gyp look for headers in dev_bundle/include/node. Thanks to @abernix for identifying this solution to the duplication between dev_bundle/include/node and dev_bundle/.node-gyp/*/node. --- scripts/generate-dev-bundle.sh | 18 +++++++++--------- tools/cli/dev-bundle-bin-helpers.js | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/generate-dev-bundle.sh b/scripts/generate-dev-bundle.sh index 6b5faae2a0..e7b84becb4 100755 --- a/scripts/generate-dev-bundle.sh +++ b/scripts/generate-dev-bundle.sh @@ -68,6 +68,15 @@ which node which npm npm version +# Make node-gyp use Node headers and libraries from $DIR/include/node. +export HOME="$DIR" +export USERPROFILE="$DIR" +export npm_config_nodedir="$DIR" + +INCLUDE_PATH="${DIR}/include/node" +echo "Contents of ${INCLUDE_PATH}:" +ls -al "$INCLUDE_PATH" + # When adding new node modules (or any software) to the dev bundle, # remember to update LICENSE.txt! Also note that we include all the # packages that these depend on, so watch out for new dependencies when @@ -115,15 +124,6 @@ cp -R node_modules/* "${DIR}/lib/node_modules/" # commands like node-gyp and node-pre-gyp. cp -R node_modules/.bin "${DIR}/lib/node_modules/" -# Make node-gyp install Node headers and libraries in $DIR/.node-gyp/. -# https://github.com/nodejs/node-gyp/blob/4ee31329e0/lib/node-gyp.js#L52 -export HOME="$DIR" -export USERPROFILE="$DIR" -node "${DIR}/lib/node_modules/node-gyp/bin/node-gyp.js" install -INCLUDE_PATH="${DIR}/.node-gyp/${NODE_VERSION}/include/node" -echo "Contents of ${INCLUDE_PATH}:" -ls -al "$INCLUDE_PATH" - cd "${DIR}/lib" # Clean up some bulky stuff. diff --git a/tools/cli/dev-bundle-bin-helpers.js b/tools/cli/dev-bundle-bin-helpers.js index 5a5bef6c3d..d0b36db9f4 100644 --- a/tools/cli/dev-bundle-bin-helpers.js +++ b/tools/cli/dev-bundle-bin-helpers.js @@ -100,8 +100,8 @@ exports.getEnv = function (options) { } // This allows node-gyp to find Node headers and libraries in - // dev_bundle/.node-gyp. - env.USERPROFILE = devBundleDir; + // dev_bundle/include/node. + env.NPM_CONFIG_NODEDIR = devBundleDir; var PATH = env.PATH || env.Path; if (PATH) { From 955bce8096d660dfa8907ee8416558f184b48454 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 21 Aug 2017 15:11:31 -0400 Subject: [PATCH 06/11] Bump $BUNDLE_VERSION to 4.8.33 before rebuilding dev bundle. --- meteor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meteor b/meteor index 447bd3fc39..fb6a8ebe64 100755 --- a/meteor +++ b/meteor @@ -1,6 +1,6 @@ #!/usr/bin/env bash -BUNDLE_VERSION=4.8.32 +BUNDLE_VERSION=4.8.33 # OS Check. Put here because here is where we download the precompiled # bundles that are arch specific. From 77d41a7084c1d1172dee3722a265384459b2c59a Mon Sep 17 00:00:00 2001 From: Jesse Rosenberger Date: Tue, 22 Aug 2017 16:36:01 +0300 Subject: [PATCH 07/11] Build Node.js with ICU support. To remain the same as official Node.js releases, we need to build with the `small-icu` ICU (International Components for Unicode) package. For Node.js 4.x this means ICU 56.x. As found with `process.config` on an official Node.js release. See https://github.com/nodejs/node/wiki/Intl#configure-node-with-specific-icu-source. --- scripts/build-node-for-dev-bundle.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/scripts/build-node-for-dev-bundle.sh b/scripts/build-node-for-dev-bundle.sh index 56f0e5ae72..6cec100e96 100755 --- a/scripts/build-node-for-dev-bundle.sh +++ b/scripts/build-node-for-dev-bundle.sh @@ -35,12 +35,25 @@ if [ ! -z ${NODE_FROM_SRC+x} ] then mkdir node-build && cd node-build downloadNode + + # Build with International Components for Unicode (ICU) Support... + # Node 4.x used 56.x. Node 8.x uses 59.x. I believe the only + # reliable location to find the correct version of ICU for a Node.js + # release is to check `process.config.icu_ver_major` from an + # official, compiled Node.js release. + # https://github.com/nodejs/node/wiki/Intl#configure-node-with-specific-icu-source + echo "Downloading International Components for Unicode (ICU)..." + curl -sL http://download.icu-project.org/files/icu4c/56.1/icu4c-56_1-src.tgz | \ + tar zx -C deps/ + + node_configure_flags=("--prefix=${DIR}" '--with-intl=small-icu') + if [ "${NODE_FROM_SRC:-}" = "debug" ] then - ./configure --debug --prefix "${DIR}" - else - ./configure --prefix "${DIR}" + node_configure_flags+=('--debug') fi + + ./configure "${node_configure_flags[@]}" make -j4 # PORTABLE=1 is a node hack to make npm look relative to itself instead # of hard coding the PREFIX. From 31c403c33f43c2fa6e2472f554a5b9bb6eefd1f1 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Tue, 22 Aug 2017 15:50:20 -0400 Subject: [PATCH 08/11] Update NODE_BUILD_NUMBER to latest Jenkins build, 33. --- scripts/build-dev-bundle-common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build-dev-bundle-common.sh b/scripts/build-dev-bundle-common.sh index b35a0d8ac6..92ada8502a 100644 --- a/scripts/build-dev-bundle-common.sh +++ b/scripts/build-dev-bundle-common.sh @@ -10,7 +10,7 @@ NODE_VERSION=4.8.4 NPM_VERSION=4.6.1 # If we built Node from source on Jenkins, this is the build number. -NODE_BUILD_NUMBER=27 +NODE_BUILD_NUMBER=33 if [ "$UNAME" == "Linux" ] ; then if [ "$ARCH" != "i686" -a "$ARCH" != "x86_64" ] ; then From e2f0bed272b53e170ffbe951457a5cc72ed99109 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Tue, 22 Aug 2017 15:50:40 -0400 Subject: [PATCH 09/11] Bump $BUNDLE_VERSION to 4.8.34 before rebuilding dev bundle. --- meteor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meteor b/meteor index fb6a8ebe64..d4bfbeaca6 100755 --- a/meteor +++ b/meteor @@ -1,6 +1,6 @@ #!/usr/bin/env bash -BUNDLE_VERSION=4.8.33 +BUNDLE_VERSION=4.8.34 # OS Check. Put here because here is where we download the precompiled # bundles that are arch specific. From e2fe03a64310e4ef0399bd957b51e66e740e80c3 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 21 Aug 2017 15:58:04 -0400 Subject: [PATCH 10/11] Add note to History.md about patching Node 4.8.4. --- History.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/History.md b/History.md index 61817d16c3..8b5632220d 100644 --- a/History.md +++ b/History.md @@ -2,6 +2,15 @@ ## v1.5.2, TBD +* Node 4.8.4 has been patched to include + https://github.com/nodejs/node/pull/14829, an important PR implemented + by our own @abernix (:tada:), which fixes a faulty backport of garbage + collection-related logic in V8 that was causing occasional segmentation + faults during Meteor development and testing, ever since Node 4.6.2 + (Meteor 1.4.2.3). When Node 4.8.5 is officially released with these + changes, we will immediately cut a small follow-up release. + [Issue #8648](https://github.com/meteor/meteor/issues/8648) + * The `meteor-babel` package has been upgraded to version 0.23.1. * The `reify` npm package has been upgraded to version 0.12.0, which From 05c8627be95b04073e6c028a7e775008ff73a29d Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Tue, 22 Aug 2017 18:41:54 -0400 Subject: [PATCH 11/11] Run `./meteor --get-ready` instead of just `./meteor --help`. A while back we switched from running `./meteor --get-ready`, a command that takes many minutes and sometimes runs out of memory, to just running `./meteor --help` to prepare for self-tests. The hope was that `./meteor --help` would fail less often, and the work that would have been done by `./meteor --get-ready` would be spread out through the actual tests. This helped, I think, but we've been seeing quite a few self-test failures due to unreliable timing of the actual tests, so I'd like to try shifting the balance back. I'm pushing this to the branch with our patched Node 4.8.4, because that branch should have a lower risk of segmentation faults, which may allow `./meteor --get-ready` to succeed more often. This is a bit of a shot in the dark, admittedly, but I want to see what happens. :crossed_fingers: --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 215c7d0461..59fdaf4bd6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -124,9 +124,9 @@ jobs: name: Get Ready command: | eval $PRE_TEST_COMMANDS; - ./meteor --help - # shouldn't take longer than 5 minutes - no_output_timeout: 5m + ./meteor --get-ready + # shouldn't take longer than 10 minutes + no_output_timeout: 10m # Clear dev_bundle/.npm to ensure consistent test runs. - run: name: Clear npm cache