Merge branch 'devel' into override-api-version

.circleci/config.yml

@@ -647,211 +647,6 @@ jobs:
       - store_artifacts:
           path: /tmp/memuse.txt
 
-  Test Browserstack 1:
-    <<: *build_machine_environment
-    steps:
-      - run:
-          <<: *run_log_mem_use
-      - run:
-          <<: *run_env_change
-      - attach_workspace:
-          at: .
-      - run:
-          name: "Print environment"
-          command: printenv
-      - run:
-          name: "Running self-test (Test Browserstack 1)"
-          command: |
-            TEST_GROUP='css hot code push';
-            echo $TEST_GROUP;
-            eval $PRE_TEST_COMMANDS;
-            ./meteor self-test \
-              "$TEST_GROUP" \
-              --browserstack \
-              --retries ${METEOR_SELF_TEST_RETRIES} \
-              --exclude "${SELF_TEST_EXCLUDE}" \
-              --headless \
-              --junit ./tmp/results/junit/bs1.xml \
-              --without-tag "custom-warehouse"
-          no_output_timeout: 40m
-      - run:
-          <<: *run_save_node_bin
-      - store_test_results:
-          path: ./tmp/results
-      - persist_to_workspace:
-          root: .
-          paths: ./tmp/results/junit
-      - store_artifacts:
-          path: ./tmp/results
-      - store_artifacts:
-          path: /tmp/core_dumps
-      - store_artifacts:
-          path: /tmp/memuse.txt
-
-  Test Browserstack 2:
-    <<: *build_machine_environment
-    steps:
-      - run:
-          <<: *run_log_mem_use
-      - run:
-          <<: *run_env_change
-      - attach_workspace:
-          at: .
-      - run:
-          name: "Print environment"
-          command: printenv
-      - run:
-          name: "Running self-test (Test Browserstack 2)"
-          command: |
-            TEST_GROUP='custom minifier - devel vs prod';
-            echo $TEST_GROUP;
-            eval $PRE_TEST_COMMANDS;
-            ./meteor self-test \
-              "$TEST_GROUP" \
-              --browserstack \
-              --retries ${METEOR_SELF_TEST_RETRIES} \
-              --exclude "${SELF_TEST_EXCLUDE}" \
-              --headless \
-              --junit ./tmp/results/junit/bs2.xml \
-              --without-tag "custom-warehouse"
-          no_output_timeout: 40m
-      - run:
-          <<: *run_save_node_bin
-      - store_test_results:
-          path: ./tmp/results
-      - persist_to_workspace:
-          root: .
-          paths: ./tmp/results/junit
-      - store_artifacts:
-          path: ./tmp/results
-      - store_artifacts:
-          path: /tmp/core_dumps
-      - store_artifacts:
-          path: /tmp/memuse.txt
-
-  Test Browserstack 3:
-    <<: *build_machine_environment
-    steps:
-      - run:
-          <<: *run_log_mem_use
-      - run:
-          <<: *run_env_change
-      - attach_workspace:
-          at: .
-      - run:
-          name: "Print environment"
-          command: printenv
-      - run:
-          name: "Running self-test (Test Browserstack 3)"
-          command: |
-            TEST_GROUP='versioning hot code push';
-            echo $TEST_GROUP;
-            eval $PRE_TEST_COMMANDS;
-            ./meteor self-test \
-              "$TEST_GROUP" \
-              --browserstack \
-              --retries ${METEOR_SELF_TEST_RETRIES} \
-              --exclude "${SELF_TEST_EXCLUDE}" \
-              --headless \
-              --junit ./tmp/results/junit/bs3.xml \
-              --without-tag "custom-warehouse"
-          no_output_timeout: 40m
-      - run:
-          <<: *run_save_node_bin
-      - store_test_results:
-          path: ./tmp/results
-      - persist_to_workspace:
-          root: .
-          paths: ./tmp/results/junit
-      - store_artifacts:
-          path: ./tmp/results
-      - store_artifacts:
-          path: /tmp/core_dumps
-      - store_artifacts:
-          path: /tmp/memuse.txt
-
-  Test Browserstack 4:
-    <<: *build_machine_environment
-    steps:
-      - run:
-          <<: *run_log_mem_use
-      - run:
-          <<: *run_env_change
-      - attach_workspace:
-          at: .
-      - run:
-          name: "Print environment"
-          command: printenv
-      - run:
-          name: "Running self-test (Test Browserstack 4)"
-          command: |
-            TEST_GROUP='javascript hot code push';
-            echo $TEST_GROUP;
-            eval $PRE_TEST_COMMANDS;
-            ./meteor self-test \
-              "$TEST_GROUP" \
-              --browserstack \
-              --retries ${METEOR_SELF_TEST_RETRIES} \
-              --exclude "${SELF_TEST_EXCLUDE}" \
-              --headless \
-              --junit ./tmp/results/junit/bs4.xml \
-              --without-tag "custom-warehouse"
-          no_output_timeout: 40m
-      - run:
-          <<: *run_save_node_bin
-      - store_test_results:
-          path: ./tmp/results
-      - persist_to_workspace:
-          root: .
-          paths: ./tmp/results/junit
-      - store_artifacts:
-          path: ./tmp/results
-      - store_artifacts:
-          path: /tmp/core_dumps
-      - store_artifacts:
-          path: /tmp/memuse.txt
-
-  Test Browserstack 5:
-    <<: *build_machine_environment
-    steps:
-      - run:
-          <<: *run_log_mem_use
-      - run:
-          <<: *run_env_change
-      - attach_workspace:
-          at: .
-      - run:
-          name: "Print environment"
-          command: printenv
-      - run:
-          name: "Running self-test (Test Browserstack 5)"
-          command: |
-            TEST_GROUP='add packages client archs';
-            echo $TEST_GROUP;
-            eval $PRE_TEST_COMMANDS;
-            ./meteor self-test \
-              "$TEST_GROUP" \
-              --browserstack \
-              --retries ${METEOR_SELF_TEST_RETRIES} \
-              --exclude "${SELF_TEST_EXCLUDE}" \
-              --headless \
-              --junit ./tmp/results/junit/bs5.xml \
-              --without-tag "custom-warehouse"
-          no_output_timeout: 40m
-      - run:
-          <<: *run_save_node_bin
-      - store_test_results:
-          path: ./tmp/results
-      - persist_to_workspace:
-          root: .
-          paths: ./tmp/results/junit
-      - store_artifacts:
-          path: ./tmp/results
-      - store_artifacts:
-          path: /tmp/core_dumps
-      - store_artifacts:
-          path: /tmp/memuse.txt
-
   # Test the JSDoc declarations which live within this codebase against the
   # Meteor Docs (https://github.com/meteor/docs) repository, where they'll
   # eventually be consumed.  This test aims to provide an early warning of
@@ -1008,21 +803,6 @@ workflows:
       - Test Group 10:
           requires:
             - Get Ready
-      - Test Browserstack 1:
-          requires:
-            - Get Ready
-      - Test Browserstack 2:
-          requires:
-            - Get Ready
-      - Test Browserstack 3:
-          requires:
-            - Get Ready
-      - Test Browserstack 4:
-          requires:
-            - Get Ready
-      - Test Browserstack 5:
-          requires:
-            - Get Ready
       - Clean Up:
           requires:
             - Isolated Tests
@@ -1037,8 +817,3 @@ workflows:
             - Test Group 8
             - Test Group 9
             - Test Group 10
-            - Test Browserstack 1
-            - Test Browserstack 2
-            - Test Browserstack 3
-            - Test Browserstack 4
-            - Test Browserstack 5

.gitignore

@@ -1,4 +1,5 @@
 .DS_Store
+node_modules
 /.meteor
 *~
 /dev_bundle

scripts/admin/browserstack-check.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+# We need to set up properly Puppeteer and BrowserStack dependencies in the
+# machine first.
+#
+# We also need to setup s3cmd and its config to be able to read the file with
+# Browserstack key from S3. Only Meteor Software employees have access to this
+# credentials.
+#
+# This script is executed in our internal machine called Jenkins V3 before
+# at least every official release to be sure these tests listed below are
+# passing.
+#
+# They will take around 26 minutes to run:
+# custom-minifier.js test:custom minifier - devel vs prod (252998 ms)
+# hot-code-push.js test:css hot code push (370241 ms)
+# hot-code-push.js test:versioning hot code push (179834 ms)
+# hot-code-push.js test:javascript hot code push (621682 ms)
+# package-tests.js test:add packages client archs (164742 ms)
+
+cd ../..
+
+./meteor self-test \
+  "css hot code push|custom minifier - devel vs prod|versioning hot code push|javascript hot code push|add packages client archs" \
+  --browserstack \
+  --retries 2 \
+  --headless
+
+cd scripts/admin

tools/tests/custom-minifier.js

@@ -25,7 +25,9 @@ selftest.define('custom minifier - devel vs prod', function (options) {
     run.match('Message (client): production_js');
 
     run.stop();
-  }, '--production');
+  }, { args: ['--production'],
+    testName: 'custom minifier - devel vs prod - part 1',
+    testFile: 'customer-minifier.js' });
 
   s.testWithAllClients(function (run) {
     run.waitSecs(5);
@@ -44,5 +46,7 @@ selftest.define('custom minifier - devel vs prod', function (options) {
     run.match('Message (client): development_js');
 
     run.stop();
-  }/*, development*/);
+  },{
+    testName:'custom minifier - devel vs prod - part 2',
+    testFile: 'custom-minifier.js'});
 });

tools/tests/hot-code-push.js

@@ -93,7 +93,9 @@ jquery`);
     run.waitSecs(30);
 
     run.stop();
-  });
+  },{
+    testName: 'css hot code push',
+    testFile: 'hot-code-push.js' });
 });
 
 selftest.define("versioning hot code push", function (options) {
@@ -118,7 +120,9 @@ selftest.define("versioning hot code push", function (options) {
     run.forbidAll("Error listening");
 
     run.stop();
-  });
+  },{
+    testName: 'versioning hot code push',
+    testFile: 'hot-code-push.js' });
 });
 
 selftest.define("javascript hot code push", function (options) {
@@ -229,8 +233,8 @@ appcache`);
     run.match("client connected: 0");
     run.match("jsVar: undefined");
 
-    // XXX: Remove me.  This shouldn't be needed, but sometimes if we run too 
-    // quickly on fast (or Linux?) machines, it looks like there's a race and we 
+    // XXX: Remove me.  This shouldn't be needed, but sometimes if we run too
+    // quickly on fast (or Linux?) machines, it looks like there's a race and we
     // see a weird state. Without this line this test was failing one time on
     // every build in CircleCI, but oddly enough would succeed on the second
     // try.
@@ -262,5 +266,7 @@ session`);
     run.match("server restarted");
 
     run.stop();
-  });
+  },{
+    testName: 'javascript hot code push',
+    testFile: 'hot-code-push.js' });
 });

tools/tests/package-tests.js

@@ -369,7 +369,9 @@ selftest.define("add packages client archs", function (options) {
       run.match("all clients " + (expectedLogNum++));
       run.match(clientType + " client " + (expectedLogNum++));
       run.stop();
-    }, args);
+    }, { args,
+      testName: 'add packages client archs',
+      testFile: 'package-tests.js' });
   };
 
   runTestWithArgs("browser", [], 3000);

tools/tests/stylus-cross-packages.js

@@ -1,6 +1,5 @@
 var selftest = require('../tool-testing/selftest.js');
 var Sandbox = selftest.Sandbox;
-var utils = require('../utils/utils.js');
 
 selftest.define("can import stylus across packages", function (options) {
   var s = new Sandbox({
@@ -25,5 +24,7 @@ selftest.define("can import stylus across packages", function (options) {
     run.match("true");
 
     run.stop();
-  });
+  },{
+    testName: 'can import stylus across packages',
+    testFile: 'stylus-cross-packages.js' });
 });

tools/tool-testing/sandbox.js

@@ -113,8 +113,10 @@ export default class Sandbox {
   //   run.connectClient();
   //   // post-connection checks
   // });
-  testWithAllClients(f, ...args) {
-    args = args.filter(arg => arg);
+  testWithAllClients(f, options) {
+    const { testName, testFile, args: argsParam } = options || {};
+
+    const args = (argsParam || []).filter(arg => arg);
 
     // Lazy-populate the clients, only when this method is called.
     if (typeof this.clients === "undefined") {
@@ -140,12 +142,14 @@ export default class Sandbox {
       }
     }
 
-    console.log(`Running test with ${this.clients.length} client(s)...`);
+    const testNameAndFile = `${testFile ? `${testFile}: ` : ''}${testName ? `"${testName}" ` : ''}`;
+
+    console.log(`Running test ${testNameAndFile}with ${this.clients.length} client(s)...`);
 
     Object.keys(this.clients).forEach((clientKey, index, array) => {
       const client = this.clients[clientKey];
       console.log(
-        `(${index+1}/${array.length}) Testing with ${client.name}...`);
+        `(${index+1}/${array.length}) Testing ${testNameAndFile}with ${client.name}...`);
       const run = new Run(this.execPath, {
         sandbox: this,
         args,