From 2a3d23694d6fb2a390e6177112697d1e502a7360 Mon Sep 17 00:00:00 2001 From: filipenevola Date: Mon, 29 Mar 2021 15:55:41 -0400 Subject: [PATCH 1/6] Browserstack tests should run only on browserstack-check branch These tests are flaky and also we are going to run them only before releases in a specific branch --- .circleci/config.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9156f57772..5764822692 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1009,18 +1009,33 @@ workflows: requires: - Get Ready - Test Browserstack 1: + filters: + branches: + only: browserstack-check requires: - Get Ready - Test Browserstack 2: + filters: + branches: + only: browserstack-check requires: - Get Ready - Test Browserstack 3: + filters: + branches: + only: browserstack-check requires: - Get Ready - Test Browserstack 4: + filters: + branches: + only: browserstack-check requires: - Get Ready - Test Browserstack 5: + filters: + branches: + only: browserstack-check requires: - Get Ready - Clean Up: From 83d99bfbc6706ca8978ed287ee8703c5cbdb7bd2 Mon Sep 17 00:00:00 2001 From: filipenevola Date: Tue, 30 Mar 2021 10:05:13 -0400 Subject: [PATCH 2/6] Browserstack tests should run only on browserstack-check branch These tests are flaky and also we are going to run them only before releases in a specific internal server --- .circleci/config.yml | 240 --------------------------- scripts/admin/browserstack-check.sh | 9 + tools/tests/custom-minifier.js | 8 +- tools/tests/hot-code-push.js | 16 +- tools/tests/package-tests.js | 4 +- tools/tests/stylus-cross-packages.js | 5 +- tools/tool-testing/sandbox.js | 12 +- 7 files changed, 40 insertions(+), 254 deletions(-) create mode 100755 scripts/admin/browserstack-check.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 5764822692..a6f3115764 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -647,211 +647,6 @@ jobs: - store_artifacts: path: /tmp/memuse.txt - Test Browserstack 1: - <<: *build_machine_environment - steps: - - run: - <<: *run_log_mem_use - - run: - <<: *run_env_change - - attach_workspace: - at: . - - run: - name: "Print environment" - command: printenv - - run: - name: "Running self-test (Test Browserstack 1)" - command: | - TEST_GROUP='css hot code push'; - echo $TEST_GROUP; - eval $PRE_TEST_COMMANDS; - ./meteor self-test \ - "$TEST_GROUP" \ - --browserstack \ - --retries ${METEOR_SELF_TEST_RETRIES} \ - --exclude "${SELF_TEST_EXCLUDE}" \ - --headless \ - --junit ./tmp/results/junit/bs1.xml \ - --without-tag "custom-warehouse" - no_output_timeout: 40m - - run: - <<: *run_save_node_bin - - store_test_results: - path: ./tmp/results - - persist_to_workspace: - root: . - paths: ./tmp/results/junit - - store_artifacts: - path: ./tmp/results - - store_artifacts: - path: /tmp/core_dumps - - store_artifacts: - path: /tmp/memuse.txt - - Test Browserstack 2: - <<: *build_machine_environment - steps: - - run: - <<: *run_log_mem_use - - run: - <<: *run_env_change - - attach_workspace: - at: . - - run: - name: "Print environment" - command: printenv - - run: - name: "Running self-test (Test Browserstack 2)" - command: | - TEST_GROUP='custom minifier - devel vs prod'; - echo $TEST_GROUP; - eval $PRE_TEST_COMMANDS; - ./meteor self-test \ - "$TEST_GROUP" \ - --browserstack \ - --retries ${METEOR_SELF_TEST_RETRIES} \ - --exclude "${SELF_TEST_EXCLUDE}" \ - --headless \ - --junit ./tmp/results/junit/bs2.xml \ - --without-tag "custom-warehouse" - no_output_timeout: 40m - - run: - <<: *run_save_node_bin - - store_test_results: - path: ./tmp/results - - persist_to_workspace: - root: . - paths: ./tmp/results/junit - - store_artifacts: - path: ./tmp/results - - store_artifacts: - path: /tmp/core_dumps - - store_artifacts: - path: /tmp/memuse.txt - - Test Browserstack 3: - <<: *build_machine_environment - steps: - - run: - <<: *run_log_mem_use - - run: - <<: *run_env_change - - attach_workspace: - at: . - - run: - name: "Print environment" - command: printenv - - run: - name: "Running self-test (Test Browserstack 3)" - command: | - TEST_GROUP='versioning hot code push'; - echo $TEST_GROUP; - eval $PRE_TEST_COMMANDS; - ./meteor self-test \ - "$TEST_GROUP" \ - --browserstack \ - --retries ${METEOR_SELF_TEST_RETRIES} \ - --exclude "${SELF_TEST_EXCLUDE}" \ - --headless \ - --junit ./tmp/results/junit/bs3.xml \ - --without-tag "custom-warehouse" - no_output_timeout: 40m - - run: - <<: *run_save_node_bin - - store_test_results: - path: ./tmp/results - - persist_to_workspace: - root: . - paths: ./tmp/results/junit - - store_artifacts: - path: ./tmp/results - - store_artifacts: - path: /tmp/core_dumps - - store_artifacts: - path: /tmp/memuse.txt - - Test Browserstack 4: - <<: *build_machine_environment - steps: - - run: - <<: *run_log_mem_use - - run: - <<: *run_env_change - - attach_workspace: - at: . - - run: - name: "Print environment" - command: printenv - - run: - name: "Running self-test (Test Browserstack 4)" - command: | - TEST_GROUP='javascript hot code push'; - echo $TEST_GROUP; - eval $PRE_TEST_COMMANDS; - ./meteor self-test \ - "$TEST_GROUP" \ - --browserstack \ - --retries ${METEOR_SELF_TEST_RETRIES} \ - --exclude "${SELF_TEST_EXCLUDE}" \ - --headless \ - --junit ./tmp/results/junit/bs4.xml \ - --without-tag "custom-warehouse" - no_output_timeout: 40m - - run: - <<: *run_save_node_bin - - store_test_results: - path: ./tmp/results - - persist_to_workspace: - root: . - paths: ./tmp/results/junit - - store_artifacts: - path: ./tmp/results - - store_artifacts: - path: /tmp/core_dumps - - store_artifacts: - path: /tmp/memuse.txt - - Test Browserstack 5: - <<: *build_machine_environment - steps: - - run: - <<: *run_log_mem_use - - run: - <<: *run_env_change - - attach_workspace: - at: . - - run: - name: "Print environment" - command: printenv - - run: - name: "Running self-test (Test Browserstack 5)" - command: | - TEST_GROUP='add packages client archs'; - echo $TEST_GROUP; - eval $PRE_TEST_COMMANDS; - ./meteor self-test \ - "$TEST_GROUP" \ - --browserstack \ - --retries ${METEOR_SELF_TEST_RETRIES} \ - --exclude "${SELF_TEST_EXCLUDE}" \ - --headless \ - --junit ./tmp/results/junit/bs5.xml \ - --without-tag "custom-warehouse" - no_output_timeout: 40m - - run: - <<: *run_save_node_bin - - store_test_results: - path: ./tmp/results - - persist_to_workspace: - root: . - paths: ./tmp/results/junit - - store_artifacts: - path: ./tmp/results - - store_artifacts: - path: /tmp/core_dumps - - store_artifacts: - path: /tmp/memuse.txt - # Test the JSDoc declarations which live within this codebase against the # Meteor Docs (https://github.com/meteor/docs) repository, where they'll # eventually be consumed. This test aims to provide an early warning of @@ -1008,36 +803,6 @@ workflows: - Test Group 10: requires: - Get Ready - - Test Browserstack 1: - filters: - branches: - only: browserstack-check - requires: - - Get Ready - - Test Browserstack 2: - filters: - branches: - only: browserstack-check - requires: - - Get Ready - - Test Browserstack 3: - filters: - branches: - only: browserstack-check - requires: - - Get Ready - - Test Browserstack 4: - filters: - branches: - only: browserstack-check - requires: - - Get Ready - - Test Browserstack 5: - filters: - branches: - only: browserstack-check - requires: - - Get Ready - Clean Up: requires: - Isolated Tests @@ -1052,8 +817,3 @@ workflows: - Test Group 8 - Test Group 9 - Test Group 10 - - Test Browserstack 1 - - Test Browserstack 2 - - Test Browserstack 3 - - Test Browserstack 4 - - Test Browserstack 5 diff --git a/scripts/admin/browserstack-check.sh b/scripts/admin/browserstack-check.sh new file mode 100755 index 0000000000..500719d131 --- /dev/null +++ b/scripts/admin/browserstack-check.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +cd ../.. + +./meteor self-test \ + "css hot code push|custom minifier - devel vs prod|versioning hot code push|javascript hot code push|add packages client archs" \ + --browserstack \ + --retries 2 \ + --headless diff --git a/tools/tests/custom-minifier.js b/tools/tests/custom-minifier.js index 10938862fa..2ea340c492 100644 --- a/tools/tests/custom-minifier.js +++ b/tools/tests/custom-minifier.js @@ -25,7 +25,9 @@ selftest.define('custom minifier - devel vs prod', function (options) { run.match('Message (client): production_js'); run.stop(); - }, '--production'); + }, { args: ['--production'], + testName: 'custom minifier - devel vs prod - part 1', + testFile: 'customer-minifier.js' }); s.testWithAllClients(function (run) { run.waitSecs(5); @@ -44,5 +46,7 @@ selftest.define('custom minifier - devel vs prod', function (options) { run.match('Message (client): development_js'); run.stop(); - }/*, development*/); + },{ + testName:'custom minifier - devel vs prod - part 2', + testFile: 'custom-minifier.js'}); }); diff --git a/tools/tests/hot-code-push.js b/tools/tests/hot-code-push.js index 03eab2122e..9c2a308888 100644 --- a/tools/tests/hot-code-push.js +++ b/tools/tests/hot-code-push.js @@ -93,7 +93,9 @@ jquery`); run.waitSecs(30); run.stop(); - }); + },{ + testName: 'css hot code push', + testFile: 'hot-code-push.js' }); }); selftest.define("versioning hot code push", function (options) { @@ -118,7 +120,9 @@ selftest.define("versioning hot code push", function (options) { run.forbidAll("Error listening"); run.stop(); - }); + },{ + testName: 'versioning hot code push', + testFile: 'hot-code-push.js' }); }); selftest.define("javascript hot code push", function (options) { @@ -229,8 +233,8 @@ appcache`); run.match("client connected: 0"); run.match("jsVar: undefined"); - // XXX: Remove me. This shouldn't be needed, but sometimes if we run too - // quickly on fast (or Linux?) machines, it looks like there's a race and we + // XXX: Remove me. This shouldn't be needed, but sometimes if we run too + // quickly on fast (or Linux?) machines, it looks like there's a race and we // see a weird state. Without this line this test was failing one time on // every build in CircleCI, but oddly enough would succeed on the second // try. @@ -262,5 +266,7 @@ session`); run.match("server restarted"); run.stop(); - }); + },{ + testName: 'javascript hot code push', + testFile: 'hot-code-push.js' }); }); diff --git a/tools/tests/package-tests.js b/tools/tests/package-tests.js index b362113d65..e7649496d2 100644 --- a/tools/tests/package-tests.js +++ b/tools/tests/package-tests.js @@ -369,7 +369,9 @@ selftest.define("add packages client archs", function (options) { run.match("all clients " + (expectedLogNum++)); run.match(clientType + " client " + (expectedLogNum++)); run.stop(); - }, args); + }, { args, + testName: 'add packages client archs', + testFile: 'package-tests.js' }); }; runTestWithArgs("browser", [], 3000); diff --git a/tools/tests/stylus-cross-packages.js b/tools/tests/stylus-cross-packages.js index 44692ca9df..4df4faba4f 100644 --- a/tools/tests/stylus-cross-packages.js +++ b/tools/tests/stylus-cross-packages.js @@ -1,6 +1,5 @@ var selftest = require('../tool-testing/selftest.js'); var Sandbox = selftest.Sandbox; -var utils = require('../utils/utils.js'); selftest.define("can import stylus across packages", function (options) { var s = new Sandbox({ @@ -25,5 +24,7 @@ selftest.define("can import stylus across packages", function (options) { run.match("true"); run.stop(); - }); + },{ + testName: 'can import stylus across packages', + testFile: 'stylus-cross-packages.js' }); }); diff --git a/tools/tool-testing/sandbox.js b/tools/tool-testing/sandbox.js index fcbde5ef5c..b036cb09b9 100644 --- a/tools/tool-testing/sandbox.js +++ b/tools/tool-testing/sandbox.js @@ -113,8 +113,10 @@ export default class Sandbox { // run.connectClient(); // // post-connection checks // }); - testWithAllClients(f, ...args) { - args = args.filter(arg => arg); + testWithAllClients(f, options) { + const { testName, testFile, args: argsParam } = options || {}; + + const args = (argsParam || []).filter(arg => arg); // Lazy-populate the clients, only when this method is called. if (typeof this.clients === "undefined") { @@ -140,12 +142,14 @@ export default class Sandbox { } } - console.log(`Running test with ${this.clients.length} client(s)...`); + const testNameAndFile = `${testFile ? `${testFile}: ` : ''}${testName ? `"${testName}" ` : ''}`; + + console.log(`Running test ${testNameAndFile}with ${this.clients.length} client(s)...`); Object.keys(this.clients).forEach((clientKey, index, array) => { const client = this.clients[clientKey]; console.log( - `(${index+1}/${array.length}) Testing with ${client.name}...`); + `(${index+1}/${array.length}) Testing ${testNameAndFile}with ${client.name}...`); const run = new Run(this.execPath, { sandbox: this, args, From 3830c679db4a62b52ff67846d981ec53999818cc Mon Sep 17 00:00:00 2001 From: filipenevola Date: Tue, 30 Mar 2021 10:39:12 -0400 Subject: [PATCH 3/6] Browserstack tests should run only on browserstack-check branch These tests are flaky and also we are going to run them only before releases in a specific internal server --- scripts/admin/browserstack-check.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/admin/browserstack-check.sh b/scripts/admin/browserstack-check.sh index 500719d131..be14a544aa 100755 --- a/scripts/admin/browserstack-check.sh +++ b/scripts/admin/browserstack-check.sh @@ -7,3 +7,5 @@ cd ../.. --browserstack \ --retries 2 \ --headless + +cd scripts/admin From 8a6a75290b497389eb3b020c451e363f7557d181 Mon Sep 17 00:00:00 2001 From: filipenevola Date: Tue, 30 Mar 2021 10:44:12 -0400 Subject: [PATCH 4/6] Browserstack tests should run only on browserstack-check branch These tests are flaky and also we are going to run them only before releases in a specific internal server --- scripts/admin/browserstack-check.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/admin/browserstack-check.sh b/scripts/admin/browserstack-check.sh index be14a544aa..324e9a7fb7 100755 --- a/scripts/admin/browserstack-check.sh +++ b/scripts/admin/browserstack-check.sh @@ -1,5 +1,16 @@ #!/usr/bin/env bash +# We need to set up properly Puppeteer and BrowserStack dependencies in the +# machine first. +# +# We also need to setup s3cmd and its config to be able to read the file with +# Browserstack key from S3. Only Meteor Software employees have access to this +# credentials. +# +# This script is executed in our internal machine called Jenkins V3 before +# at least every official release to be sure these tests listed below are +# passing. + cd ../.. ./meteor self-test \ From ee6e405e75be03003e36aee36788de33fed2f442 Mon Sep 17 00:00:00 2001 From: filipenevola Date: Tue, 30 Mar 2021 10:47:05 -0400 Subject: [PATCH 5/6] Browserstack tests should run only on browserstack-check branch These tests are flaky and also we are going to run them only before releases in a specific internal server --- scripts/admin/browserstack-check.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/admin/browserstack-check.sh b/scripts/admin/browserstack-check.sh index 324e9a7fb7..af2e1a5f6b 100755 --- a/scripts/admin/browserstack-check.sh +++ b/scripts/admin/browserstack-check.sh @@ -10,6 +10,13 @@ # This script is executed in our internal machine called Jenkins V3 before # at least every official release to be sure these tests listed below are # passing. +# +# They will take around 26 minutes to run: +# custom-minifier.js test:custom minifier - devel vs prod (252998 ms) +# hot-code-push.js test:css hot code push (370241 ms) +# hot-code-push.js test:versioning hot code push (179834 ms) +# hot-code-push.js test:javascript hot code push (621682 ms) +# package-tests.js test:add packages client archs (164742 ms) cd ../.. From c8e011751e2c058e7c4da5b3923a3d0f93c963f7 Mon Sep 17 00:00:00 2001 From: filipenevola Date: Wed, 31 Mar 2021 11:16:31 -0400 Subject: [PATCH 6/6] Updates .gitignore to include root node_modules --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7f297e1f6f..f34843314b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .DS_Store +node_modules /.meteor *~ /dev_bundle