From 83d99bfbc6706ca8978ed287ee8703c5cbdb7bd2 Mon Sep 17 00:00:00 2001 From: filipenevola Date: Tue, 30 Mar 2021 10:05:13 -0400 Subject: [PATCH] Browserstack tests should run only on browserstack-check branch These tests are flaky and also we are going to run them only before releases in a specific internal server --- .circleci/config.yml | 240 --------------------------- scripts/admin/browserstack-check.sh | 9 + tools/tests/custom-minifier.js | 8 +- tools/tests/hot-code-push.js | 16 +- tools/tests/package-tests.js | 4 +- tools/tests/stylus-cross-packages.js | 5 +- tools/tool-testing/sandbox.js | 12 +- 7 files changed, 40 insertions(+), 254 deletions(-) create mode 100755 scripts/admin/browserstack-check.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 5764822692..a6f3115764 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -647,211 +647,6 @@ jobs: - store_artifacts: path: /tmp/memuse.txt - Test Browserstack 1: - <<: *build_machine_environment - steps: - - run: - <<: *run_log_mem_use - - run: - <<: *run_env_change - - attach_workspace: - at: . - - run: - name: "Print environment" - command: printenv - - run: - name: "Running self-test (Test Browserstack 1)" - command: | - TEST_GROUP='css hot code push'; - echo $TEST_GROUP; - eval $PRE_TEST_COMMANDS; - ./meteor self-test \ - "$TEST_GROUP" \ - --browserstack \ - --retries ${METEOR_SELF_TEST_RETRIES} \ - --exclude "${SELF_TEST_EXCLUDE}" \ - --headless \ - --junit ./tmp/results/junit/bs1.xml \ - --without-tag "custom-warehouse" - no_output_timeout: 40m - - run: - <<: *run_save_node_bin - - store_test_results: - path: ./tmp/results - - persist_to_workspace: - root: . - paths: ./tmp/results/junit - - store_artifacts: - path: ./tmp/results - - store_artifacts: - path: /tmp/core_dumps - - store_artifacts: - path: /tmp/memuse.txt - - Test Browserstack 2: - <<: *build_machine_environment - steps: - - run: - <<: *run_log_mem_use - - run: - <<: *run_env_change - - attach_workspace: - at: . - - run: - name: "Print environment" - command: printenv - - run: - name: "Running self-test (Test Browserstack 2)" - command: | - TEST_GROUP='custom minifier - devel vs prod'; - echo $TEST_GROUP; - eval $PRE_TEST_COMMANDS; - ./meteor self-test \ - "$TEST_GROUP" \ - --browserstack \ - --retries ${METEOR_SELF_TEST_RETRIES} \ - --exclude "${SELF_TEST_EXCLUDE}" \ - --headless \ - --junit ./tmp/results/junit/bs2.xml \ - --without-tag "custom-warehouse" - no_output_timeout: 40m - - run: - <<: *run_save_node_bin - - store_test_results: - path: ./tmp/results - - persist_to_workspace: - root: . - paths: ./tmp/results/junit - - store_artifacts: - path: ./tmp/results - - store_artifacts: - path: /tmp/core_dumps - - store_artifacts: - path: /tmp/memuse.txt - - Test Browserstack 3: - <<: *build_machine_environment - steps: - - run: - <<: *run_log_mem_use - - run: - <<: *run_env_change - - attach_workspace: - at: . - - run: - name: "Print environment" - command: printenv - - run: - name: "Running self-test (Test Browserstack 3)" - command: | - TEST_GROUP='versioning hot code push'; - echo $TEST_GROUP; - eval $PRE_TEST_COMMANDS; - ./meteor self-test \ - "$TEST_GROUP" \ - --browserstack \ - --retries ${METEOR_SELF_TEST_RETRIES} \ - --exclude "${SELF_TEST_EXCLUDE}" \ - --headless \ - --junit ./tmp/results/junit/bs3.xml \ - --without-tag "custom-warehouse" - no_output_timeout: 40m - - run: - <<: *run_save_node_bin - - store_test_results: - path: ./tmp/results - - persist_to_workspace: - root: . - paths: ./tmp/results/junit - - store_artifacts: - path: ./tmp/results - - store_artifacts: - path: /tmp/core_dumps - - store_artifacts: - path: /tmp/memuse.txt - - Test Browserstack 4: - <<: *build_machine_environment - steps: - - run: - <<: *run_log_mem_use - - run: - <<: *run_env_change - - attach_workspace: - at: . - - run: - name: "Print environment" - command: printenv - - run: - name: "Running self-test (Test Browserstack 4)" - command: | - TEST_GROUP='javascript hot code push'; - echo $TEST_GROUP; - eval $PRE_TEST_COMMANDS; - ./meteor self-test \ - "$TEST_GROUP" \ - --browserstack \ - --retries ${METEOR_SELF_TEST_RETRIES} \ - --exclude "${SELF_TEST_EXCLUDE}" \ - --headless \ - --junit ./tmp/results/junit/bs4.xml \ - --without-tag "custom-warehouse" - no_output_timeout: 40m - - run: - <<: *run_save_node_bin - - store_test_results: - path: ./tmp/results - - persist_to_workspace: - root: . - paths: ./tmp/results/junit - - store_artifacts: - path: ./tmp/results - - store_artifacts: - path: /tmp/core_dumps - - store_artifacts: - path: /tmp/memuse.txt - - Test Browserstack 5: - <<: *build_machine_environment - steps: - - run: - <<: *run_log_mem_use - - run: - <<: *run_env_change - - attach_workspace: - at: . - - run: - name: "Print environment" - command: printenv - - run: - name: "Running self-test (Test Browserstack 5)" - command: | - TEST_GROUP='add packages client archs'; - echo $TEST_GROUP; - eval $PRE_TEST_COMMANDS; - ./meteor self-test \ - "$TEST_GROUP" \ - --browserstack \ - --retries ${METEOR_SELF_TEST_RETRIES} \ - --exclude "${SELF_TEST_EXCLUDE}" \ - --headless \ - --junit ./tmp/results/junit/bs5.xml \ - --without-tag "custom-warehouse" - no_output_timeout: 40m - - run: - <<: *run_save_node_bin - - store_test_results: - path: ./tmp/results - - persist_to_workspace: - root: . - paths: ./tmp/results/junit - - store_artifacts: - path: ./tmp/results - - store_artifacts: - path: /tmp/core_dumps - - store_artifacts: - path: /tmp/memuse.txt - # Test the JSDoc declarations which live within this codebase against the # Meteor Docs (https://github.com/meteor/docs) repository, where they'll # eventually be consumed. This test aims to provide an early warning of @@ -1008,36 +803,6 @@ workflows: - Test Group 10: requires: - Get Ready - - Test Browserstack 1: - filters: - branches: - only: browserstack-check - requires: - - Get Ready - - Test Browserstack 2: - filters: - branches: - only: browserstack-check - requires: - - Get Ready - - Test Browserstack 3: - filters: - branches: - only: browserstack-check - requires: - - Get Ready - - Test Browserstack 4: - filters: - branches: - only: browserstack-check - requires: - - Get Ready - - Test Browserstack 5: - filters: - branches: - only: browserstack-check - requires: - - Get Ready - Clean Up: requires: - Isolated Tests @@ -1052,8 +817,3 @@ workflows: - Test Group 8 - Test Group 9 - Test Group 10 - - Test Browserstack 1 - - Test Browserstack 2 - - Test Browserstack 3 - - Test Browserstack 4 - - Test Browserstack 5 diff --git a/scripts/admin/browserstack-check.sh b/scripts/admin/browserstack-check.sh new file mode 100755 index 0000000000..500719d131 --- /dev/null +++ b/scripts/admin/browserstack-check.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +cd ../.. + +./meteor self-test \ + "css hot code push|custom minifier - devel vs prod|versioning hot code push|javascript hot code push|add packages client archs" \ + --browserstack \ + --retries 2 \ + --headless diff --git a/tools/tests/custom-minifier.js b/tools/tests/custom-minifier.js index 10938862fa..2ea340c492 100644 --- a/tools/tests/custom-minifier.js +++ b/tools/tests/custom-minifier.js @@ -25,7 +25,9 @@ selftest.define('custom minifier - devel vs prod', function (options) { run.match('Message (client): production_js'); run.stop(); - }, '--production'); + }, { args: ['--production'], + testName: 'custom minifier - devel vs prod - part 1', + testFile: 'customer-minifier.js' }); s.testWithAllClients(function (run) { run.waitSecs(5); @@ -44,5 +46,7 @@ selftest.define('custom minifier - devel vs prod', function (options) { run.match('Message (client): development_js'); run.stop(); - }/*, development*/); + },{ + testName:'custom minifier - devel vs prod - part 2', + testFile: 'custom-minifier.js'}); }); diff --git a/tools/tests/hot-code-push.js b/tools/tests/hot-code-push.js index 03eab2122e..9c2a308888 100644 --- a/tools/tests/hot-code-push.js +++ b/tools/tests/hot-code-push.js @@ -93,7 +93,9 @@ jquery`); run.waitSecs(30); run.stop(); - }); + },{ + testName: 'css hot code push', + testFile: 'hot-code-push.js' }); }); selftest.define("versioning hot code push", function (options) { @@ -118,7 +120,9 @@ selftest.define("versioning hot code push", function (options) { run.forbidAll("Error listening"); run.stop(); - }); + },{ + testName: 'versioning hot code push', + testFile: 'hot-code-push.js' }); }); selftest.define("javascript hot code push", function (options) { @@ -229,8 +233,8 @@ appcache`); run.match("client connected: 0"); run.match("jsVar: undefined"); - // XXX: Remove me. This shouldn't be needed, but sometimes if we run too - // quickly on fast (or Linux?) machines, it looks like there's a race and we + // XXX: Remove me. This shouldn't be needed, but sometimes if we run too + // quickly on fast (or Linux?) machines, it looks like there's a race and we // see a weird state. Without this line this test was failing one time on // every build in CircleCI, but oddly enough would succeed on the second // try. @@ -262,5 +266,7 @@ session`); run.match("server restarted"); run.stop(); - }); + },{ + testName: 'javascript hot code push', + testFile: 'hot-code-push.js' }); }); diff --git a/tools/tests/package-tests.js b/tools/tests/package-tests.js index b362113d65..e7649496d2 100644 --- a/tools/tests/package-tests.js +++ b/tools/tests/package-tests.js @@ -369,7 +369,9 @@ selftest.define("add packages client archs", function (options) { run.match("all clients " + (expectedLogNum++)); run.match(clientType + " client " + (expectedLogNum++)); run.stop(); - }, args); + }, { args, + testName: 'add packages client archs', + testFile: 'package-tests.js' }); }; runTestWithArgs("browser", [], 3000); diff --git a/tools/tests/stylus-cross-packages.js b/tools/tests/stylus-cross-packages.js index 44692ca9df..4df4faba4f 100644 --- a/tools/tests/stylus-cross-packages.js +++ b/tools/tests/stylus-cross-packages.js @@ -1,6 +1,5 @@ var selftest = require('../tool-testing/selftest.js'); var Sandbox = selftest.Sandbox; -var utils = require('../utils/utils.js'); selftest.define("can import stylus across packages", function (options) { var s = new Sandbox({ @@ -25,5 +24,7 @@ selftest.define("can import stylus across packages", function (options) { run.match("true"); run.stop(); - }); + },{ + testName: 'can import stylus across packages', + testFile: 'stylus-cross-packages.js' }); }); diff --git a/tools/tool-testing/sandbox.js b/tools/tool-testing/sandbox.js index fcbde5ef5c..b036cb09b9 100644 --- a/tools/tool-testing/sandbox.js +++ b/tools/tool-testing/sandbox.js @@ -113,8 +113,10 @@ export default class Sandbox { // run.connectClient(); // // post-connection checks // }); - testWithAllClients(f, ...args) { - args = args.filter(arg => arg); + testWithAllClients(f, options) { + const { testName, testFile, args: argsParam } = options || {}; + + const args = (argsParam || []).filter(arg => arg); // Lazy-populate the clients, only when this method is called. if (typeof this.clients === "undefined") { @@ -140,12 +142,14 @@ export default class Sandbox { } } - console.log(`Running test with ${this.clients.length} client(s)...`); + const testNameAndFile = `${testFile ? `${testFile}: ` : ''}${testName ? `"${testName}" ` : ''}`; + + console.log(`Running test ${testNameAndFile}with ${this.clients.length} client(s)...`); Object.keys(this.clients).forEach((clientKey, index, array) => { const client = this.clients[clientKey]; console.log( - `(${index+1}/${array.length}) Testing with ${client.name}...`); + `(${index+1}/${array.length}) Testing ${testNameAndFile}with ${client.name}...`); const run = new Run(this.execPath, { sandbox: this, args,