From 44f7b43bbb7718355a050b1d18ee5d459776aa9f Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 30 Aug 2020 02:35:09 -0500 Subject: [PATCH 01/13] retryOrFailTest --- script/test | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/script/test b/script/test index 0825b3abd..bc000ddef 100755 --- a/script/test +++ b/script/test @@ -116,6 +116,24 @@ function spawnTest(executablePath, testArguments, options, callback, testName, f } +const retryNumber = 1 // the number of times a tests repeats +const retriedTests = new Map() // a cache of retried tests + +// Retries the tests if it is timed out for a number of times. Fails the rest of the tests or those that are tried enough times. +function retryOrFailTest(stderrOutput, executablePath, testArguments, options, callback, testName, finalize) { + const testKey = createTestKey(executablePath, testArguments, testName) + if (isTimedOut(stderrOutput) && shouldTryAgain(testKey)) { + // retry the timed out test + let triedNumber = retriedTests.get(testKey) || 0 + retriedTests.set(testKey, triedNumber + 1) + console.warn(`\n##[warning] Retrying the timed out step: ${testName} \n`) + spawnTest(executablePath, testArguments, options, callback, testName, finalize) + } else { + // fail the test + console.log(`##[error] Tests for ${testName} failed.`.red) + console.log(stderrOutput) + } +} function runCoreMainProcessTests (callback) { const testPath = path.join(CONFIG.repositoryRootPath, 'spec', 'main-process') const testArguments = [ From 3b10e6320413d89fdbfc3b76155f4db15d42b654 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 30 Aug 2020 02:35:15 -0500 Subject: [PATCH 02/13] createTestKey --- script/test | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/script/test b/script/test index bc000ddef..598710f68 100755 --- a/script/test +++ b/script/test @@ -134,6 +134,11 @@ function retryOrFailTest(stderrOutput, executablePath, testArguments, options, c console.log(stderrOutput) } } + +// creates a key that is specific to a certain test +function createTestKey(executablePath, testArguments, testName) { + return `${executablePath} ${testArguments.join(' ')} ${testName}` +} function runCoreMainProcessTests (callback) { const testPath = path.join(CONFIG.repositoryRootPath, 'spec', 'main-process') const testArguments = [ From d7f5ea58e058c5a96b572b2ae5f0e63f26d8f751 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 30 Aug 2020 02:35:20 -0500 Subject: [PATCH 03/13] isTimedOut --- script/test | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/script/test b/script/test index 598710f68..80b2e1bae 100755 --- a/script/test +++ b/script/test @@ -139,6 +139,15 @@ function retryOrFailTest(stderrOutput, executablePath, testArguments, options, c function createTestKey(executablePath, testArguments, testName) { return `${executablePath} ${testArguments.join(' ')} ${testName}` } + +// check if a test is timed out +function isTimedOut(stderrOutput) { + if (stderrOutput) { + return ( stderrOutput.includes("timeout: timed out after") ) + } else { + return false + } +} function runCoreMainProcessTests (callback) { const testPath = path.join(CONFIG.repositoryRootPath, 'spec', 'main-process') const testArguments = [ From 5d09c889277dd8053fd9349cf374d088d9ba0c57 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 30 Aug 2020 02:35:29 -0500 Subject: [PATCH 04/13] shouldTryAgain --- script/test | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/script/test b/script/test index 80b2e1bae..2df252dce 100755 --- a/script/test +++ b/script/test @@ -148,6 +148,17 @@ function isTimedOut(stderrOutput) { return false } } + +// check if a tests should be tried again +function shouldTryAgain(testKey) { + if (retriedTests.has(testKey)) { + return (retriedTests.get(testKey) < retryNumber) + } else { + return true + } +} + + function runCoreMainProcessTests (callback) { const testPath = path.join(CONFIG.repositoryRootPath, 'spec', 'main-process') const testArguments = [ From 7063b48f05fb1c572e43d4b4020ef88a48e5b818 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 30 Aug 2020 03:38:33 -0500 Subject: [PATCH 05/13] Retry the timed out tests for a number of times --- script/test | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/script/test b/script/test index 2df252dce..21460d388 100755 --- a/script/test +++ b/script/test @@ -102,16 +102,15 @@ function spawnTest(executablePath, testArguments, options, callback, testName, f // on close cp.on('close', exitCode => { - if (exitCode !== 0) { - console.log(`##[error] Tests for ${testName} failed.`.red) - console.log(stderrOutput) - } - if (finalize) { finalize() } // if finalizer provided - callback(null, { - exitCode, - step: testName, - testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`, - }) + if (finalize) { finalize() } // if finalizer provided + callback(null, { + exitCode, + step: testName, + testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`, + }) + if (exitCode !== 0) { + retryOrFailTest(stderrOutput, executablePath, testArguments, options, callback, testName, finalize) + } }) } From 4f0acf4a83a352cd219ae28e5f79fd847f732528 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 30 Aug 2020 05:48:13 -0500 Subject: [PATCH 06/13] call the callback after retrying --- script/test | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/script/test b/script/test index 21460d388..5dc0c255e 100755 --- a/script/test +++ b/script/test @@ -103,14 +103,14 @@ function spawnTest(executablePath, testArguments, options, callback, testName, f // on close cp.on('close', exitCode => { if (finalize) { finalize() } // if finalizer provided - callback(null, { - exitCode, - step: testName, - testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`, - }) if (exitCode !== 0) { retryOrFailTest(stderrOutput, executablePath, testArguments, options, callback, testName, finalize) } + callback(null, { + exitCode, + step: testName, + testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`, + }) }) } From 8450f16bdb5d30b16e20e68cd4dd16c44b1784fd Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 30 Aug 2020 08:52:37 -0500 Subject: [PATCH 07/13] call the callback only on fail or success Don't call the callback on retry --- script/test | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/script/test b/script/test index 5dc0c255e..946a4ff51 100755 --- a/script/test +++ b/script/test @@ -100,17 +100,18 @@ function spawnTest(executablePath, testArguments, options, callback, testName, f callback(error) }) - // on close + // on close cp.on('close', exitCode => { - if (finalize) { finalize() } // if finalizer provided - if (exitCode !== 0) { - retryOrFailTest(stderrOutput, executablePath, testArguments, options, callback, testName, finalize) - } - callback(null, { - exitCode, - step: testName, - testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`, - }) + if (finalize) { finalize() } // if finalizer provided + if (exitCode !== 0) { + retryOrFailTest(stderrOutput, executablePath, testArguments, options, callback, testName, finalize) + } else { + callback(null, { + exitCode, + step: testName, + testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`, + }) + } }) } @@ -131,6 +132,11 @@ function retryOrFailTest(stderrOutput, executablePath, testArguments, options, c // fail the test console.log(`##[error] Tests for ${testName} failed.`.red) console.log(stderrOutput) + callback(null, { + exitCode, + step: testName, + testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`, + }) } } From c48aacd0e7642777be8ecac8bba4ea98bebcb5cc Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 30 Aug 2020 21:13:47 -0500 Subject: [PATCH 08/13] pass exitCode --- script/test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/test b/script/test index 946a4ff51..bc9a98361 100755 --- a/script/test +++ b/script/test @@ -104,7 +104,7 @@ function spawnTest(executablePath, testArguments, options, callback, testName, f cp.on('close', exitCode => { if (finalize) { finalize() } // if finalizer provided if (exitCode !== 0) { - retryOrFailTest(stderrOutput, executablePath, testArguments, options, callback, testName, finalize) + retryOrFailTest(stderrOutput, exitCode, executablePath, testArguments, options, callback, testName, finalize) } else { callback(null, { exitCode, @@ -120,7 +120,7 @@ const retryNumber = 1 // the number of times a tests repeats const retriedTests = new Map() // a cache of retried tests // Retries the tests if it is timed out for a number of times. Fails the rest of the tests or those that are tried enough times. -function retryOrFailTest(stderrOutput, executablePath, testArguments, options, callback, testName, finalize) { +function retryOrFailTest(stderrOutput, exitCode, executablePath, testArguments, options, callback, testName, finalize) { const testKey = createTestKey(executablePath, testArguments, testName) if (isTimedOut(stderrOutput) && shouldTryAgain(testKey)) { // retry the timed out test From e187bf26cbaa5b97738ff41454a63b2df169cb78 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Mon, 31 Aug 2020 03:34:52 -0500 Subject: [PATCH 09/13] Retry 4 times --- script/test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/test b/script/test index bc9a98361..aaf8abec4 100755 --- a/script/test +++ b/script/test @@ -116,7 +116,7 @@ function spawnTest(executablePath, testArguments, options, callback, testName, f } -const retryNumber = 1 // the number of times a tests repeats +const retryNumber = 4 // the number of times a tests repeats const retriedTests = new Map() // a cache of retried tests // Retries the tests if it is timed out for a number of times. Fails the rest of the tests or those that are tried enough times. From 0572568ef5104b07d2afbb18d95513fcf3a5ba2e Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Mon, 14 Sep 2020 16:04:18 -0500 Subject: [PATCH 10/13] increase the retryNumber to 6 --- script/test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/test b/script/test index aaf8abec4..149eacf33 100755 --- a/script/test +++ b/script/test @@ -116,7 +116,7 @@ function spawnTest(executablePath, testArguments, options, callback, testName, f } -const retryNumber = 4 // the number of times a tests repeats +const retryNumber = 6 // the number of times a tests repeats const retriedTests = new Map() // a cache of retried tests // Retries the tests if it is timed out for a number of times. Fails the rest of the tests or those that are tried enough times. From 30c35eec46bda17b1077484b8d9477bbc5f3c784 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Mon, 14 Sep 2020 16:07:50 -0500 Subject: [PATCH 11/13] add Error Downloading Update to the retry commands --- script/test | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/script/test b/script/test index 149eacf33..c9bc5c202 100755 --- a/script/test +++ b/script/test @@ -148,7 +148,10 @@ function createTestKey(executablePath, testArguments, testName) { // check if a test is timed out function isTimedOut(stderrOutput) { if (stderrOutput) { - return ( stderrOutput.includes("timeout: timed out after") ) + return ( + stderrOutput.includes("timeout: timed out after") || + stderrOutput.includes("Error Downloading Update: Could not get code signature for running application") + ) } else { return false } From 7d14d560c16a140a3bc83c453a7efd8801b7ef15 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 15 Sep 2020 21:10:35 -0500 Subject: [PATCH 12/13] Finalize in the end (not between retry attempts) don't finalize until the end which the test fails or passes. --- script/test | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/test b/script/test index c9bc5c202..1a9bf2aa4 100755 --- a/script/test +++ b/script/test @@ -102,10 +102,11 @@ function spawnTest(executablePath, testArguments, options, callback, testName, f // on close cp.on('close', exitCode => { - if (finalize) { finalize() } // if finalizer provided if (exitCode !== 0) { retryOrFailTest(stderrOutput, exitCode, executablePath, testArguments, options, callback, testName, finalize) } else { + // successful test + if (finalize) { finalize() } // if finalizer provided callback(null, { exitCode, step: testName, @@ -130,6 +131,7 @@ function retryOrFailTest(stderrOutput, exitCode, executablePath, testArguments, spawnTest(executablePath, testArguments, options, callback, testName, finalize) } else { // fail the test + if (finalize) { finalize() } // if finalizer provided console.log(`##[error] Tests for ${testName} failed.`.red) console.log(stderrOutput) callback(null, { From f3e9434410037711c8de7629833f1206cae41ac6 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Thu, 17 Sep 2020 19:51:47 -0500 Subject: [PATCH 13/13] add core main timeout retry trigger --- script/test | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/test b/script/test index 1a9bf2aa4..72a8a6e82 100755 --- a/script/test +++ b/script/test @@ -151,8 +151,9 @@ function createTestKey(executablePath, testArguments, testName) { function isTimedOut(stderrOutput) { if (stderrOutput) { return ( - stderrOutput.includes("timeout: timed out after") || - stderrOutput.includes("Error Downloading Update: Could not get code signature for running application") + stderrOutput.includes("timeout: timed out after") || // happens in core renderer tests + stderrOutput.includes("Error: timeout of") || // happens in core main tests + stderrOutput.includes("Error Downloading Update: Could not get code signature for running application") // happens in github tests ) } else { return false