split ATs into privacy/permissioning/etc and the rest (#4989)

* split ATs into privacy/permissioning/etc and the rest

* don't run non-mainnet ATs on PR builds

Signed-off-by: Sally MacFarlane <macfarla.github@gmail.com>

.circleci/config.yml

@@ -225,8 +225,8 @@ jobs:
       - attach_workspace:
           at: ~/project
       - run:
-          name: AcceptanceTests
-          no_output_timeout: 30m
+          name: AcceptanceTests (Mainnet)
+          no_output_timeout: 20m
           command: |
             CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \
               | sed 's@.*/src/test/java/@@' \
@@ -235,7 +235,29 @@ jobs:
               | circleci tests split --split-by=timings --timings-type=classname)
             # Format the arguments to "./gradlew test"
             GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}')
-            ./gradlew --no-daemon acceptanceTest $GRADLE_ARGS
+            ./gradlew --no-daemon acceptanceTestMainnet $GRADLE_ARGS
+      - capture_test_results
+      - capture_test_logs
+
+  acceptanceTestsNonMainnet:
+    parallelism: 6
+    executor: xl_machine_executor
+    steps:
+      - prepare
+      - attach_workspace:
+          at: ~/project
+      - run:
+          name: AcceptanceTests (Non-Mainnet)
+          no_output_timeout: 20m
+          command: |
+            CLASSNAMES=$(circleci tests glob "acceptance-tests/tests/src/test/java/**/*.java" \
+              | sed 's@.*/src/test/java/@@' \
+              | sed 's@/@.@g' \
+              | sed 's/.\{5\}$//' \
+              | circleci tests split --split-by=timings --timings-type=classname)
+            # Format the arguments to "./gradlew test"
+            GRADLE_ARGS=$(echo $CLASSNAMES | awk '{for (i=1; i<=NF; i++) print "--tests",$i}')
+            ./gradlew --no-daemon acceptanceTestNonMainnet $GRADLE_ARGS
       - capture_test_results
       - capture_test_logs
 
@@ -439,3 +461,4 @@ workflows:
     jobs:
       - assemble
       - dockerScan
+      - acceptanceTestsNonMainnet

acceptance-tests/tests/build.gradle

@@ -106,7 +106,62 @@ task acceptanceTest(type: Test) {
   systemProperty 'acctests.runBesuAsProcess', 'true'
   systemProperty 'java.security.properties', "${buildDir}/resources/test/acceptanceTesting.security"
   mustRunAfter rootProject.subprojects*.test
-  description = 'Runs Besu acceptance tests.'
+  description = 'Runs ALL Besu acceptance tests (mainnet and non-mainnet).'
+  group = 'verification'
+
+  jvmArgs "-XX:ErrorFile=${buildDir}/jvmErrorLogs/java_err_pid%p.log"
+
+  testLogging {
+    exceptionFormat = 'full'
+    showStackTraces = true
+    showStandardStreams = Boolean.getBoolean('acctests.showStandardStreams')
+    showExceptions = true
+    showCauses = true
+  }
+
+  doFirst { mkdir "${buildDir}/jvmErrorLogs" }
+}
+task acceptanceTestMainnet(type: Test) {
+  inputs.property "integration.date", LocalTime.now() // so it runs at every invocation
+  exclude '**/bft/**'
+  exclude '**/clique/**'
+  exclude '**/permissioning/**'
+  exclude '**/privacy/**'
+
+  dependsOn(rootProject.installDist)
+  setSystemProperties(test.getSystemProperties())
+  systemProperty 'acctests.runBesuAsProcess', 'true'
+  systemProperty 'java.security.properties', "${buildDir}/resources/test/acceptanceTesting.security"
+  mustRunAfter rootProject.subprojects*.test
+  description = 'Runs MAINNET Besu acceptance tests (excluding permissioning, privacy and some other stable features).'
+  group = 'verification'
+
+  jvmArgs "-XX:ErrorFile=${buildDir}/jvmErrorLogs/java_err_pid%p.log"
+
+  testLogging {
+    exceptionFormat = 'full'
+    showStackTraces = true
+    showStandardStreams = Boolean.getBoolean('acctests.showStandardStreams')
+    showExceptions = true
+    showCauses = true
+  }
+
+  doFirst { mkdir "${buildDir}/jvmErrorLogs" }
+}
+
+task acceptanceTestNonMainnet(type: Test) {
+  inputs.property "integration.date", LocalTime.now() // so it runs at every invocation
+  include '**/bft/**'
+  include '**/clique/**'
+  include '**/permissioning/**'
+  include '**/privacy/**'
+
+  dependsOn(rootProject.installDist)
+  setSystemProperties(test.getSystemProperties())
+  systemProperty 'acctests.runBesuAsProcess', 'true'
+  systemProperty 'java.security.properties', "${buildDir}/resources/test/acceptanceTesting.security"
+  mustRunAfter rootProject.subprojects*.test
+  description = 'Runs NON-MAINNET Besu acceptance tests for permissioning and privacy (excluding mainnet features).'
   group = 'verification'
 
   jvmArgs "-XX:ErrorFile=${buildDir}/jvmErrorLogs/java_err_pid%p.log"

container-tests/tests/build.gradle

@@ -29,7 +29,7 @@ dependencies {
 }
 
 test.enabled = false
-
+// TODO make this run only on nightly
 task containerTests(type: Test) {
   description = 'Runs GoQuorum <> Besu container tests.'
   dependsOn(rootProject.distDocker)