[NC-1449] test docker quickstart (#41)

* Added docker quickstart test

- quickstart folder is now a module
- added junit tests that runs the bash scripts
- added new gradle task for testing quickstart scripts
- added jenkinsfile step for the gradle task
- switched the build image to one with docker in it
- split into nodes and docker env
- parallel jobs
- modified host usage in bash scripts to take the docker-in-docker architecture in account
Signed-off-by: Adrian Sutton <adrian.sutton@consensys.net>

Jenkinsfile

@@ -18,48 +18,70 @@ if (env.BRANCH_NAME == "master") {
     ])
 }
 
-node {
-    checkout scm
-    docker.image('docker:18.06.0-ce-dind').withRun('--privileged') { d ->
-        docker.image('openjdk:8u181-jdk').inside("--link ${d.id}:docker") {
-            try {
-                stage('Compile') {
-                    sh './gradlew --no-daemon --parallel clean compileJava'
-                }
-                stage('compile tests') {
-                    sh './gradlew --no-daemon --parallel compileTestJava'
-                }
-                stage('assemble') {
-                    sh './gradlew --no-daemon --parallel assemble'
-                }
-                stage('Build') {
-                    sh './gradlew --no-daemon --parallel build'
-                }
-                stage('Reference tests') {
-                    sh './gradlew --no-daemon --parallel referenceTest'
-                }
-                stage('Integration Tests') {
-                    sh './gradlew --no-daemon --parallel integrationTest'
-                }
-                stage('Acceptance Tests') {
-                    sh './gradlew --no-daemon --parallel acceptanceTest'
-                }
-                stage('Check Licenses') {
-                    sh './gradlew --no-daemon --parallel checkLicenses'
-                }
-                stage('Check javadoc') {
-                    sh './gradlew --no-daemon --parallel javadoc'
-                }
-                stage('Jacoco root report') {
-                    sh './gradlew --no-daemon jacocoRootReport'
-                }
-            } finally {
-                archiveArtifacts '**/build/reports/**'
-                archiveArtifacts '**/build/test-results/**'
-                archiveArtifacts 'build/reports/**'
-                archiveArtifacts 'build/distributions/**'
+stage('Pantheon tests') {
+    parallel javaTests: {
+        node {
+            checkout scm
+            docker.image('docker:18.06.0-ce-dind').withRun('--privileged') { d ->
+                docker.image('pegasyseng/pantheon-build:0.0.1').inside("--link ${d.id}:docker") {
+                    try {
+                        stage('Compile') {
+                            sh './gradlew --no-daemon --parallel clean compileJava'
+                        }
+                        stage('compile tests') {
+                            sh './gradlew --no-daemon --parallel compileTestJava'
+                        }
+                        stage('assemble') {
+                            sh './gradlew --no-daemon --parallel assemble'
+                        }
+                        stage('Build') {
+                            sh './gradlew --no-daemon --parallel build'
+                        }
+                        stage('Reference tests') {
+                            sh './gradlew --no-daemon --parallel referenceTest'
+                        }
+                        stage('Integration Tests') {
+                            sh './gradlew --no-daemon --parallel integrationTest'
+                        }
+                        stage('Acceptance Tests') {
+                            sh './gradlew --no-daemon --parallel acceptanceTest --tests Web3Sha3AcceptanceTest --tests PantheonClusterAcceptanceTest --tests MiningAcceptanceTest'
+                        }
+                        stage('Check Licenses') {
+                            sh './gradlew --no-daemon --parallel checkLicenses'
+                        }
+                        stage('Check javadoc') {
+                            sh './gradlew --no-daemon --parallel javadoc'
+                        }
+                        stage('Jacoco root report') {
+                            sh './gradlew --no-daemon jacocoRootReport'
+                        }
+                    } finally {
+                        archiveArtifacts '**/build/reports/**'
+                        archiveArtifacts '**/build/test-results/**'
+                        archiveArtifacts 'build/reports/**'
+                        archiveArtifacts 'build/distributions/**'
 
-                junit '**/build/test-results/**/*.xml'
+                        junit '**/build/test-results/**/*.xml'
+                    }
+                }
+            }
+        }
+    }, quickstartTests: {
+        node {
+            checkout scm
+            docker.image('docker:18.06.0-ce-dind').withRun('--privileged') { d ->
+                docker.image('pegasyseng/pantheon-build:0.0.1').inside("--link ${d.id}:docker") {
+                    try {
+                        stage('Docker quickstart Tests') {
+                            sh 'DOCKER_HOST=tcp://docker:2375 ./gradlew --no-daemon --parallel clean dockerQuickstartTest'
+                        }
+                    } finally {
+                        archiveArtifacts '**/build/test-results/**'
+                        archiveArtifacts '**/build/reports/**'
+
+                        junit '**/build/test-results/**/*.xml'
+                    }
+                }
             }
         }
     }

quickstart/build.gradle

@@ -0,0 +1,20 @@
+dependencies {
+  testRuntime 'org.apache.logging.log4j:log4j-core'
+  testRuntime 'org.apache.logging.log4j:log4j-slf4j-impl'
+
+  testImplementation 'junit:junit'
+  testImplementation 'org.assertj:assertj-core'
+  testImplementation 'org.web3j:core'
+  testImplementation 'org.awaitility:awaitility'
+  testImplementation 'com.squareup.okhttp3:okhttp'
+  testImplementation 'io.vertx:vertx-core'
+  testImplementation project( path: ':pantheon')
+}
+
+test.enabled = false
+
+task dockerQuickstartTest(type: Test) {
+  mustRunAfter rootProject.subprojects*.test
+  description = 'Runs Pantheon Docker quickstart tests.'
+  group = 'verification'
+}

quickstart/listQuickstartServices.sh

@@ -11,6 +11,7 @@ fi
 
 COMPOSE_CONFIG_FILE_OPTION="-f ${QUICKSTART_FOLDER}/docker-compose.yml"
 EXPLORER_SERVICE=explorer
+HOST=${DOCKER_PORT_2375_TCP_ADDR:-"localhost"}
 
 # Displays services list with port mapping
 docker-compose ${COMPOSE_CONFIG_FILE_OPTION} ps
@@ -20,14 +21,20 @@ rpcMapping=`docker-compose ${COMPOSE_CONFIG_FILE_OPTION} port rpcnode 8545`
 wsMapping=`docker-compose ${COMPOSE_CONFIG_FILE_OPTION} port rpcnode 8546`
 explorerMapping=`docker-compose ${COMPOSE_CONFIG_FILE_OPTION} port explorer 3000`
 
-# replaces the mix explorer rpc endpoint by ours
-docker-compose ${COMPOSE_CONFIG_FILE_OPTION} exec ${EXPLORER_SERVICE} /bin/sed -i \
-"s/fallbackUrlPlaceHolder/http:\/\/localhost:${rpcMapping##*:}/g" \
-src/constants/index.js
+#Check if we run in a tty before using exec and otherwise set $TERM as it fails if not set.
+if [ ! -t 1 ] ;
+then
+  export TERM=xterm-color
+fi
 
-docker-compose ${COMPOSE_CONFIG_FILE_OPTION} exec ${EXPLORER_SERVICE} /bin/sed -i \
-"s/rpcHostPlaceHolder/http:\/\/localhost:${rpcMapping##*:}/g" \
-src/components/App.js
+# replaces the mix explorer rpc endpoint by ours
+`docker-compose ${COMPOSE_CONFIG_FILE_OPTION} exec -T ${EXPLORER_SERVICE} /bin/sed -i \
+"s/fallbackUrlPlaceHolder/http:\/\/${HOST}:${rpcMapping##*:}/g" \
+src/constants/index.js`
+
+`docker-compose ${COMPOSE_CONFIG_FILE_OPTION} exec -T ${EXPLORER_SERVICE} /bin/sed -i \
+"s/rpcHostPlaceHolder/http:\/\/${HOST}:${rpcMapping##*:}/g" \
+src/components/App.js`
 
 # Displays links to exposed services
 ORANGE='\033[0;33m'
@@ -36,15 +43,21 @@ BOLD=$(tput bold)
 NORMAL=$(tput sgr0)
 
 echo "${CYAN}****************************************************************"
-echo "JSON-RPC ${BOLD}HTTP${NORMAL}${CYAN} service endpoint      : ${ORANGE}http://localhost:${rpcMapping##*:}${CYAN}   *"
-echo "JSON-RPC ${BOLD}WebSocket${NORMAL}${CYAN} service endpoint : ${ORANGE}http://localhost:${wsMapping##*:}${CYAN}   *"
-
-while [ "$(curl -m 1 -s -o /dev/null -w ''%{http_code}'' http://localhost:${explorerMapping##*:})" != "200" ]
+echo "JSON-RPC ${BOLD}HTTP${NORMAL}${CYAN} service endpoint      : ${ORANGE}http://${HOST}:${rpcMapping##*:}${CYAN}   *"
+echo "JSON-RPC ${BOLD}WebSocket${NORMAL}${CYAN} service endpoint : ${ORANGE}http://${HOST}:${wsMapping##*:}${CYAN}   *"
+dots=""
+maxRetryCount=50
+while [ "$(curl -m 1 -s -o /dev/null -w ''%{http_code}'' http://${HOST}:${explorerMapping##*:})" != "200" ] && [ ${#dots} -le ${maxRetryCount} ]
 do
-    dots=$dots"."
-    printf "${CYAN} Block explorer is starting, please wait ${ORANGE}$dots${NORMAL}\\r"
-    sleep 1
+  dots=$dots"."
+  printf "${CYAN} Block explorer is starting, please wait ${ORANGE}$dots${NORMAL}\\r"
+  sleep 1
 done
 
-echo "${CYAN}Web block explorer address          : ${ORANGE}http://localhost:${explorerMapping##*:}${CYAN}   *                                                                             "
-echo "****************************************************************${NORMAL}"
+if [ ${#dots} -gt ${maxRetryCount} ]; then
+  (>&2 echo "${ORANGE}ERROR: Web block explorer is not started at http://${HOST}:${explorerMapping##*:}$ !${CYAN}   *                                                                             ")
+  echo "****************************************************************${NORMAL}"
+else
+  echo "${CYAN}Web block explorer address          : ${ORANGE}http://${HOST}:${explorerMapping##*:}${CYAN}   *                                                                             "
+  echo "****************************************************************${NORMAL}"
+fi

quickstart/runPantheonPrivateNetwork.sh

@@ -11,8 +11,8 @@ fi
 
 COMPOSE_CONFIG_FILE_OPTION="-f ${QUICKSTART_FOLDER}/docker-compose.yml"
 
-
 # Build and run containers and network
-docker-compose ${COMPOSE_CONFIG_FILE_OPTION} up -d --scale node=4 --build
+docker-compose ${COMPOSE_CONFIG_FILE_OPTION} build --force-rm
+docker-compose ${COMPOSE_CONFIG_FILE_OPTION} up -d --scale node=4
 
 ${QUICKSTART_FOLDER}/listQuickstartServices.sh

quickstart/src/test/java/tech/pegasys/pantheon/tests/quickstart/DockerQuickstartTest.java

@@ -0,0 +1,339 @@
+package tech.pegasys.pantheon.tests.quickstart;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import tech.pegasys.pantheon.PantheonInfo;
+import tech.pegasys.pantheon.tests.quickstart.DockerQuickstartTest.Service.ExposedPort;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.lang.ProcessBuilder.Redirect;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import io.vertx.core.Vertx;
+import io.vertx.core.http.HttpClientOptions;
+import io.vertx.core.http.RequestOptions;
+import io.vertx.core.http.WebSocket;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.Response;
+import org.awaitility.Awaitility;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.web3j.protocol.Web3j;
+import org.web3j.protocol.http.HttpService;
+import org.web3j.utils.Async;
+
+public class DockerQuickstartTest {
+
+  private static final String PROJECT_ROOT =
+      Paths.get(System.getProperty("user.dir")).getParent().toString();
+  private static final String DOCKER_COMPOSE_PROJECT_PREFIX = "quickstart_";
+  private static final int DEFAULT_HTTP_RPC_PORT = 8545;
+  private static final int DEFAULT_WS_RPC_PORT = 8546;
+  private static final String DEFAULT_RPC_HOST =
+      Optional.ofNullable(System.getenv("DOCKER_PORT_2375_TCP_ADDR")).orElse("localhost");
+  private static final String DEFAULT_HTTP_RPC_HOST = "http://" + DEFAULT_RPC_HOST;
+  private final Map<ServicesIdentifier, Service> services = new EnumMap<>(ServicesIdentifier.class);
+  private final Map<EndpointsIdentifier, String> endpoints =
+      new EnumMap<>(EndpointsIdentifier.class);
+  private Web3j web3HttpClient;
+
+  @BeforeClass
+  public static void runPantheonPrivateNetwork() throws IOException, InterruptedException {
+    ProcessBuilder processBuilder = new ProcessBuilder("quickstart/runPantheonPrivateNetwork.sh");
+    processBuilder.directory(new File(PROJECT_ROOT)); // going up one level is the project root
+    processBuilder.inheritIO(); // redirect all output to logs
+    Process process = processBuilder.start();
+
+    int exitValue = process.waitFor();
+
+    if (exitValue != 0) {
+      // check for errors, error messages and causes are redirected to logs already
+      throw new RuntimeException("execution of script failed!");
+    }
+  }
+
+  @Before
+  public void listQuickstartServices() throws IOException, InterruptedException {
+    ProcessBuilder processBuilder = new ProcessBuilder("quickstart/listQuickstartServices.sh");
+    processBuilder.directory(new File(PROJECT_ROOT)); // going up one level is the project root
+    // redirect only error output to logs as we want to be able to
+    // keep the standard output available for reading
+    processBuilder.redirectError(Redirect.INHERIT);
+    Process process = processBuilder.start();
+
+    int exitValue = process.waitFor();
+
+    if (exitValue != 0) {
+      // check for errors, error messages and causes are redirected to logs already
+      throw new RuntimeException("execution of script failed!");
+    }
+
+    BufferedReader reader =
+        new BufferedReader(new InputStreamReader(process.getInputStream(), UTF_8));
+
+    reader.lines().forEach(this::populateServicesAndEndpoints);
+    reader.close();
+
+    assertThat(services).isNotNull().isNotEmpty();
+    assertThat(endpoints).isNotNull().isNotEmpty();
+
+    web3HttpClient =
+        Web3j.build(
+            new HttpService(
+                DEFAULT_HTTP_RPC_HOST
+                    + ":"
+                    + services
+                        .get(ServicesIdentifier.RPCNODE)
+                        .exposedPorts
+                        .get(DEFAULT_HTTP_RPC_PORT)
+                        .externalPort),
+            2000,
+            Async.defaultExecutorService());
+
+    assertThat(web3HttpClient).isNotNull();
+  }
+
+  private void populateServicesAndEndpoints(final String line) {
+    // We check that the output of the script displays the right endpoints and services states
+    // each endpoint and service will be stored in a map for later use.
+
+    for (ServicesIdentifier servicesIdentifier : ServicesIdentifier.values()) {
+      Matcher regexMatcher = servicesIdentifier.pattern.matcher(line);
+      if (regexMatcher.find()) {
+        Service service = new Service();
+        service.name = regexMatcher.group(1);
+        service.state = regexMatcher.group(2).toLowerCase();
+        String portMappings[] = regexMatcher.group(3).split(",", -1);
+        for (String mapping : portMappings) {
+          ExposedPort port = new ExposedPort(mapping);
+          service.exposedPorts.put(port.internalPort, port);
+        }
+        services.put(servicesIdentifier, service);
+      }
+    }
+
+    for (EndpointsIdentifier endpointsIdentifier : EndpointsIdentifier.values()) {
+      Matcher regexMatcher = endpointsIdentifier.pattern.matcher(line);
+      if (regexMatcher.find()) {
+        String endpoint = regexMatcher.group(1);
+        endpoints.put(endpointsIdentifier, endpoint);
+      }
+    }
+  }
+
+  @After
+  public void closeConnections() {
+    assertThat(web3HttpClient).isNotNull();
+    web3HttpClient.shutdown();
+  }
+
+  @AfterClass
+  public static void removePantheonPrivateNetwork() throws IOException, InterruptedException {
+    ProcessBuilder processBuilder =
+        new ProcessBuilder("quickstart/removePantheonPrivateNetwork.sh");
+    processBuilder.inheritIO(); // redirect all output to logs
+    processBuilder.directory(new File(PROJECT_ROOT)); // going up one level is the project root
+    Process process = processBuilder.start();
+
+    int exitValue = process.waitFor();
+    if (exitValue != 0) {
+      // check for errors, all output and then also error messages and causes are redirected to logs
+      throw new RuntimeException("execution of script failed!");
+    }
+  }
+
+  @Test
+  public void servicesShouldBeUp() {
+    Awaitility.await()
+        .ignoreExceptions()
+        .atMost(60, TimeUnit.SECONDS)
+        .untilAsserted(
+            () ->
+                Arrays.stream(ServicesIdentifier.values())
+                    .forEach(
+                        servicesIdentifier ->
+                            assertThat(services.get(servicesIdentifier).state).isEqualTo("up")));
+  }
+
+  @Test
+  public void servicesEndpointsShouldBeExposed() {
+    Awaitility.await()
+        .ignoreExceptions()
+        .atMost(60, TimeUnit.SECONDS)
+        .untilAsserted(
+            () ->
+                Arrays.stream(EndpointsIdentifier.values())
+                    .forEach(
+                        endpointsIdentifier ->
+                            assertThat(endpoints.get(endpointsIdentifier))
+                                .isNotNull()
+                                .isNotEmpty()));
+  }
+
+  @Test
+  public void rpcNodeShouldFindPeers() {
+    // Peers are those defined in docker-compose.yml and launched with scaling of 4 regular nodes
+    // which gives us 6 peers of the RPC node: bootnode, minernode and 4 regular nodes.
+    int expectecNumberOfPeers = 6;
+
+    Awaitility.await()
+        .ignoreExceptions()
+        .atMost(60, TimeUnit.SECONDS)
+        .untilAsserted(
+            () ->
+                assertThat(web3HttpClient.netPeerCount().send().getQuantity().intValueExact())
+                    .isEqualTo(expectecNumberOfPeers));
+  }
+
+  @Test
+  public void rpcNodeShouldReturnCorrectVersion() {
+    String expectedVersion = PantheonInfo.version();
+    Awaitility.await()
+        .ignoreExceptions()
+        .atMost(60, TimeUnit.SECONDS)
+        .untilAsserted(
+            () ->
+                assertThat(web3HttpClient.web3ClientVersion().send().getWeb3ClientVersion())
+                    .isEqualTo(expectedVersion));
+  }
+
+  @Test
+  public void mustMineSomeBlocks() {
+    // A bug occurred that failed mining after 2 blocks, so testing at least 10.
+    int expectedAtLeastBlockNumber = 10;
+    Awaitility.await()
+        .ignoreExceptions()
+        .atMost(5, TimeUnit.MINUTES)
+        .untilAsserted(
+            () ->
+                assertThat(web3HttpClient.ethBlockNumber().send().getBlockNumber().intValueExact())
+                    .isGreaterThan(expectedAtLeastBlockNumber));
+  }
+
+  @Test
+  public void webSocketRpcServiceMustConnect() {
+    RequestOptions options = new RequestOptions();
+    options.setPort(
+        services
+            .get(ServicesIdentifier.RPCNODE)
+            .exposedPorts
+            .get(DEFAULT_WS_RPC_PORT)
+            .externalPort);
+    options.setHost(DEFAULT_RPC_HOST);
+
+    final WebSocket[] wsConnection = new WebSocket[1];
+    final Vertx vertx = Vertx.vertx();
+    try {
+      vertx
+          .createHttpClient(new HttpClientOptions())
+          .websocket(options, websocket -> wsConnection[0] = websocket);
+
+      Awaitility.await()
+          .ignoreExceptions()
+          .atMost(30, TimeUnit.SECONDS)
+          .untilAsserted(() -> assertThat(wsConnection[0]).isNotNull());
+    } finally {
+      assertThat(wsConnection[0]).isNotNull();
+      wsConnection[0].close();
+    }
+  }
+
+  @Test
+  public void explorerShouldHaveModifiedHttpRpcEndpoint() throws IOException {
+    // we have to check that the sed command well replaced the endpoint placeholder with the
+    // real dynamic endpoint. But as this is a react app, we have to search for the value in the JS
+    // as we can't find it in the HTML page because source code is rendered dynamically.
+    Request request =
+        new Request.Builder()
+            .get()
+            .url(endpoints.get(EndpointsIdentifier.EXPLORER) + "/static/js/bundle.js")
+            .build();
+
+    OkHttpClient httpClient = new OkHttpClient();
+    try (Response resp = httpClient.newCall(request).execute()) {
+      assertThat(resp.code()).isEqualTo(200);
+      assertThat(resp.body()).isNotNull();
+      assertThat(resp.body().string())
+          .containsOnlyOnce(
+              "var fallbackUrl = '" + endpoints.get(EndpointsIdentifier.HTTP_RPC) + "';");
+    }
+  }
+
+  private enum ServicesIdentifier {
+    BOOTNODE,
+    EXPLORER,
+    MINERNODE,
+    NODE,
+    RPCNODE;
+
+    final Pattern pattern;
+
+    ServicesIdentifier() {
+      pattern =
+          Pattern.compile(
+              "(^"
+                  + DOCKER_COMPOSE_PROJECT_PREFIX
+                  + this.name().toLowerCase()
+                  + "_[0-9]+)\\s+.+\\s{3,}(\\w+)\\s+(.+)",
+              Pattern.DOTALL);
+    }
+  }
+
+  private enum EndpointsIdentifier {
+    EXPLORER("Web block explorer address"),
+    HTTP_RPC("JSON-RPC.+HTTP.+service endpoint"),
+    WS_RPC("JSON-RPC.+WebSocket.+service endpoint");
+
+    final Pattern pattern;
+
+    EndpointsIdentifier(final String lineLabel) {
+      pattern = Pattern.compile(lineLabel + ".+(http://.+:[0-9]+)", Pattern.DOTALL);
+    }
+  }
+
+  static class Service {
+
+    String name;
+    String state;
+    Map<Integer, ExposedPort> exposedPorts = new HashMap<>();
+
+    static class ExposedPort {
+
+      final Integer internalPort;
+      final Integer externalPort;
+      private final Pattern pattern = Pattern.compile("[0-9]+", Pattern.DOTALL);
+
+      ExposedPort(final String portDescription) {
+        if (portDescription.contains("->")) {
+          String[] ports = portDescription.split("->", 2);
+
+          String[] internalPortInfos = ports[1].split("/", 2);
+          internalPort = Integer.valueOf(internalPortInfos[0]);
+          String[] externalPortInfos = ports[0].split(":", 2);
+
+          externalPort = Integer.valueOf(externalPortInfos[1]);
+        } else {
+          Matcher regexMatcher = pattern.matcher(portDescription);
+          internalPort = regexMatcher.find() ? Integer.valueOf(regexMatcher.group(0)) : null;
+          externalPort = null;
+        }
+      }
+    }
+  }
+}

quickstart/src/test/resources/log4j2.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="INFO" monitorInterval="30">
+  <Properties>
+    <Property name="root.log.level">INFO</Property>
+  </Properties>
+  <Appenders>
+    <Console name="Console" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSSZZZ} | %t | %-5level | %c{1} | %msg%n" />
+  </Console>
+  </Appenders>
+  <Loggers>
+    <Root level="${sys:root.log.level}">
+      <AppenderRef ref="Console" />
+    </Root>
+  </Loggers>
+</Configuration>

settings.gradle

@@ -19,3 +19,4 @@ include 'services:kvstore'
 include 'testutil'
 include 'util'
 include 'errorprone-checks'
+include 'quickstart'