diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 2581ffd7..c6dad762 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -30,6 +30,9 @@ on: traces_api_facade_changed: required: true type: string + transaction_exclusion_api_changed: + required: true + type: string coordinator_image_tagged: required: true type: string @@ -42,6 +45,9 @@ on: traces_api_facade_image_tagged: required: true type: string + transaction_exclusion_api_image_tagged: + required: true + type: string secrets: DOCKERHUB_USERNAME: required: true @@ -96,3 +102,15 @@ jobs: untested_tag_suffix: ${{ inputs.untested_tag_suffix }} image_name: consensys/linea-traces-api-facade secrets: inherit + + transaction_exclusion_api: + uses: ./.github/workflows/transaction-exclusion-api-build-and-publish.yml + if: ${{ always() && (inputs.transaction_exclusion_api_changed == 'true' || inputs.transaction_exclusion_api_image_tagged != 'true') }} + with: + commit_tag: ${{ inputs.commit_tag }} + last_commit_tag: ${{ inputs.last_commit_tag }} + common_ancestor_tag: ${{ inputs.common_ancestor_tag }} + develop_tag: ${{ inputs.develop_tag }} + untested_tag_suffix: ${{ inputs.untested_tag_suffix }} + image_name: consensys/linea-transaction-exclusion-api + secrets: inherit diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1751b2d4..94241b45 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,8 +18,9 @@ jobs: postman: ${{ steps.filter.outputs.postman }} prover: ${{ steps.filter.outputs.prover }} traces-api-facade: ${{ steps.filter.outputs.traces-api-facade }} + transaction-exclusion-api: ${{ steps.filter.outputs.transaction-exclusion-api }} finalized-tag-updater: ${{ steps.filter.outputs.finalized-tag-updater }} - no-changes: ${{ steps.filter.outputs.coordinator == 'false' && steps.filter.outputs.postman == 'false' && steps.filter.outputs.prover == 'false' && steps.filter.outputs.traces-api-facade == 'false' }} + no-changes: ${{ steps.filter.outputs.coordinator == 'false' && steps.filter.outputs.postman == 'false' && steps.filter.outputs.prover == 'false' && steps.filter.outputs.traces-api-facade == 'false' && steps.filter.outputs.transaction-exclusion-api == 'false' }} steps: - name: Checkout uses: actions/checkout@v3 @@ -83,6 +84,25 @@ jobs: - 'build.gradle' - 'gradle.properties' - 'settings.gradle' + transaction-exclusion-api: + - 'transaction-exclusion-api/**' + - 'jvm-libs/generic/extensions/futures/**' + - 'jvm-libs/generic/extensions/kotlin/**' + - 'jvm-libs/generic/json-rpc/**' + - 'jvm-libs/generic/persistence/**' + - 'jvm-libs/generic/vertx-helper/**' + - 'jvm-libs/linea/core/long-running-service/**' + - 'jvm-libs/linea/core/metrics/**' + - 'jvm-libs/linea/metrics/**' + - '.github/workflows/transaction-exclusion-api-*.yml' + - '.github/workflows/build-and-publish.yml' + - '.github/workflows/main.yml' + - '.github/workflows/reuse-*.yml' + - 'buildSrc/**' + - 'gradle/**' + - 'build.gradle' + - 'gradle.properties' + - 'settings.gradle' finalized-tag-updater: - 'jvm-libs/linea/core/long-running-service/**' - 'jvm-libs/linea/web3j-extensions/**' @@ -105,6 +125,7 @@ jobs: postman_changed: ${{ needs.filter-commit-changes.outputs.postman }} prover_changed: ${{ needs.filter-commit-changes.outputs.prover }} traces_api_facade_changed: ${{ needs.filter-commit-changes.outputs.traces-api-facade }} + transaction_exclusion_api_changed: ${{ needs.filter-commit-changes.outputs.transaction-exclusion-api }} secrets: inherit manual-docker-build-and-e2e-tests: @@ -129,10 +150,12 @@ jobs: postman_changed: ${{ needs.filter-commit-changes.outputs.postman }} prover_changed: ${{ needs.filter-commit-changes.outputs.prover }} traces_api_facade_changed: ${{ needs.filter-commit-changes.outputs.traces-api-facade }} + transaction_exclusion_api_changed: ${{ needs.filter-commit-changes.outputs.transaction-exclusion-api }} coordinator_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_coordinator }} postman_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_postman }} prover_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_prover }} traces_api_facade_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_traces_api_facade }} + transaction_exclusion_api_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_transaction_exclusion_api }} secrets: inherit # Comment out the auto build and release step below as the plugin release should be @@ -154,9 +177,11 @@ jobs: postman_changed: ${{ needs.filter-commit-changes.outputs.postman }} prover_changed: ${{ needs.filter-commit-changes.outputs.prover }} traces_api_facade_changed: ${{ needs.filter-commit-changes.outputs.traces-api-facade }} + transaction_exclusion_api_changed: ${{ needs.filter-commit-changes.outputs.transaction-exclusion-api }} coordinator_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_coordinator }} postman_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_postman }} traces_api_facade_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_traces_api_facade }} + transaction_exclusion_api_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_transaction_exclusion_api }} secrets: inherit run-e2e-tests-geth-tracing: @@ -195,7 +220,7 @@ jobs: commit_tag: ${{ needs.store-image-name-and-tags.outputs.commit_tag }} develop_tag: ${{ needs.store-image-name-and-tags.outputs.develop_tag }} untested_tag_suffix: ${{ needs.store-image-name-and-tags.outputs.untested_tag_suffix }} - image_names: '["consensys/linea-coordinator", "consensys/linea-postman", "consensys/linea-prover", "consensys/linea-traces-api-facade"]' + image_names: '["consensys/linea-coordinator", "consensys/linea-postman", "consensys/linea-prover", "consensys/linea-traces-api-facade", "consensys/linea-transaction-exclusion-api"]' secrets: inherit cleanup-deployments: diff --git a/.github/workflows/reuse-check-images-tags-and-push.yml b/.github/workflows/reuse-check-images-tags-and-push.yml index ab2e4d6e..4d9ae8d2 100644 --- a/.github/workflows/reuse-check-images-tags-and-push.yml +++ b/.github/workflows/reuse-check-images-tags-and-push.yml @@ -29,6 +29,9 @@ on: traces_api_facade_changed: required: true type: string + transaction_exclusion_api_changed: + required: true + type: string outputs: image_tagged_coordinator: value: ${{ jobs.image_tag_push.outputs.image_tagged_coordinator }} @@ -38,6 +41,8 @@ on: value: ${{ jobs.image_tag_push.outputs.image_tagged_postman }} image_tagged_traces_api_facade: value: ${{ jobs.image_tag_push.outputs.image_tagged_traces_api_facade }} + image_tagged_transaction_exclusion_api: + value: ${{ jobs.image_tag_push.outputs.image_tagged_transaction_exclusion_api }} secrets: DOCKERHUB_USERNAME: required: true @@ -61,6 +66,8 @@ jobs: common_ancestor_commit_tag_exists_prover: ${{ steps.check_image_tags_exist_prover.outputs.common_ancestor_commit_tag_exists }} last_commit_tag_exists_traces_api_facade: ${{ steps.check_image_tags_exist_traces_api_facade.outputs.last_commit_tag_exists }} common_ancestor_commit_tag_exists_traces_api_facade: ${{ steps.check_image_tags_exist_traces_api_facade.outputs.common_ancestor_commit_tag_exists }} + last_commit_tag_exists_transaction_exclusion_api: ${{ steps.check_image_tags_exist_transaction_exclusion_api.outputs.last_commit_tag_exists }} + common_ancestor_commit_tag_exists_transaction_exclusion_api: ${{ steps.check_image_tags_exist_transaction_exclusion_api.outputs.common_ancestor_commit_tag_exists }} steps: - name: Checkout uses: actions/checkout@v4 @@ -108,6 +115,17 @@ jobs: image_name: consensys/linea-traces-api-facade docker_username: ${{ secrets.DOCKERHUB_USERNAME }} docker_password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Check image tags exist for transaction-exclusion-api + uses: ./.github/actions/check-image-tags-exist + if: ${{ inputs.transaction_exclusion_api_changed == 'false' }} + id: check_image_tags_exist_transaction_exclusion_api + with: + last_commit_tag: ${{ inputs.last_commit_tag }} + common_ancestor_tag: ${{ inputs.common_ancestor_tag }} + image_name: consensys/linea-transaction-exclusion-api + docker_username: ${{ secrets.DOCKERHUB_USERNAME }} + docker_password: ${{ secrets.DOCKERHUB_TOKEN }} image_tag_push: runs-on: [self-hosted, ubuntu-20.04, X64, small] @@ -118,6 +136,7 @@ jobs: image_tagged_prover: ${{ steps.image_tag_push_prover.outputs.image_tagged }} image_tagged_postman: ${{ steps.image_tag_push_postman.outputs.image_tagged }} image_tagged_traces_api_facade: ${{ steps.image_tag_push_traces_api_facade.outputs.image_tagged }} + image_tagged_transaction_exclusion_api: ${{ steps.image_tag_push_transaction_exclusion_api.outputs.image_tagged }} steps: - name: Checkout uses: actions/checkout@v4 @@ -185,3 +204,19 @@ jobs: common_ancestor_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.common_ancestor_commit_tag_exists_traces_api_facade }} docker_username: ${{ secrets.DOCKERHUB_USERNAME }} docker_password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Tag and push transaction exclusion api image + id: image_tag_push_transaction_exclusion_api + uses: ./.github/actions/image-tag-and-push + if: ${{ inputs.transaction_exclusion_api_changed == 'false' }} + with: + commit_tag: ${{ inputs.commit_tag }} + last_commit_tag: ${{ inputs.last_commit_tag }} + common_ancestor_tag: ${{ inputs.common_ancestor_tag }} + develop_tag: ${{ inputs.develop_tag }} + untested_tag_suffix: ${{ inputs.untested_tag_suffix }} + image_name: consensys/linea-transaction-exclusion-api + last_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.last_commit_tag_exists_transaction_exclusion_api }} + common_ancestor_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.common_ancestor_commit_tag_exists_transaction_exclusion_api }} + docker_username: ${{ secrets.DOCKERHUB_USERNAME }} + docker_password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml index 4df31eae..a5cb81be 100644 --- a/.github/workflows/reuse-run-e2e-tests.yml +++ b/.github/workflows/reuse-run-e2e-tests.yml @@ -72,6 +72,7 @@ jobs: POSTMAN_TAG: ${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} PROVER_TAG: ${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} TRACES_API_TAG: ${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} + TRANSACTION_EXCLUSION_API_TAG: ${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }} outputs: tests_outcome: ${{ steps.run_e2e_tests.outcome }} @@ -165,6 +166,8 @@ jobs: docker logs postman --since 1h &>> docker_logs/postman.txt docker logs traces-node --since 1h &>> docker_logs/traces-node.txt docker logs traces-node-v2 --since 1h &>> docker_logs/traces-node-v2.txt; + docker logs l2-node-besu --since 1h &>> docker_logs/l2-node-besu.txt; + docker logs transaction-exclusion-api --since 1h &>> docker_logs/transaction-exclusion-api.txt docker logs sequencer --since 1h &>> docker_logs/sequencer.txt - name: Archive debug logs uses: actions/upload-artifact@v4 diff --git a/.github/workflows/reuse-store-image-name-and-tags.yml b/.github/workflows/reuse-store-image-name-and-tags.yml index bad0023f..e16c23d6 100644 --- a/.github/workflows/reuse-store-image-name-and-tags.yml +++ b/.github/workflows/reuse-store-image-name-and-tags.yml @@ -42,7 +42,7 @@ jobs: - name: Compute version tags id: step2 run: | - echo COMMIT_TAG=$(git rev-parse --short "$GITHUB_SHA") >> $GITHUB_OUTPUT + echo COMMIT_TAG=$(git rev-parse --short "${{ github.event.pull_request.head.sha }}") >> $GITHUB_OUTPUT echo LAST_COMMIT_TAG=$(git rev-parse --short "${{ env.EVENT_BEFORE }}") >> $GITHUB_OUTPUT echo DEVELOP_TAG=develop >> $GITHUB_OUTPUT echo COMMON_ANCESTOR_TAG=$(git rev-parse --short "${{ env.COMMON_ANCESTOR }}") >> $GITHUB_OUTPUT diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ac0ef42d..571fc7cc 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -18,6 +18,9 @@ on: traces_api_facade_changed: required: true type: string + transaction_exclusion_api_changed: + required: true + type: string coordinator_image_tagged: required: true type: string @@ -27,6 +30,9 @@ on: traces_api_facade_image_tagged: required: true type: string + transaction_exclusion_api_image_tagged: + required: true + type: string secrets: DOCKERHUB_USERNAME: required: true @@ -55,3 +61,8 @@ jobs: uses: ./.github/workflows/traces-api-facade-testing.yml if: ${{ always() && (inputs.traces_api_facade_changed == 'true' || inputs.traces_api_facade_image_tagged != 'true') }} secrets: inherit + + transaction-exclusion-api: + uses: ./.github/workflows/transaction-exclusion-api-testing.yml + if: ${{ always() && (inputs.transaction_exclusion_api_changed == 'true' || inputs.transaction_exclusion_api_image_tagged != 'true') }} + secrets: inherit diff --git a/.github/workflows/traces-api-facade-testing.yml b/.github/workflows/traces-api-facade-testing.yml index 8b9b0823..6df13304 100644 --- a/.github/workflows/traces-api-facade-testing.yml +++ b/.github/workflows/traces-api-facade-testing.yml @@ -43,7 +43,7 @@ jobs: retry_on: error timeout_minutes: 20 command: | - ./gradlew traces-api-facade:app:buildNeeded jacocoRootReport + ./gradlew -V traces-api-facade:app:buildNeeded jacocoRootReport - name: Run tests without coverage uses: nick-fields/retry@v2 if: ${{ !inputs.coverage }} @@ -52,4 +52,4 @@ jobs: retry_on: error timeout_minutes: 20 command: | - ./gradlew traces-api-facade:app:buildNeeded + ./gradlew -V traces-api-facade:app:buildNeeded diff --git a/.github/workflows/transaction-exclusion-api-build-and-publish.yml b/.github/workflows/transaction-exclusion-api-build-and-publish.yml new file mode 100644 index 00000000..3a1a3ce0 --- /dev/null +++ b/.github/workflows/transaction-exclusion-api-build-and-publish.yml @@ -0,0 +1,80 @@ +name: Transaction-Exclusion-Api build and publish CI + +on: + workflow_call: + inputs: + commit_tag: + required: true + type: string + last_commit_tag: + required: true + type: string + common_ancestor_tag: + required: true + type: string + develop_tag: + required: true + type: string + untested_tag_suffix: + required: true + type: string + image_name: + required: true + type: string + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_TOKEN: + required: true + +jobs: + build-and-publish: + runs-on: [self-hosted, ubuntu-20.04, X64, small] + name: Transaction exclusion api build + env: + COMMIT_TAG: ${{ inputs.commit_tag }} + DEVELOP_TAG: ${{ inputs.develop_tag }} + UNTESTED_TAG_SUFFIX: ${{ inputs.untested_tag_suffix }} + IMAGE_NAME: ${{ inputs.image_name }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ github.head_ref }} + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + - name: Build dist + run: | + ./gradlew transaction-exclusion-api:app:distZip --no-daemon + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Docker meta + id: transaction-exclusion-api + uses: docker/metadata-action@v3 + with: + images: consensys/linea-transaction-exclusion-api + - name: Build & push + uses: docker/build-push-action@v6 + with: + context: . + build-contexts: zip=./transaction-exclusion-api/app/build/distributions/ + file: ./transaction-exclusion-api/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }}-${{ env.UNTESTED_TAG_SUFFIX }} + cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache + cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache,mode=max diff --git a/.github/workflows/transaction-exclusion-api-testing.yml b/.github/workflows/transaction-exclusion-api-testing.yml new file mode 100644 index 00000000..77fd024d --- /dev/null +++ b/.github/workflows/transaction-exclusion-api-testing.yml @@ -0,0 +1,41 @@ +name: transaction-exclusion-api-testing + +on: + workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_TOKEN: + required: true + workflow_dispatch: + inputs: + coverage: + description: To generate test report + required: false + type: boolean + default: false + +jobs: + run-tests: + runs-on: [self-hosted, ubuntu-20.04, X64, small] + name: Transaction exclusion api tests + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ github.head_ref }} + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + - name: Run tests with coverage + if: ${{ inputs.coverage }} + run: | + ./gradlew transaction-exclusion-api:app:buildNeeded jacocoRootReport + - name: Run tests without coverage + if: ${{ !inputs.coverage }} + run: | + ./gradlew transaction-exclusion-api:app:buildNeeded diff --git a/.run/TransactionExclusionApi.run.xml.template b/.run/TransactionExclusionApi.run.xml.template new file mode 100644 index 00000000..4669d5e4 --- /dev/null +++ b/.run/TransactionExclusionApi.run.xml.template @@ -0,0 +1,23 @@ + + + + diff --git a/Makefile b/Makefile index d26be8c1..bca9ec1b 100644 --- a/Makefile +++ b/Makefile @@ -48,15 +48,9 @@ start-whole-environment: # docker compose -f docker/compose.yml -f docker/compose-local-dev.overrides.yml build prover docker compose -f docker/compose.yml -f docker/compose-local-dev.overrides.yml --profile l1 --profile l2 up -d -start-whole-environment-with-finalized-tag-updater: - docker compose -f docker/compose.yml -f docker/compose-local-dev.overrides.yml -f docker/compose-local-dev-finalized-tag-updater.overrides.yml --profile l1 --profile l2 up -d - start-whole-environment-traces-v2: docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 up -d -start-whole-environment-traces-v2-with-finalized-tag-updater: - docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml -f docker/compose-local-dev-finalized-tag-updater.overrides.yml --profile l1 --profile l2 up -d - pull-all-images: docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull @@ -122,10 +116,6 @@ fresh-start-all-traces-v2: make clean-environment make start-all-traces-v2 -fresh-start-all-traces-v2-with-finalized-tag-updater: - make clean-environment - make start-all-traces-v2-with-finalized-tag-updater - start-all-smc-v4: L1_GENESIS_TIME=$(get_future_time) make start-whole-environment make deploy-contracts-v4 @@ -134,18 +124,10 @@ start-all: L1_GENESIS_TIME=$(get_future_time) make start-whole-environment make deploy-contracts -start-all-with-finalized-tag-updater: - L1_GENESIS_TIME=$(get_future_time) make start-whole-environment-with-finalized-tag-updater - make deploy-contracts - start-all-traces-v2: L1_GENESIS_TIME=$(get_future_time) make start-whole-environment-traces-v2 make deploy-contracts -start-all-traces-v2-with-finalized-tag-updater: - L1_GENESIS_TIME=$(get_future_time) make start-whole-environment-traces-v2-with-finalized-tag-updater - make deploy-contracts - deploy-contracts-v4: make compile-contracts $(MAKE) -j2 deploy-linea-rollup-v4 deploy-l2messageservice diff --git a/build.gradle b/build.gradle index 57064789..18ec1cda 100644 --- a/build.gradle +++ b/build.gradle @@ -147,6 +147,18 @@ dockerCompose { projectName = "docker" environment.put("L1_GENESIS_TIME", "${Instant.now().plusSeconds(3).getEpochSecond()}") } + + localStackPostgresDbOnly { + startedServices = [ + "postgres" + ] + useComposeFiles = ["${project.rootDir.path}/docker/compose.yml"] + waitForHealthyStateTimeout = Duration.ofMinutes(3) + waitForTcpPorts = false + removeOrphans = true + noRecreate = true + projectName = "docker" + } } static Boolean hasKotlinPlugin(Project proj) { diff --git a/config/coordinator/coordinator-docker-traces-v2-override.config.toml b/config/coordinator/coordinator-docker-traces-v2-override.config.toml index a2f15d52..8cd51ef8 100644 --- a/config/coordinator/coordinator-docker-traces-v2-override.config.toml +++ b/config/coordinator/coordinator-docker-traces-v2-override.config.toml @@ -14,6 +14,7 @@ eth-api="http://traces-node-v2:8545" [traces] switch-to-linea-besu=true +blob-compressor-version="V1_0_1" expected-traces-api-version-v2="v0.8.0-rc3" [traces.counters-v2] endpoints=["http://traces-node-v2:8545/"] @@ -26,7 +27,7 @@ request-limit-per-endpoint=2 request-retry.backoff-delay="PT1S" request-retry.failures-warning-threshold=2 -[dynamic-gas-price-service] +[l2-network-gas-pricing.json-rpc-pricing-propagation] geth-gas-price-update-recipients=[ "http://l2-node:8545/" ] diff --git a/config/traces-api/traces-app-docker.config.toml b/config/traces-api/traces-app-docker.config.toml index cbb6dd14..4c20bd30 100644 --- a/config/traces-api/traces-app-docker.config.toml +++ b/config/traces-api/traces-app-docker.config.toml @@ -3,12 +3,6 @@ output_traces_directory = "/data/traces/v1/conflated" traces_api_version = "0.2.0" traces_file_extension = "json.gz" -# Inmemory cache of the traces read from the file system -# Save IO and CPU unzipping and JSON parsing -[read_traces_cache] -size = 20 -expiration_duration = "PT20M" - [api] port = 8080 path = "/" diff --git a/config/transaction-exclusion-api/log4j2-dev.xml b/config/transaction-exclusion-api/log4j2-dev.xml new file mode 100644 index 00000000..2408e82f --- /dev/null +++ b/config/transaction-exclusion-api/log4j2-dev.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/config/transaction-exclusion-api/transaction-exclusion-app-docker.config.toml b/config/transaction-exclusion-api/transaction-exclusion-app-docker.config.toml new file mode 100644 index 00000000..14829140 --- /dev/null +++ b/config/transaction-exclusion-api/transaction-exclusion-app-docker.config.toml @@ -0,0 +1,30 @@ +data-queryable-window-since-rejected-timestamp="P7D" + +[database] +schema="linea_transaction_exclusion" +read_pool_size=10 +read_pipelining_limit=10 +transactional_pool_size=10 +[database.read] +host="postgres" +port="5432" +username="postgres" +password="postgres" +[database.write] +host="postgres" +port="5432" +username="postgres" +password="postgres" +[database.cleanup] +polling-interval="PT60S" +storage-period="P7D" +[database.persistence-retry] +#max-retries = 10 commented as can be null +backoff-delay = "PT5S" +timeout = "PT20S" + +[api] +port=8080 +path="/" +number-of-verticles=1 #if 0 will create one verticle per core (or hyperthread if supported) +observability-port=8090 diff --git a/config/transaction-exclusion-api/transaction-exclusion-app-local-dev.config.overrides.toml b/config/transaction-exclusion-api/transaction-exclusion-app-local-dev.config.overrides.toml new file mode 100644 index 00000000..8ee011ad --- /dev/null +++ b/config/transaction-exclusion-api/transaction-exclusion-app-local-dev.config.overrides.toml @@ -0,0 +1,9 @@ +[database] +[database.read] +host="localhost" +[database.write] +host="localhost" + +[api] +port=8082 +number-of-verticles=1 diff --git a/config/transaction-exclusion-api/vertx-options.json b/config/transaction-exclusion-api/vertx-options.json new file mode 100644 index 00000000..6fa83073 --- /dev/null +++ b/config/transaction-exclusion-api/vertx-options.json @@ -0,0 +1,16 @@ +{ + "preferNativeTransport": true, + "logStacktraceThreshold": 500, + "maxEventLoopExecuteTime": 2, + "maxEventLoopExecuteTimeUnit": "MINUTES", + "warnEventLoopBlocked": 5000, + "maxWorkerExecuteTime": 2, + "maxWorkerExecuteTimeUnit": "MINUTES", + "metricsOptions": { + "enabled": true, + "prometheusOptions": { + "enabled": true, + "publishQuantiles": true + } + } +} diff --git a/coordinator/app/build.gradle b/coordinator/app/build.gradle index a01d78ae..99b61ae2 100644 --- a/coordinator/app/build.gradle +++ b/coordinator/app/build.gradle @@ -14,6 +14,7 @@ dependencies { implementation project(':jvm-libs:generic:http-rest') implementation project(':jvm-libs:generic:vertx-helper') implementation project(':jvm-libs:generic:extensions:futures') + implementation project(':jvm-libs:generic:persistence:db') implementation project(':jvm-libs:linea:web3j-extensions') implementation project(':jvm-libs:linea:core:metrics') implementation project(':jvm-libs:linea:metrics:micrometer') @@ -31,11 +32,11 @@ dependencies { implementation project(':coordinator:ethereum:blob-submitter') implementation project(':coordinator:ethereum:message-anchoring') implementation project(':coordinator:clients:web3signer-client') - implementation project(':coordinator:persistence:db') implementation project(':coordinator:persistence:blob') implementation project(':coordinator:persistence:aggregation') implementation project(':coordinator:persistence:batch') implementation project(':coordinator:persistence:feehistory') + implementation project(':coordinator:persistence:db-common') implementation project(":jvm-libs:linea:teku-execution-client") implementation "tech.pegasys.teku.internal:bytes:${libs.versions.teku.get()}" diff --git a/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/app/config/CoordinatorConfigTest.kt b/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/app/config/CoordinatorConfigTest.kt index b6d09b72..60ecf2e2 100644 --- a/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/app/config/CoordinatorConfigTest.kt +++ b/coordinator/app/src/test/kotlin/net/consensys/zkevm/coordinator/app/config/CoordinatorConfigTest.kt @@ -833,6 +833,7 @@ class CoordinatorConfigTest { ), traces = tracesConfig.copy( switchToLineaBesu = true, + blobCompressorVersion = BlobCompressorVersion.V1_0_1, expectedTracesApiVersionV2 = "v0.8.0-rc3", conflationV2 = tracesConfig.conflation.copy( endpoints = listOf(URI("http://traces-node-v2:8545/").toURL()) diff --git a/coordinator/clients/prover-client/file-based-client/build.gradle b/coordinator/clients/prover-client/file-based-client/build.gradle index 369148c8..d16051ab 100644 --- a/coordinator/clients/prover-client/file-based-client/build.gradle +++ b/coordinator/clients/prover-client/file-based-client/build.gradle @@ -1,6 +1,5 @@ plugins { id 'net.consensys.zkevm.kotlin-library-conventions' - id 'java-test-fixtures' } dependencies { diff --git a/coordinator/clients/type2-state-manager-client/src/test/kotlin/net/consensys/zkevm/ethereum/type2statemanagerjsonrpcclient/Type2StateManagerJsonRpcClientTest.kt b/coordinator/clients/type2-state-manager-client/src/test/kotlin/net/consensys/zkevm/coordinator/clients/Type2StateManagerJsonRpcClientTest.kt similarity index 96% rename from coordinator/clients/type2-state-manager-client/src/test/kotlin/net/consensys/zkevm/ethereum/type2statemanagerjsonrpcclient/Type2StateManagerJsonRpcClientTest.kt rename to coordinator/clients/type2-state-manager-client/src/test/kotlin/net/consensys/zkevm/coordinator/clients/Type2StateManagerJsonRpcClientTest.kt index 924c7005..a907b11c 100644 --- a/coordinator/clients/type2-state-manager-client/src/test/kotlin/net/consensys/zkevm/ethereum/type2statemanagerjsonrpcclient/Type2StateManagerJsonRpcClientTest.kt +++ b/coordinator/clients/type2-state-manager-client/src/test/kotlin/net/consensys/zkevm/coordinator/clients/Type2StateManagerJsonRpcClientTest.kt @@ -1,4 +1,4 @@ -package net.consensys.zkevm.ethereum.type2statemanagerjsonrpcclient +package net.consensys.zkevm.coordinator.clients import com.fasterxml.jackson.databind.node.ArrayNode import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper @@ -17,9 +17,6 @@ import net.consensys.linea.async.get import net.consensys.linea.errors.ErrorResponse import net.consensys.linea.jsonrpc.client.RequestRetryConfig import net.consensys.linea.jsonrpc.client.VertxHttpJsonRpcClientFactory -import net.consensys.zkevm.coordinator.clients.GetZkEVMStateMerkleProofResponse -import net.consensys.zkevm.coordinator.clients.Type2StateManagerErrorType -import net.consensys.zkevm.coordinator.clients.Type2StateManagerJsonRpcClient import org.apache.tuweni.bytes.Bytes32 import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatIllegalArgumentException diff --git a/coordinator/ethereum/blob-submitter/build.gradle b/coordinator/ethereum/blob-submitter/build.gradle index d9c07ef0..06c4a733 100644 --- a/coordinator/ethereum/blob-submitter/build.gradle +++ b/coordinator/ethereum/blob-submitter/build.gradle @@ -12,7 +12,6 @@ dependencies { implementation(project(':coordinator:ethereum:gas-pricing')) implementation(project(':coordinator:ethereum:test-utils')) implementation(project(":coordinator:persistence:blob")) - implementation(project(":coordinator:persistence:db")) implementation("com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}") implementation("com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}") implementation("com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}") @@ -25,10 +24,11 @@ dependencies { testImplementation(project(":jvm-libs:linea:testing:l1-blob-and-proof-submission")) testImplementation(project(":coordinator:persistence:aggregation")) - testImplementation(testFixtures(project(":coordinator:persistence:db"))) + testImplementation(project(":coordinator:persistence:db-common")) testImplementation(testFixtures(project(":coordinator:ethereum:gas-pricing"))) testImplementation(testFixtures(project(":coordinator:core"))) testImplementation(testFixtures(project(":jvm-libs:generic:extensions:kotlin"))) + testImplementation(testFixtures(project(":jvm-libs:generic:persistence:db"))) testImplementation("io.vertx:vertx-junit5") testImplementation("com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}") testImplementation("com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}") diff --git a/coordinator/ethereum/blob-submitter/src/integrationTest/kotlin/net/consensys/zkevm/ethereum/finalization/BlobAndAggregationFinalizationIntTest.kt b/coordinator/ethereum/blob-submitter/src/integrationTest/kotlin/net/consensys/zkevm/ethereum/finalization/BlobAndAggregationFinalizationIntTest.kt index 6790f43b..305e06cb 100644 --- a/coordinator/ethereum/blob-submitter/src/integrationTest/kotlin/net/consensys/zkevm/ethereum/finalization/BlobAndAggregationFinalizationIntTest.kt +++ b/coordinator/ethereum/blob-submitter/src/integrationTest/kotlin/net/consensys/zkevm/ethereum/finalization/BlobAndAggregationFinalizationIntTest.kt @@ -23,7 +23,7 @@ import net.consensys.zkevm.persistence.dao.aggregation.PostgresAggregationsDao import net.consensys.zkevm.persistence.dao.blob.BlobsPostgresDao import net.consensys.zkevm.persistence.dao.blob.BlobsRepositoryImpl import net.consensys.zkevm.persistence.db.DbHelper -import net.consensys.zkevm.persistence.test.CleanDbTestSuiteParallel +import net.consensys.zkevm.persistence.db.test.CleanDbTestSuiteParallel import org.assertj.core.api.Assertions.assertThat import org.awaitility.Awaitility.waitAtMost import org.junit.jupiter.api.Test @@ -36,6 +36,10 @@ import kotlin.time.toJavaDuration @ExtendWith(VertxExtension::class) class BlobAndAggregationFinalizationIntTest : CleanDbTestSuiteParallel() { + init { + target = "4" + } + override val databaseName = DbHelper.generateUniqueDbName("coordinator-tests-submission-int-test") private val fakeClock = FakeFixedClock() private lateinit var lineaRollupContractForAggregationSubmission: LineaRollupSmartContractClient diff --git a/coordinator/persistence/aggregation/build.gradle b/coordinator/persistence/aggregation/build.gradle index 4724313b..bbe37e23 100644 --- a/coordinator/persistence/aggregation/build.gradle +++ b/coordinator/persistence/aggregation/build.gradle @@ -4,14 +4,16 @@ plugins { dependencies { api(project(":coordinator:core")) - api project(':coordinator:clients:prover-client:serialization') - implementation(project(":coordinator:persistence:db")) + api(project(":coordinator:clients:prover-client:serialization")) + implementation(project(":jvm-libs:generic:persistence:db")) implementation(project(":coordinator:persistence:batch")) testImplementation(project(":coordinator:persistence:batch")) testImplementation(project(":coordinator:persistence:blob")) - testImplementation(testFixtures(project(":coordinator:persistence:db"))) + testImplementation(project(":coordinator:persistence:db-common")) testImplementation(testFixtures(project(":coordinator:core"))) + testImplementation(testFixtures(project(":coordinator:persistence:db-common"))) + testImplementation(testFixtures(project(":jvm-libs:generic:persistence:db"))) testImplementation(testFixtures(project(":jvm-libs:generic:extensions:kotlin"))) testImplementation("io.vertx:vertx-junit5") } diff --git a/coordinator/persistence/aggregation/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/aggregation/AggregationsPostgresDaoTest.kt b/coordinator/persistence/aggregation/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/aggregation/AggregationsPostgresDaoTest.kt index 230d3db8..e0e43929 100644 --- a/coordinator/persistence/aggregation/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/aggregation/AggregationsPostgresDaoTest.kt +++ b/coordinator/persistence/aggregation/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/aggregation/AggregationsPostgresDaoTest.kt @@ -25,8 +25,8 @@ import net.consensys.zkevm.persistence.dao.batch.persistence.BatchesPostgresDao import net.consensys.zkevm.persistence.dao.blob.BlobsPostgresDao import net.consensys.zkevm.persistence.db.DbHelper import net.consensys.zkevm.persistence.db.DuplicatedRecordException -import net.consensys.zkevm.persistence.test.CleanDbTestSuiteParallel -import net.consensys.zkevm.persistence.test.DbQueries +import net.consensys.zkevm.persistence.db.test.CleanDbTestSuiteParallel +import net.consensys.zkevm.persistence.db.test.DbQueries import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach @@ -39,6 +39,10 @@ import java.util.concurrent.TimeUnit @ExtendWith(VertxExtension::class) class AggregationsPostgresDaoTest : CleanDbTestSuiteParallel() { + init { + target = "4" + } + override val databaseName = DbHelper.generateUniqueDbName("coordinator-tests-aggregations-dao") private val maxBlobReturnLimit = 10u diff --git a/coordinator/persistence/aggregation/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/aggregation/RecordsCleanupFinalizationHandlerTest.kt b/coordinator/persistence/aggregation/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/aggregation/RecordsCleanupFinalizationHandlerTest.kt index a5e29576..db7b558b 100644 --- a/coordinator/persistence/aggregation/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/aggregation/RecordsCleanupFinalizationHandlerTest.kt +++ b/coordinator/persistence/aggregation/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/aggregation/RecordsCleanupFinalizationHandlerTest.kt @@ -21,7 +21,7 @@ import net.consensys.zkevm.persistence.dao.batch.persistence.PostgresBatchesRepo import net.consensys.zkevm.persistence.dao.blob.BlobsPostgresDao import net.consensys.zkevm.persistence.dao.blob.BlobsRepositoryImpl import net.consensys.zkevm.persistence.db.DbHelper -import net.consensys.zkevm.persistence.test.CleanDbTestSuiteParallel +import net.consensys.zkevm.persistence.db.test.CleanDbTestSuiteParallel import org.apache.tuweni.bytes.Bytes32 import org.assertj.core.api.Assertions import org.junit.jupiter.api.BeforeEach @@ -31,6 +31,9 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture @ExtendWith(VertxExtension::class) class RecordsCleanupFinalizationHandlerTest : CleanDbTestSuiteParallel() { + init { + target = "4" + } override val databaseName = DbHelper.generateUniqueDbName("records-cleanup-on-finalization") private var fakeClock = FakeFixedClock(Clock.System.now()) diff --git a/coordinator/persistence/batch/build.gradle b/coordinator/persistence/batch/build.gradle index 6a21860d..041ca226 100644 --- a/coordinator/persistence/batch/build.gradle +++ b/coordinator/persistence/batch/build.gradle @@ -1,23 +1,22 @@ plugins { id "net.consensys.zkevm.kotlin-library-conventions" id "net.consensys.zkevm.linea-contracts-helper" - id 'java-test-fixtures' } dependencies { api(project(":coordinator:core")) - implementation(project(":coordinator:persistence:db")) + implementation(project(":jvm-libs:generic:persistence:db")) implementation(project(":jvm-libs:linea:core:metrics")) implementation(project(":coordinator:clients:prover-client:file-based-client")) { because "ProverResponseIndex is a part of it" } + testImplementation(project(":coordinator:persistence:db-common")) testImplementation(testFixtures(project(":coordinator:core"))) - testImplementation(testFixtures(project(":coordinator:persistence:db"))) + testImplementation(testFixtures(project(":coordinator:persistence:db-common"))) + testImplementation(testFixtures(project(":jvm-libs:generic:persistence:db"))) testImplementation(testFixtures(project(":jvm-libs:generic:extensions:kotlin"))) testImplementation("io.vertx:vertx-junit5") - - testFixturesImplementation(testFixtures(project(":coordinator:persistence:db"))) } sourceSets { diff --git a/coordinator/persistence/batch/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/batch/persistence/BatchesPostgresDaoTest.kt b/coordinator/persistence/batch/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/batch/persistence/BatchesPostgresDaoTest.kt index 17f4b7c6..a71226b3 100644 --- a/coordinator/persistence/batch/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/batch/persistence/BatchesPostgresDaoTest.kt +++ b/coordinator/persistence/batch/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/batch/persistence/BatchesPostgresDaoTest.kt @@ -12,8 +12,8 @@ import net.consensys.zkevm.domain.createBatch import net.consensys.zkevm.persistence.dao.batch.persistence.BatchesDao.Companion.batchesDaoTableName import net.consensys.zkevm.persistence.db.DbHelper import net.consensys.zkevm.persistence.db.DuplicatedRecordException -import net.consensys.zkevm.persistence.test.CleanDbTestSuiteParallel -import net.consensys.zkevm.persistence.test.DbQueries +import net.consensys.zkevm.persistence.db.test.CleanDbTestSuiteParallel +import net.consensys.zkevm.persistence.db.test.DbQueries import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach @@ -26,6 +26,10 @@ import kotlin.time.Duration.Companion.seconds @ExtendWith(VertxExtension::class) class BatchesPostgresDaoTest : CleanDbTestSuiteParallel() { + init { + target = "4" + } + override val databaseName = DbHelper.generateUniqueDbName("coordinator-tests-batches") private var fakeClockTime = Instant.parse("2023-12-11T00:00:00.000Z") private var fakeClock = FakeFixedClock(fakeClockTime) diff --git a/coordinator/persistence/batch/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/batch/persistence/PostgresBatchesRepositoryTest.kt b/coordinator/persistence/batch/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/batch/persistence/PostgresBatchesRepositoryTest.kt index 7a3b389b..ea843e16 100644 --- a/coordinator/persistence/batch/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/batch/persistence/PostgresBatchesRepositoryTest.kt +++ b/coordinator/persistence/batch/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/batch/persistence/PostgresBatchesRepositoryTest.kt @@ -10,8 +10,8 @@ import net.consensys.zkevm.domain.Batch import net.consensys.zkevm.domain.createBatch import net.consensys.zkevm.persistence.db.DbHelper import net.consensys.zkevm.persistence.db.DuplicatedRecordException -import net.consensys.zkevm.persistence.test.CleanDbTestSuiteParallel -import net.consensys.zkevm.persistence.test.DbQueries +import net.consensys.zkevm.persistence.db.test.CleanDbTestSuiteParallel +import net.consensys.zkevm.persistence.db.test.DbQueries import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -23,6 +23,10 @@ import kotlin.time.Duration.Companion.seconds @ExtendWith(VertxExtension::class) class PostgresBatchesRepositoryTest : CleanDbTestSuiteParallel() { + init { + target = "4" + } + private var fakeClockTime = Instant.parse("2023-12-11T00:00:00.000Z") override val databaseName = DbHelper.generateUniqueDbName("coordinator-tests-batches-repository") diff --git a/coordinator/persistence/blob/build.gradle b/coordinator/persistence/blob/build.gradle index 1e925547..d199412e 100644 --- a/coordinator/persistence/blob/build.gradle +++ b/coordinator/persistence/blob/build.gradle @@ -3,29 +3,22 @@ import org.gradle.api.tasks.testing.logging.TestLogEvent plugins { id "net.consensys.zkevm.kotlin-library-conventions" - id 'java-test-fixtures' } dependencies { api(project(":coordinator:core")) - api project(':coordinator:clients:prover-client:serialization') - implementation(project(":coordinator:persistence:db")) + api(project(":coordinator:clients:prover-client:serialization")) + implementation(project(":jvm-libs:generic:persistence:db")) testImplementation("com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}") testImplementation("com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}") testImplementation("com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}") testImplementation("tech.pegasys.teku.internal:executionclient:${libs.versions.teku.get()}") - testImplementation(testFixtures(project(":coordinator:persistence:db"))) + testImplementation(project(":coordinator:persistence:db-common")) testImplementation(testFixtures(project(":coordinator:core"))) testImplementation(testFixtures(project(":jvm-libs:generic:extensions:kotlin"))) + testImplementation(testFixtures(project(":jvm-libs:generic:persistence:db"))) testImplementation("io.vertx:vertx-junit5") - - testFixturesImplementation('tech.pegasys.teku.internal:async:23.1.1') - testFixturesImplementation('org.apache.tuweni:tuweni-units:2.3.1') - testFixturesImplementation('com.michael-bull.kotlin-result:kotlin-result:1.1.16') - testFixturesImplementation('org.mockito.kotlin:mockito-kotlin:5.1.0') - testFixturesApi(platform("org.junit:junit-bom:${libs.versions.junit.get()}")) - testFixturesApi("io.vertx:vertx-junit5:${libs.versions.vertx.get()}") } sourceSets { diff --git a/coordinator/persistence/blob/src/integrationTest/kotlin/net/consensys/zkevm/ethereum/coordination/blob/BlobCompressionProofCoordinatorIntTest.kt b/coordinator/persistence/blob/src/integrationTest/kotlin/net/consensys/zkevm/ethereum/coordination/blob/BlobCompressionProofCoordinatorIntTest.kt index a78fd980..390c04d1 100644 --- a/coordinator/persistence/blob/src/integrationTest/kotlin/net/consensys/zkevm/ethereum/coordination/blob/BlobCompressionProofCoordinatorIntTest.kt +++ b/coordinator/persistence/blob/src/integrationTest/kotlin/net/consensys/zkevm/ethereum/coordination/blob/BlobCompressionProofCoordinatorIntTest.kt @@ -23,7 +23,7 @@ import net.consensys.zkevm.persistence.BlobsRepository import net.consensys.zkevm.persistence.dao.blob.BlobsPostgresDao import net.consensys.zkevm.persistence.dao.blob.BlobsRepositoryImpl import net.consensys.zkevm.persistence.db.DbHelper -import net.consensys.zkevm.persistence.test.CleanDbTestSuiteParallel +import net.consensys.zkevm.persistence.db.test.CleanDbTestSuiteParallel import org.apache.tuweni.bytes.Bytes32 import org.assertj.core.api.Assertions.assertThat import org.awaitility.Awaitility.waitAtMost @@ -47,6 +47,10 @@ import kotlin.time.toJavaDuration @ExtendWith(VertxExtension::class) class BlobCompressionProofCoordinatorIntTest : CleanDbTestSuiteParallel() { + init { + target = "4" + } + override val databaseName = DbHelper.generateUniqueDbName( "blob-compression-proof-coordinator" ) diff --git a/coordinator/persistence/blob/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/blob/BlobsPostgresDaoTest.kt b/coordinator/persistence/blob/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/blob/BlobsPostgresDaoTest.kt index 008da7cf..b65fe696 100644 --- a/coordinator/persistence/blob/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/blob/BlobsPostgresDaoTest.kt +++ b/coordinator/persistence/blob/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/blob/BlobsPostgresDaoTest.kt @@ -18,7 +18,7 @@ import net.consensys.zkevm.domain.BlockIntervals import net.consensys.zkevm.domain.createBlobRecord import net.consensys.zkevm.persistence.db.DbHelper import net.consensys.zkevm.persistence.db.DuplicatedRecordException -import net.consensys.zkevm.persistence.test.CleanDbTestSuiteParallel +import net.consensys.zkevm.persistence.db.test.CleanDbTestSuiteParallel import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -32,6 +32,10 @@ import kotlin.time.toJavaDuration @ExtendWith(VertxExtension::class) class BlobsPostgresDaoTest : CleanDbTestSuiteParallel() { + init { + target = "4" + } + override val databaseName = DbHelper.generateUniqueDbName("coordinator-tests-blobs-dao") private val maxBlobsToReturn = 6u private fun blobsContentQuery(): PreparedQuery> = diff --git a/coordinator/persistence/db-common/build.gradle b/coordinator/persistence/db-common/build.gradle new file mode 100644 index 00000000..7ecf65d8 --- /dev/null +++ b/coordinator/persistence/db-common/build.gradle @@ -0,0 +1,38 @@ +plugins { + id "net.consensys.zkevm.kotlin-library-conventions" + id 'java-test-fixtures' +} + +dependencies { + testImplementation(project(":jvm-libs:generic:extensions:futures")) + testImplementation(project(":jvm-libs:generic:extensions:kotlin")) + testImplementation(project(":jvm-libs:generic:persistence:db")) + testImplementation(testFixtures(project(":jvm-libs:generic:extensions:kotlin"))) + testImplementation(testFixtures(project(":jvm-libs:generic:persistence:db"))) + + testFixturesImplementation(project(":coordinator:core")) + testFixturesImplementation("io.vertx:vertx-pg-client:${libs.versions.vertx.get()}") +} + +sourceSets { + integrationTest { + kotlin { + compileClasspath += main.output + runtimeClasspath += main.output + } + compileClasspath += sourceSets.main.output + sourceSets.main.compileClasspath + sourceSets.test.compileClasspath + runtimeClasspath += sourceSets.main.output + sourceSets.main.runtimeClasspath + sourceSets.test.runtimeClasspath + } +} + +task integrationTest(type: Test) { + test -> + description = "Runs integration tests." + group = "verification" + useJUnitPlatform() + + classpath = sourceSets.integrationTest.runtimeClasspath + testClassesDirs = sourceSets.integrationTest.output.classesDirs + + dependsOn(":localStackComposeUp") +} diff --git a/coordinator/persistence/db/src/integrationTest/kotlin/net/consensys/zkevm/persistence/db/DbSchemaUpdatesIntTest.kt b/coordinator/persistence/db-common/src/integrationTest/kotlin/net/consensys/zkevm/persistence/db/test/DbSchemaUpdatesIntTest.kt similarity index 92% rename from coordinator/persistence/db/src/integrationTest/kotlin/net/consensys/zkevm/persistence/db/DbSchemaUpdatesIntTest.kt rename to coordinator/persistence/db-common/src/integrationTest/kotlin/net/consensys/zkevm/persistence/db/test/DbSchemaUpdatesIntTest.kt index b2b801b2..a6ffc662 100644 --- a/coordinator/persistence/db/src/integrationTest/kotlin/net/consensys/zkevm/persistence/db/DbSchemaUpdatesIntTest.kt +++ b/coordinator/persistence/db-common/src/integrationTest/kotlin/net/consensys/zkevm/persistence/db/test/DbSchemaUpdatesIntTest.kt @@ -1,4 +1,4 @@ -package net.consensys.zkevm.persistence.db +package net.consensys.zkevm.persistence.db.test import io.vertx.core.Vertx import io.vertx.junit5.VertxExtension @@ -7,7 +7,8 @@ import io.vertx.sqlclient.SqlClient import kotlinx.datetime.Clock import net.consensys.encodeHex import net.consensys.linea.async.get -import net.consensys.zkevm.persistence.test.DbQueries +import net.consensys.zkevm.persistence.db.Db +import net.consensys.zkevm.persistence.db.DbHelper import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach @@ -26,8 +27,8 @@ class DbSchemaUpdatesIntTest { private val databaseName = DbHelper.generateUniqueDbName("coordinator-db-migration-tests") private val username = "postgres" private val password = "postgres" - private lateinit var dataSource: DataSource + private lateinit var dataSource: DataSource private lateinit var pool: Pool private lateinit var sqlClient: SqlClient @@ -73,7 +74,10 @@ class DbSchemaUpdatesIntTest { val schemaTarget = "1" DbHelper.dropAllTables(dataSource) - Db.applyDbMigrations(dataSource, schemaTarget) + Db.applyDbMigrations( + dataSource = dataSource, + target = schemaTarget + ) val paramsV1 = listOf( Clock.System.now().toEpochMilliseconds(), @@ -106,7 +110,10 @@ class DbSchemaUpdatesIntTest { val schemaTarget = "2" DbHelper.dropAllTables(dataSource) - Db.applyDbMigrations(dataSource, schemaTarget) + Db.applyDbMigrations( + dataSource = dataSource, + target = schemaTarget + ) val batchParamsV2 = listOf( Clock.System.now().toEpochMilliseconds(), @@ -133,7 +140,7 @@ class DbSchemaUpdatesIntTest { Clock.System.now().toEpochMilliseconds(), 3, ByteArray(32).encodeHex(), - "" + "{}" ) DbQueries.insertBlob(sqlClient, DbQueries.insertBlobQuery, blobParams).get() diff --git a/coordinator/persistence/db/src/main/resources/db/V001__initial_schema.sql b/coordinator/persistence/db-common/src/main/resources/db/V001__initial_schema.sql similarity index 100% rename from coordinator/persistence/db/src/main/resources/db/V001__initial_schema.sql rename to coordinator/persistence/db-common/src/main/resources/db/V001__initial_schema.sql diff --git a/coordinator/persistence/db/src/main/resources/db/V002__add_blob_and_aggregation_schema.sql b/coordinator/persistence/db-common/src/main/resources/db/V002__add_blob_and_aggregation_schema.sql similarity index 100% rename from coordinator/persistence/db/src/main/resources/db/V002__add_blob_and_aggregation_schema.sql rename to coordinator/persistence/db-common/src/main/resources/db/V002__add_blob_and_aggregation_schema.sql diff --git a/coordinator/persistence/db/src/main/resources/db/V003__add_feehistory_schema.sql b/coordinator/persistence/db-common/src/main/resources/db/V003__add_feehistory_schema.sql similarity index 100% rename from coordinator/persistence/db/src/main/resources/db/V003__add_feehistory_schema.sql rename to coordinator/persistence/db-common/src/main/resources/db/V003__add_feehistory_schema.sql diff --git a/coordinator/persistence/db/src/main/resources/db/V004__remove_versioning_schema.sql b/coordinator/persistence/db-common/src/main/resources/db/V004__remove_versioning_schema.sql similarity index 100% rename from coordinator/persistence/db/src/main/resources/db/V004__remove_versioning_schema.sql rename to coordinator/persistence/db-common/src/main/resources/db/V004__remove_versioning_schema.sql diff --git a/coordinator/persistence/db/src/testFixtures/kotlin/net/consensys/zkevm/persistence/test/DbQueries.kt b/coordinator/persistence/db-common/src/testFixtures/kotlin/net/consensys/zkevm/persistence/db/test/DbQueries.kt similarity index 95% rename from coordinator/persistence/db/src/testFixtures/kotlin/net/consensys/zkevm/persistence/test/DbQueries.kt rename to coordinator/persistence/db-common/src/testFixtures/kotlin/net/consensys/zkevm/persistence/db/test/DbQueries.kt index be943062..ff561f70 100644 --- a/coordinator/persistence/db/src/testFixtures/kotlin/net/consensys/zkevm/persistence/test/DbQueries.kt +++ b/coordinator/persistence/db-common/src/testFixtures/kotlin/net/consensys/zkevm/persistence/db/test/DbQueries.kt @@ -1,4 +1,4 @@ -package net.consensys.zkevm.persistence.test +package net.consensys.zkevm.persistence.db.test import io.vertx.core.Future import io.vertx.sqlclient.PreparedQuery @@ -46,7 +46,7 @@ object DbQueries { (created_epoch_milli, start_block_number, end_block_number, conflation_calculator_version, blob_hash, status, start_block_timestamp, end_block_timestamp, batches_count, expected_shnarf, blob_compression_proof) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, CAST($11::text as jsonb)) """.trimIndent() val insertBatchQueryV1 = diff --git a/coordinator/persistence/feehistory/build.gradle b/coordinator/persistence/feehistory/build.gradle index 3af97d53..f4f7adbe 100644 --- a/coordinator/persistence/feehistory/build.gradle +++ b/coordinator/persistence/feehistory/build.gradle @@ -6,15 +6,18 @@ plugins { } dependencies { - api project(':coordinator:clients:prover-client:serialization') - implementation(project(":coordinator:persistence:db")) implementation(project(":coordinator:ethereum:gas-pricing:dynamic-cap")) + implementation(project(":jvm-libs:generic:extensions:futures")) + implementation(project(":jvm-libs:generic:extensions:kotlin")) + implementation(project(":jvm-libs:generic:persistence:db")) + implementation(project(":jvm-libs:linea:core:domain-models")) + testImplementation(project(":coordinator:persistence:db-common")) testImplementation("com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}") testImplementation("com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}") testImplementation("com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}") testImplementation("tech.pegasys.teku.internal:executionclient:${libs.versions.teku.get()}") - testImplementation(testFixtures(project(":coordinator:persistence:db"))) + testImplementation(testFixtures(project(":jvm-libs:generic:persistence:db"))) testImplementation(testFixtures(project(":jvm-libs:generic:extensions:kotlin"))) testImplementation("io.vertx:vertx-junit5") } diff --git a/coordinator/persistence/feehistory/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/feehistory/FeeHistoriesPostgresDaoTest.kt b/coordinator/persistence/feehistory/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/feehistory/FeeHistoriesPostgresDaoTest.kt index 642fe766..57b3b113 100644 --- a/coordinator/persistence/feehistory/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/feehistory/FeeHistoriesPostgresDaoTest.kt +++ b/coordinator/persistence/feehistory/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/feehistory/FeeHistoriesPostgresDaoTest.kt @@ -9,7 +9,7 @@ import net.consensys.FakeFixedClock import net.consensys.linea.FeeHistory import net.consensys.linea.async.get import net.consensys.zkevm.persistence.db.DbHelper -import net.consensys.zkevm.persistence.test.CleanDbTestSuiteParallel +import net.consensys.zkevm.persistence.db.test.CleanDbTestSuiteParallel import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -17,6 +17,9 @@ import org.junit.jupiter.api.extension.ExtendWith @ExtendWith(VertxExtension::class) class FeeHistoriesPostgresDaoTest : CleanDbTestSuiteParallel() { + init { + target = "4" + } fun createFeeHistory( oldestBlockNumber: ULong, diff --git a/coordinator/persistence/feehistory/src/integrationTest/resources/log4j2.xml b/coordinator/persistence/feehistory/src/integrationTest/resources/log4j2.xml index eab23f77..2b3fb69d 100644 --- a/coordinator/persistence/feehistory/src/integrationTest/resources/log4j2.xml +++ b/coordinator/persistence/feehistory/src/integrationTest/resources/log4j2.xml @@ -6,7 +6,7 @@ - + diff --git a/docker/compose-local-dev-traces-v2.overrides.yml b/docker/compose-local-dev-traces-v2.overrides.yml index ad4a81e9..e4397f96 100644 --- a/docker/compose-local-dev-traces-v2.overrides.yml +++ b/docker/compose-local-dev-traces-v2.overrides.yml @@ -7,9 +7,15 @@ services: JAVA_OPTS: -XX:+UnlockExperimentalVMOptions -XX:-UseG1GC -XX:+UseZGC volumes: - ../config/common/traces-limits-besu-v2.toml:/var/lib/besu/traces-limits.toml:ro + + l2-node-besu: + environment: + JAVA_OPTS: -XX:+UnlockExperimentalVMOptions -XX:-UseG1GC -XX:+UseZGC + volumes: + - ../config/common/traces-limits-besu-v2.toml:/var/lib/besu/traces-limits.toml:ro linea-besu-sequencer-plugin-downloader: - command: [ "sh", "/file-downloader.sh", "https://github.com/Consensys/linea-sequencer/releases/download/v0.8.0-rc1.1/linea-sequencer-v0.8.0-rc1.1.jar", "/linea-besu-sequencer" ] + command: [ "sh", "/file-downloader.sh", "https://github.com/Consensys/linea-sequencer/releases/download/v0.8.0-rc4.1/linea-sequencer-v0.8.0-rc4.1.jar", "/linea-besu-sequencer" ] traces-node: command: ['echo', 'forced exit as replaced by traces-node-v2'] @@ -40,6 +46,7 @@ services: retries: 120 restart: "no" environment: + JAVA_OPTS: -XX:+UnlockExperimentalVMOptions -XX:-UseG1GC -XX:+UseZGC LOG4J_CONFIGURATION_FILE: /var/lib/besu/log4j.xml entrypoint: - /bin/bash @@ -51,7 +58,6 @@ services: --bootnodes=enode://14408801a444dafc44afbccce2eb755f902aed3b5743fed787b3c790e021fef28b8c827ed896aa4e8fb46e22bd67c39f994a73768b4b382f8597b0d44370e15d@11.11.11.101:30303 volumes: - ./config/traces-node-v2/traces-node-v2-config.toml:/var/lib/besu/traces-node-v2.config.toml:ro - - ./config/traces-node-v2/key:/var/lib/besu/key:ro - ./config/traces-node-v2/log4j.xml:/var/lib/besu/log4j.xml:ro - ./config/linea-local-dev-genesis-PoA-besu.json/:/var/lib/besu/genesis.json:ro - ../tmp/traces-node-v2/plugins:/opt/besu/plugins/ diff --git a/docker/compose-local-dev.overrides.yml b/docker/compose-local-dev.overrides.yml index 2bdf4521..3aff1b4a 100644 --- a/docker/compose-local-dev.overrides.yml +++ b/docker/compose-local-dev.overrides.yml @@ -48,3 +48,7 @@ services: l1-el-node: environment: JAVA_OPTS: -XX:+UnlockExperimentalVMOptions -XX:-UseG1GC -XX:+UseZGC + + l2-node-besu: + environment: + JAVA_OPTS: -XX:+UnlockExperimentalVMOptions -XX:-UseG1GC -XX:+UseZGC diff --git a/docker/compose.yml b/docker/compose.yml index 4f4bcf2f..fa8bccec 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -54,7 +54,9 @@ services: --node-private-key-file="/var/lib/besu/key" \ --plugin-linea-l1-polling-interval="PT12S" \ --plugin-linea-l1-smart-contract-address="0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" \ - --plugin-linea-l1-rpc-endpoint="http://l1-el-node:8545" + --plugin-linea-l1-rpc-endpoint="http://l1-el-node:8545" \ + --plugin-linea-rejected-tx-endpoint="http://transaction-exclusion-api:8080" \ + --plugin-linea-node-type="SEQUENCER" volumes: - ./config/linea-besu-sequencer/sequencer.config.toml:/var/lib/besu/sequencer.config.toml:ro - ./config/linea-besu-sequencer/deny-list.txt:/var/lib/besu/deny-list.txt:ro @@ -71,7 +73,7 @@ services: linea-besu-sequencer-plugin-downloader: image: busybox:1.36.1 # profiles: ["l2", "l2-bc"] this works locally but breakes on CI, maybe Docker compose version issue - command: [ "sh", "/file-downloader.sh", "https://github.com/Consensys/linea-sequencer/releases/download/v0.1.4-test34/besu-sequencer-plugins-v0.1.4-test34.jar", "/linea-besu-sequencer" ] + command: [ "sh", "/file-downloader.sh", "https://github.com/Consensys/linea-sequencer/releases/download/v0.1.4-test35/besu-sequencer-plugins-v0.1.4-test35.jar", "/linea-besu-sequencer" ] volumes: - ./scripts/file-downloader.sh:/file-downloader.sh:ro - ../tmp/linea-besu-sequencer/plugins:/linea-besu-sequencer/ @@ -116,6 +118,54 @@ services: networks: linea: ipv4_address: 11.11.11.209 + + l2-node-besu: + hostname: l2-node-besu + container_name: l2-node-besu + image: consensys/linea-besu:24.10-delivery34 + profiles: [ "l2", "l2-bc", "debug" ] + depends_on: + sequencer: + condition: service_healthy + ports: + - "9045:8545" + - "9046:8546" + - "9050:8550" + - "9051:8548" + - "30309:30303" + healthcheck: + test: [ "CMD-SHELL", "bash -c \"[ -f /tmp/pid ]\"" ] + interval: 1s + timeout: 1s + retries: 120 + restart: "no" + environment: + LOG4J_CONFIGURATION_FILE: /var/lib/besu/log4j.xml + entrypoint: + - /bin/bash + - -c + - | + /opt/besu/bin/besu \ + --config-file=/var/lib/besu/l2-node-besu.config.toml \ + --genesis-file=/var/lib/besu/genesis.json \ + --plugin-linea-l1-polling-interval="PT12S" \ + --plugin-linea-l1-smart-contract-address="0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" \ + --plugin-linea-l1-rpc-endpoint="http://l1-el-node:8545" \ + --plugin-linea-rejected-tx-endpoint="http://transaction-exclusion-api:8080" \ + --plugin-linea-node-type="RPC" \ + --bootnodes=enode://14408801a444dafc44afbccce2eb755f902aed3b5743fed787b3c790e021fef28b8c827ed896aa4e8fb46e22bd67c39f994a73768b4b382f8597b0d44370e15d@11.11.11.101:30303 + volumes: + - ./config/l2-node-besu/l2-node-besu-config.toml:/var/lib/besu/l2-node-besu.config.toml:ro + - ./config/linea-besu-sequencer/deny-list.txt:/var/lib/besu/deny-list.txt:ro + - ./config/l2-node-besu/log4j.xml:/var/lib/besu/log4j.xml:ro + - ./config/linea-local-dev-genesis-PoA-besu.json/:/var/lib/besu/genesis.json:ro + - ../config/common/traces-limits-besu-v1.toml:/var/lib/besu/traces-limits.toml:ro + - ../tmp/linea-besu-sequencer/plugins:/opt/besu/plugins/ + - ../tmp/local/:/data/:rw + networks: + l1-network: + linea: + ipv4_address: 11.11.11.119 traces-node: container_name: traces-node @@ -255,6 +305,27 @@ services: linea: ipv4_address: 11.11.11.105 + transaction-exclusion-api: + hostname: transaction-exclusion-api + container_name: transaction-exclusion-api + image: consensys/linea-transaction-exclusion-api:${TRANSACTION_EXCLUSION_API_TAG:-d227ddf} + profiles: [ "l2", "debug" ] + restart: on-failure + depends_on: + postgres: + condition: service_healthy + ports: + - "8082:8080" + command: [ 'java', '-Dvertx.configurationFile=config/vertx-options.json', '-Dlog4j2.configurationFile=config/log4j2-dev.xml', '-jar', 'libs/transaction-exclusion-api.jar', 'config/transaction-exclusion-app-docker.config.toml', ] + volumes: + - ../config/transaction-exclusion-api/transaction-exclusion-app-docker.config.toml:/opt/consensys/linea/transaction-exclusion-api/config/transaction-exclusion-app-docker.config.toml:ro + - ../config/transaction-exclusion-api/vertx-options.json:/opt/consensys/linea/transaction-exclusion-api/config/vertx-options.json:ro + - ../config/transaction-exclusion-api/log4j2-dev.xml:/opt/consensys/linea/transaction-exclusion-api/config/log4j2-dev.xml:ro + - local-dev:/data/ + networks: + linea: + ipv4_address: 11.11.11.110 + coordinator: hostname: coordinator container_name: coordinator @@ -330,7 +401,6 @@ services: POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} PGDATA: /data/postgres - POSTGRES_DB: ${POSTGRES_DB:-coordinator} command: postgres -c config_file=/etc/postgresql/postgresql.conf # uncomment command below if you need to log and debug queries to PG # command: @@ -500,7 +570,6 @@ services: volumes: - ./config/zkbesu-shomei/zkbesu-config.toml:/var/lib/besu/zkbesu-config.toml:ro - - ./config/zkbesu-shomei/key:/var/lib/besu/key:ro - ./config/zkbesu-shomei/log4j.xml:/var/lib/besu/log4j.xml:ro - ./config/linea-local-dev-genesis-PoA-besu.json/:/var/lib/besu/genesis.json:ro - ../tmp/zkbesu-shomei/plugins:/opt/besu/plugins/ diff --git a/docker/config/l2-node-besu/l2-node-besu-config.toml b/docker/config/l2-node-besu/l2-node-besu-config.toml new file mode 100644 index 00000000..d56e459f --- /dev/null +++ b/docker/config/l2-node-besu/l2-node-besu-config.toml @@ -0,0 +1,50 @@ +data-path="/opt/besu/data" +host-whitelist=["*"] +sync-mode="FULL" +p2p-port=30303 + +min-gas-price=0 + +# engine +engine-host-allowlist=["*"] +engine-rpc-port=8550 + +# rpc +rpc-http-enabled=true +rpc-http-host="0.0.0.0" +rpc-http-port=8545 +rpc-http-cors-origins=["*"] +rpc-http-api=["ADMIN","DEBUG","NET","ETH","WEB3","PLUGINS","LINEA"] +rpc-http-max-active-connections=200 + +# ws +rpc-ws-enabled=true +rpc-ws-host="0.0.0.0" +rpc-ws-port=8546 +rpc-ws-api=["ADMIN","TXPOOL","WEB3","ETH","NET","PERM"] +rpc-ws-max-active-connections=200 + +# graphql +graphql-http-enabled=false + +# metrics +metrics-enabled=true +metrics-host="0.0.0.0" +metrics-port=9545 + +# database +data-storage-format="BONSAI" + +# plugins +plugin-linea-module-limit-file-path="/var/lib/besu/traces-limits.toml" +plugin-linea-deny-list-path="/var/lib/besu/deny-list.txt" +plugin-linea-l1l2-bridge-contract="0xe537D669CA013d86EBeF1D64e40fC74CADC91987" +plugin-linea-l1l2-bridge-topic="e856c2b8bd4eb0027ce32eeaf595c21b0b6b4644b326e5b7bd80a1cf8db72e6c" +plugin-linea-tx-pool-profitability-check-p2p-enabled=false +plugin-linea-tx-pool-profitability-check-api-enabled=false +plugin-linea-tx-pool-simulation-check-api-enabled=true +plugin-linea-tx-pool-simulation-check-p2p-enabled=false +plugin-linea-estimate-gas-compatibility-mode-enabled=false + +Xbonsai-limit-trie-logs-enabled=false +bonsai-historical-block-limit=1024 diff --git a/docker/config/l2-node-besu/log4j.xml b/docker/config/l2-node-besu/log4j.xml new file mode 100644 index 00000000..1f91382a --- /dev/null +++ b/docker/config/l2-node-besu/log4j.xml @@ -0,0 +1,44 @@ + + + + WARN + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker/postgres/init/create-schema.sql b/docker/postgres/init/create-schema.sql index 907e1855..3841fe4c 100644 --- a/docker/postgres/init/create-schema.sql +++ b/docker/postgres/init/create-schema.sql @@ -2,4 +2,4 @@ CREATE DATABASE linea_coordinator; CREATE DATABASE postman_db; CREATE DATABASE l1_blockscout_db; CREATE DATABASE l2_blockscout_db; - +CREATE DATABASE linea_transaction_exclusion; diff --git a/e2e/src/abi/TestContract.json b/e2e/src/abi/TestContract.json new file mode 100644 index 00000000..5494a22a --- /dev/null +++ b/e2e/src/abi/TestContract.json @@ -0,0 +1,306 @@ +{ + "contractName": "TestContract", + "sourceName": "testing-tools/app/src/main/solidity/TestContract.sol", + "abi": [ + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "sender", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "value", + "type": "uint256" + } + ], + "name": "TestEvent", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "address", + "name": "sender", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "value", + "type": "uint256" + } + ], + "name": "TestEventWithoutIndexing", + "type": "event" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "_key", + "type": "address" + }, + { + "internalType": "uint256", + "name": "_value", + "type": "uint256" + } + ], + "name": "addToStorageMap", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_value", + "type": "uint256" + } + ], + "name": "createStruct", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "_key", + "type": "address" + } + ], + "name": "deleteFromStorageMap", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "name": "storageMap", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_count", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "_mod", + "type": "uint256" + } + ], + "name": "testAddmod", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "string", + "name": "_input", + "type": "string" + } + ], + "name": "testEncoding", + "outputs": [ + { + "internalType": "bytes", + "name": "", + "type": "bytes" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "string", + "name": "_input", + "type": "string" + } + ], + "name": "testEncodingPacked", + "outputs": [ + { + "internalType": "bytes", + "name": "", + "type": "bytes" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_value", + "type": "uint256" + } + ], + "name": "testEventEmitting", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "target", + "type": "address" + }, + { + "internalType": "bytes", + "name": "_data", + "type": "bytes" + } + ], + "name": "testExternalCalls", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_value", + "type": "uint256" + } + ], + "name": "testIndexedEventEmitting", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "string", + "name": "_input", + "type": "string" + } + ], + "name": "testKeccak", + "outputs": [ + { + "internalType": "bytes32", + "name": "", + "type": "bytes32" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "string", + "name": "_input", + "type": "string" + } + ], + "name": "testKeccak2", + "outputs": [ + { + "internalType": "bytes32", + "name": "", + "type": "bytes32" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_count", + "type": "uint256" + } + ], + "name": "testKeccak3", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_count", + "type": "uint256" + } + ], + "name": "testKeccak4", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_count", + "type": "uint256" + }, + { + "internalType": "uint256", + "name": "_mod", + "type": "uint256" + } + ], + "name": "testMulmod", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "testStruct", + "outputs": [ + { + "internalType": "uint256", + "name": "value", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + } + ], + "bytecode": "0x608060405234801561001057600080fd5b5061103b806100206000396000f3fe608060405234801561001057600080fd5b50600436106101005760003560e01c806381cec0fe11610097578063c7dd97f611610066578063c7dd97f6146102a7578063d75522da146102d7578063de36d657146102f3578063eb6c7bc01461030f57610100565b806381cec0fe146102355780639b0e317c14610251578063c221b6dc1461026f578063c2d7ff5b1461028b57610100565b8063445a31dc116100d3578063445a31dc1461018957806348286a55146101b957806365e88438146101e95780636bd969ad1461020557610100565b8063101eb93a1461010557806314b05a3814610121578063293fd05d146101515780633e8b68c11461016d575b600080fd5b61011f600480360381019061011a91906107ff565b61032b565b005b61013b60048036038101906101369190610972565b610375565b60405161014891906109d4565b60405180910390f35b61016b600480360381019061016691906107ff565b6103a5565b005b610187600480360381019061018291906109ef565b6103ef565b005b6101a3600480360381019061019e9190610a8d565b610430565b6040516101b09190610ac9565b60405180910390f35b6101d360048036038101906101ce9190610972565b610448565b6040516101e09190610b63565b60405180910390f35b61020360048036038101906101fe91906107ff565b610471565b005b61021f600480360381019061021a9190610972565b6104c2565b60405161022c9190610b63565b60405180910390f35b61024f600480360381019061024a9190610c26565b6104eb565b005b610259610648565b6040516102669190610ac9565b60405180910390f35b610289600480360381019061028491906109ef565b610654565b005b6102a560048036038101906102a091906107ff565b610695565b005b6102c160048036038101906102bc9190610972565b6106d1565b6040516102ce91906109d4565b60405180910390f35b6102f160048036038101906102ec91906107ff565b610701565b005b61030d60048036038101906103089190610c82565b610727565b005b61032960048036038101906103249190610a8d565b61076f565b005b60005b8181101561037157806040516020016103479190610ce3565b6040516020818303038152906040528051906020012050808061036990610d2d565b91505061032e565b5050565b6000816040516020016103889190610dbc565b604051602081830303815290604052805190602001209050919050565b60005b818110156103eb57806040516020016103c19190610ce3565b604051602081830303815290604052805190602001205080806103e390610d2d565b9150506103a8565b5050565b60005b8281101561042b57818061040957610408610dd3565b5b6001826104169190610e02565b5050808061042390610d2d565b9150506103f2565b505050565b60016020528060005260406000206000915090505481565b60608160405160200161045b9190610e80565b6040516020818303038152906040529050919050565b3373ffffffffffffffffffffffffffffffffffffffff167f2a1343a7ef16865394327596242ebb1d13cafbd9dbb29027e89cbc0212cfa737826040516104b79190610ac9565b60405180910390a250565b6060816040516020016104d59190610dbc565b6040516020818303038152906040529050919050565b60008273ffffffffffffffffffffffffffffffffffffffff16826040516105129190610ede565b6000604051808303816000865af19150503d806000811461054f576040519150601f19603f3d011682016040523d82523d6000602084013e610554565b606091505b5050905080610598576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161058f90610f41565b60405180910390fd5b3073ffffffffffffffffffffffffffffffffffffffff16826040516105bd9190610ede565b600060405180830381855af49150503d80600081146105f8576040519150601f19603f3d011682016040523d82523d6000602084013e6105fd565b606091505b50508091505080610643576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161063a90610fad565b60405180910390fd5b505050565b60008060000154905081565b60005b8281101561069057818061066e5761066d610dd3565b5b60018261067b9190610e02565b5050808061068890610d2d565b915050610657565b505050565b7f86a4f961b36de6c45328ad9f8656c035c3f2414b5710cb261e19c9b0a885120333826040516106c6929190610fdc565b60405180910390a150565b6000816040516020016106e49190610e80565b604051602081830303815290604052805190602001209050919050565b600060405180602001604052808381525090508060008082015181600001559050505050565b80600160008473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020819055505050565b600160008273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000905550565b6000604051905090565b600080fd5b600080fd5b6000819050919050565b6107dc816107c9565b81146107e757600080fd5b50565b6000813590506107f9816107d3565b92915050565b600060208284031215610815576108146107bf565b5b6000610823848285016107ea565b91505092915050565b600080fd5b600080fd5b6000601f19601f8301169050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b61087f82610836565b810181811067ffffffffffffffff8211171561089e5761089d610847565b5b80604052505050565b60006108b16107b5565b90506108bd8282610876565b919050565b600067ffffffffffffffff8211156108dd576108dc610847565b5b6108e682610836565b9050602081019050919050565b82818337600083830152505050565b6000610915610910846108c2565b6108a7565b90508281526020810184848401111561093157610930610831565b5b61093c8482856108f3565b509392505050565b600082601f8301126109595761095861082c565b5b8135610969848260208601610902565b91505092915050565b600060208284031215610988576109876107bf565b5b600082013567ffffffffffffffff8111156109a6576109a56107c4565b5b6109b284828501610944565b91505092915050565b6000819050919050565b6109ce816109bb565b82525050565b60006020820190506109e960008301846109c5565b92915050565b60008060408385031215610a0657610a056107bf565b5b6000610a14858286016107ea565b9250506020610a25858286016107ea565b9150509250929050565b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610a5a82610a2f565b9050919050565b610a6a81610a4f565b8114610a7557600080fd5b50565b600081359050610a8781610a61565b92915050565b600060208284031215610aa357610aa26107bf565b5b6000610ab184828501610a78565b91505092915050565b610ac3816107c9565b82525050565b6000602082019050610ade6000830184610aba565b92915050565b600081519050919050565b600082825260208201905092915050565b60005b83811015610b1e578082015181840152602081019050610b03565b60008484015250505050565b6000610b3582610ae4565b610b3f8185610aef565b9350610b4f818560208601610b00565b610b5881610836565b840191505092915050565b60006020820190508181036000830152610b7d8184610b2a565b905092915050565b600067ffffffffffffffff821115610ba057610b9f610847565b5b610ba982610836565b9050602081019050919050565b6000610bc9610bc484610b85565b6108a7565b905082815260208101848484011115610be557610be4610831565b5b610bf08482856108f3565b509392505050565b600082601f830112610c0d57610c0c61082c565b5b8135610c1d848260208601610bb6565b91505092915050565b60008060408385031215610c3d57610c3c6107bf565b5b6000610c4b85828601610a78565b925050602083013567ffffffffffffffff811115610c6c57610c6b6107c4565b5b610c7885828601610bf8565b9150509250929050565b60008060408385031215610c9957610c986107bf565b5b6000610ca785828601610a78565b9250506020610cb8858286016107ea565b9150509250929050565b6000819050919050565b610cdd610cd8826107c9565b610cc2565b82525050565b6000610cef8284610ccc565b60208201915081905092915050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b6000610d38826107c9565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8203610d6a57610d69610cfe565b5b600182019050919050565b600081519050919050565b600081905092915050565b6000610d9682610d75565b610da08185610d80565b9350610db0818560208601610b00565b80840191505092915050565b6000610dc88284610d8b565b915081905092915050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601260045260246000fd5b6000610e0d826107c9565b9150610e18836107c9565b9250828201905080821115610e3057610e2f610cfe565b5b92915050565b600082825260208201905092915050565b6000610e5282610d75565b610e5c8185610e36565b9350610e6c818560208601610b00565b610e7581610836565b840191505092915050565b60006020820190508181036000830152610e9a8184610e47565b905092915050565b600081905092915050565b6000610eb882610ae4565b610ec28185610ea2565b9350610ed2818560208601610b00565b80840191505092915050565b6000610eea8284610ead565b915081905092915050565b7f43616c6c206661696c6564000000000000000000000000000000000000000000600082015250565b6000610f2b600b83610e36565b9150610f3682610ef5565b602082019050919050565b60006020820190508181036000830152610f5a81610f1e565b9050919050565b7f44656c656761746563616c6c206661696c656400000000000000000000000000600082015250565b6000610f97601383610e36565b9150610fa282610f61565b602082019050919050565b60006020820190508181036000830152610fc681610f8a565b9050919050565b610fd681610a4f565b82525050565b6000604082019050610ff16000830185610fcd565b610ffe6020830184610aba565b939250505056fea264697066735822122080fe310546887f97e98457cfe618fd37892ac5ce4ea3a38651d63a88de7d782a64736f6c63430008130033", + "deployedBytecode": "0x608060405234801561001057600080fd5b50600436106101005760003560e01c806381cec0fe11610097578063c7dd97f611610066578063c7dd97f6146102a7578063d75522da146102d7578063de36d657146102f3578063eb6c7bc01461030f57610100565b806381cec0fe146102355780639b0e317c14610251578063c221b6dc1461026f578063c2d7ff5b1461028b57610100565b8063445a31dc116100d3578063445a31dc1461018957806348286a55146101b957806365e88438146101e95780636bd969ad1461020557610100565b8063101eb93a1461010557806314b05a3814610121578063293fd05d146101515780633e8b68c11461016d575b600080fd5b61011f600480360381019061011a91906107ff565b61032b565b005b61013b60048036038101906101369190610972565b610375565b60405161014891906109d4565b60405180910390f35b61016b600480360381019061016691906107ff565b6103a5565b005b610187600480360381019061018291906109ef565b6103ef565b005b6101a3600480360381019061019e9190610a8d565b610430565b6040516101b09190610ac9565b60405180910390f35b6101d360048036038101906101ce9190610972565b610448565b6040516101e09190610b63565b60405180910390f35b61020360048036038101906101fe91906107ff565b610471565b005b61021f600480360381019061021a9190610972565b6104c2565b60405161022c9190610b63565b60405180910390f35b61024f600480360381019061024a9190610c26565b6104eb565b005b610259610648565b6040516102669190610ac9565b60405180910390f35b610289600480360381019061028491906109ef565b610654565b005b6102a560048036038101906102a091906107ff565b610695565b005b6102c160048036038101906102bc9190610972565b6106d1565b6040516102ce91906109d4565b60405180910390f35b6102f160048036038101906102ec91906107ff565b610701565b005b61030d60048036038101906103089190610c82565b610727565b005b61032960048036038101906103249190610a8d565b61076f565b005b60005b8181101561037157806040516020016103479190610ce3565b6040516020818303038152906040528051906020012050808061036990610d2d565b91505061032e565b5050565b6000816040516020016103889190610dbc565b604051602081830303815290604052805190602001209050919050565b60005b818110156103eb57806040516020016103c19190610ce3565b604051602081830303815290604052805190602001205080806103e390610d2d565b9150506103a8565b5050565b60005b8281101561042b57818061040957610408610dd3565b5b6001826104169190610e02565b5050808061042390610d2d565b9150506103f2565b505050565b60016020528060005260406000206000915090505481565b60608160405160200161045b9190610e80565b6040516020818303038152906040529050919050565b3373ffffffffffffffffffffffffffffffffffffffff167f2a1343a7ef16865394327596242ebb1d13cafbd9dbb29027e89cbc0212cfa737826040516104b79190610ac9565b60405180910390a250565b6060816040516020016104d59190610dbc565b6040516020818303038152906040529050919050565b60008273ffffffffffffffffffffffffffffffffffffffff16826040516105129190610ede565b6000604051808303816000865af19150503d806000811461054f576040519150601f19603f3d011682016040523d82523d6000602084013e610554565b606091505b5050905080610598576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161058f90610f41565b60405180910390fd5b3073ffffffffffffffffffffffffffffffffffffffff16826040516105bd9190610ede565b600060405180830381855af49150503d80600081146105f8576040519150601f19603f3d011682016040523d82523d6000602084013e6105fd565b606091505b50508091505080610643576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161063a90610fad565b60405180910390fd5b505050565b60008060000154905081565b60005b8281101561069057818061066e5761066d610dd3565b5b60018261067b9190610e02565b5050808061068890610d2d565b915050610657565b505050565b7f86a4f961b36de6c45328ad9f8656c035c3f2414b5710cb261e19c9b0a885120333826040516106c6929190610fdc565b60405180910390a150565b6000816040516020016106e49190610e80565b604051602081830303815290604052805190602001209050919050565b600060405180602001604052808381525090508060008082015181600001559050505050565b80600160008473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020819055505050565b600160008273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000905550565b6000604051905090565b600080fd5b600080fd5b6000819050919050565b6107dc816107c9565b81146107e757600080fd5b50565b6000813590506107f9816107d3565b92915050565b600060208284031215610815576108146107bf565b5b6000610823848285016107ea565b91505092915050565b600080fd5b600080fd5b6000601f19601f8301169050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b61087f82610836565b810181811067ffffffffffffffff8211171561089e5761089d610847565b5b80604052505050565b60006108b16107b5565b90506108bd8282610876565b919050565b600067ffffffffffffffff8211156108dd576108dc610847565b5b6108e682610836565b9050602081019050919050565b82818337600083830152505050565b6000610915610910846108c2565b6108a7565b90508281526020810184848401111561093157610930610831565b5b61093c8482856108f3565b509392505050565b600082601f8301126109595761095861082c565b5b8135610969848260208601610902565b91505092915050565b600060208284031215610988576109876107bf565b5b600082013567ffffffffffffffff8111156109a6576109a56107c4565b5b6109b284828501610944565b91505092915050565b6000819050919050565b6109ce816109bb565b82525050565b60006020820190506109e960008301846109c5565b92915050565b60008060408385031215610a0657610a056107bf565b5b6000610a14858286016107ea565b9250506020610a25858286016107ea565b9150509250929050565b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610a5a82610a2f565b9050919050565b610a6a81610a4f565b8114610a7557600080fd5b50565b600081359050610a8781610a61565b92915050565b600060208284031215610aa357610aa26107bf565b5b6000610ab184828501610a78565b91505092915050565b610ac3816107c9565b82525050565b6000602082019050610ade6000830184610aba565b92915050565b600081519050919050565b600082825260208201905092915050565b60005b83811015610b1e578082015181840152602081019050610b03565b60008484015250505050565b6000610b3582610ae4565b610b3f8185610aef565b9350610b4f818560208601610b00565b610b5881610836565b840191505092915050565b60006020820190508181036000830152610b7d8184610b2a565b905092915050565b600067ffffffffffffffff821115610ba057610b9f610847565b5b610ba982610836565b9050602081019050919050565b6000610bc9610bc484610b85565b6108a7565b905082815260208101848484011115610be557610be4610831565b5b610bf08482856108f3565b509392505050565b600082601f830112610c0d57610c0c61082c565b5b8135610c1d848260208601610bb6565b91505092915050565b60008060408385031215610c3d57610c3c6107bf565b5b6000610c4b85828601610a78565b925050602083013567ffffffffffffffff811115610c6c57610c6b6107c4565b5b610c7885828601610bf8565b9150509250929050565b60008060408385031215610c9957610c986107bf565b5b6000610ca785828601610a78565b9250506020610cb8858286016107ea565b9150509250929050565b6000819050919050565b610cdd610cd8826107c9565b610cc2565b82525050565b6000610cef8284610ccc565b60208201915081905092915050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b6000610d38826107c9565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8203610d6a57610d69610cfe565b5b600182019050919050565b600081519050919050565b600081905092915050565b6000610d9682610d75565b610da08185610d80565b9350610db0818560208601610b00565b80840191505092915050565b6000610dc88284610d8b565b915081905092915050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601260045260246000fd5b6000610e0d826107c9565b9150610e18836107c9565b9250828201905080821115610e3057610e2f610cfe565b5b92915050565b600082825260208201905092915050565b6000610e5282610d75565b610e5c8185610e36565b9350610e6c818560208601610b00565b610e7581610836565b840191505092915050565b60006020820190508181036000830152610e9a8184610e47565b905092915050565b600081905092915050565b6000610eb882610ae4565b610ec28185610ea2565b9350610ed2818560208601610b00565b80840191505092915050565b6000610eea8284610ead565b915081905092915050565b7f43616c6c206661696c6564000000000000000000000000000000000000000000600082015250565b6000610f2b600b83610e36565b9150610f3682610ef5565b602082019050919050565b60006020820190508181036000830152610f5a81610f1e565b9050919050565b7f44656c656761746563616c6c206661696c656400000000000000000000000000600082015250565b6000610f97601383610e36565b9150610fa282610f61565b602082019050919050565b60006020820190508181036000830152610fc681610f8a565b9050919050565b610fd681610a4f565b82525050565b6000604082019050610ff16000830185610fcd565b610ffe6020830184610aba565b939250505056fea264697066735822122080fe310546887f97e98457cfe618fd37892ac5ce4ea3a38651d63a88de7d782a64736f6c63430008130033", + "linkReferences": {}, + "deployedLinkReferences": {} +} \ No newline at end of file diff --git a/e2e/src/common/utils.ts b/e2e/src/common/utils.ts index 319aae07..499f2019 100644 --- a/e2e/src/common/utils.ts +++ b/e2e/src/common/utils.ts @@ -1,6 +1,6 @@ import * as fs from "fs"; import assert from "assert"; -import { BaseContract, BlockTag, TransactionReceipt, Wallet, ethers } from "ethers"; +import { BaseContract, BlockTag, TransactionReceipt, TransactionRequest, Wallet, ethers } from "ethers"; import path from "path"; import { exec } from "child_process"; import { L2MessageService, LineaRollup } from "../typechain"; @@ -72,6 +72,68 @@ export class RollupGetZkEVMBlockNumberClient { } } +export class TransactionExclusionClient { + private endpoint: URL; + + public constructor(endpoint: URL) { + this.endpoint = endpoint; + } + + public async getTransactionExclusionStatusV1(txHash: String): Promise { + const request = { + method: "post", + body: JSON.stringify({ + jsonrpc: "2.0", + method: "linea_getTransactionExclusionStatusV1", + params: [txHash], + id: 1, + }), + }; + const response = await fetch(this.endpoint, request); + return await response.json(); + } + + public async saveRejectedTransactionV1( + txRejectionStage: String, + timestamp: String, // ISO-8601 + blockNumber: Number | null, + transactionRLP: String, + reasonMessage: String, + overflows: { module: String; count: Number; limit: Number }[], + ): Promise { + let params: any = { + txRejectionStage, + timestamp, + transactionRLP, + reasonMessage, + overflows, + }; + if (blockNumber != null) { + params = { + ...params, + blockNumber, + }; + } + const request = { + method: "post", + body: JSON.stringify({ + jsonrpc: "2.0", + method: "linea_saveRejectedTransactionV1", + params: params, + id: 1, + }), + }; + const response = await fetch(this.endpoint, request); + return await response.json(); + } +} + +export async function getTransactionHash(txRequest: TransactionRequest, signer: Wallet): Promise { + const rawTransaction = await signer.populateTransaction(txRequest); + const signature = await signer.signTransaction(rawTransaction); + return ethers.keccak256(signature); +} + export async function getBlockByNumberOrBlockTag(rpcUrl: URL, blockTag: BlockTag): Promise { const provider = new ethers.JsonRpcProvider(rpcUrl.href); try { diff --git a/e2e/src/config/jest/global-setup.ts b/e2e/src/config/jest/global-setup.ts index 2d610c04..96a4d777 100644 --- a/e2e/src/config/jest/global-setup.ts +++ b/e2e/src/config/jest/global-setup.ts @@ -1,7 +1,7 @@ /* eslint-disable no-var */ import { config } from "../tests-config"; import { deployContract } from "../../common/deployments"; -import { DummyContract__factory } from "../../typechain"; +import { DummyContract__factory, TestContract__factory } from "../../typechain"; import { etherToWei, sendTransactionsToGenerateTrafficWithInterval } from "../../common/utils"; declare global { @@ -16,10 +16,12 @@ export default async (): Promise => { deployContract(new DummyContract__factory(), account), deployContract(new DummyContract__factory(), l2Account), ]); - console.log(`L1 Dummy contract deployed at address: ${await dummyContract.getAddress()}`); console.log(`L2 Dummy contract deployed at address: ${await l2DummyContract.getAddress()}`); + const l2TestContract = await deployContract(new TestContract__factory(), l2Account); + console.log(`L2 Test contract deployed at address: ${await l2TestContract.getAddress()}`); + // Send ETH to the LineaRollup contract const lineaRollup = config.getLineaRollupContract(account); const l1JsonRpcProvider = config.getL1Provider(); diff --git a/e2e/src/config/tests-config/environments/local.ts b/e2e/src/config/tests-config/environments/local.ts index c4607eed..30d27d87 100644 --- a/e2e/src/config/tests-config/environments/local.ts +++ b/e2e/src/config/tests-config/environments/local.ts @@ -5,9 +5,11 @@ import { Config } from "../types"; const L1_RPC_URL = new URL("http://localhost:8445"); const L2_RPC_URL = new URL("http://localhost:8845"); +const L2_BESU_NODE_RPC_URL = new URL("http://localhost:9045"); const SHOMEI_ENDPOINT = new URL("http://localhost:8998"); const SHOMEI_FRONTEND_ENDPOINT = new URL("http://localhost:8889"); const SEQUENCER_ENDPOINT = new URL("http://localhost:8545"); +const TRANSACTION_EXCLUSION_ENDPOINT = new URL("http://localhost:8082"); const config: Config = { L1: { @@ -25,8 +27,10 @@ const config: Config = { }, L2: { rpcUrl: L2_RPC_URL, + besuNodeRpcUrl: L2_BESU_NODE_RPC_URL, chainId: 1337, l2MessageServiceAddress: "0xe537D669CA013d86EBeF1D64e40fC74CADC91987", + l2TestContractAddress: "0xeB0b0a14F92e3BA35aEF3a2B6A24D7ED1D11631B", dummyContractAddress: "0x2f6dAaF8A81AB675fbD37Ca6Ed5b72cf86237453", accountManager: new GenesisBasedAccountManager( new ethers.JsonRpcProvider(L2_RPC_URL.toString()), @@ -38,6 +42,7 @@ const config: Config = { shomeiEndpoint: SHOMEI_ENDPOINT, shomeiFrontendEndpoint: SHOMEI_FRONTEND_ENDPOINT, sequencerEndpoint: SEQUENCER_ENDPOINT, + transactionExclusionEndpoint: TRANSACTION_EXCLUSION_ENDPOINT, }, }; diff --git a/e2e/src/config/tests-config/setup.ts b/e2e/src/config/tests-config/setup.ts index d54879ba..600fe127 100644 --- a/e2e/src/config/tests-config/setup.ts +++ b/e2e/src/config/tests-config/setup.ts @@ -7,6 +7,8 @@ import { L2MessageService__factory, LineaRollup, LineaRollup__factory, + TestContract, + TestContract__factory, } from "../../typechain"; import { AccountManager } from "./accounts/account-manager"; @@ -21,6 +23,22 @@ export default class TestSetup { return new JsonRpcProvider(this.config.L2.rpcUrl.toString()); } + public getL2SequencerProvider(): JsonRpcProvider | undefined { + if (this.config.L2.sequencerEndpoint) { + return new JsonRpcProvider(this.config.L2.sequencerEndpoint.toString()); + } else { + return undefined; + } + } + + public getL2BesuNodeProvider(): JsonRpcProvider | undefined { + if (this.config.L2.besuNodeRpcUrl) { + return new JsonRpcProvider(this.config.L2.besuNodeRpcUrl.toString()); + } else { + return undefined; + } + } + public getL1ChainId(): number { return this.config.L1.chainId; } @@ -41,6 +59,10 @@ export default class TestSetup { return this.config.L2.sequencerEndpoint; } + public getTransactionExclusionEndpoint(): URL | undefined { + return this.config.L2.transactionExclusionEndpoint; + } + public getLineaRollupContract(signer?: Wallet): LineaRollup { const lineaRollup: LineaRollup = LineaRollup__factory.connect( this.config.L1.lineaRollupAddress, @@ -87,6 +109,20 @@ export default class TestSetup { return dummyContract; } + public getL2TestContract(signer?: Wallet): TestContract | undefined { + if (this.config.L2.l2TestContractAddress) { + const testContract = TestContract__factory.connect(this.config.L2.l2TestContractAddress, this.getL2Provider()); + + if (signer) { + return testContract.connect(signer); + } + + return testContract; + } else { + return undefined; + } + } + public getL1AccountManager(): AccountManager { return this.config.L1.accountManager; } diff --git a/e2e/src/config/tests-config/types.ts b/e2e/src/config/tests-config/types.ts index 832ca0ad..c4a3d167 100644 --- a/e2e/src/config/tests-config/types.ts +++ b/e2e/src/config/tests-config/types.ts @@ -13,9 +13,12 @@ export type L1Config = BaseConfig & { export type L2Config = BaseConfig & { l2MessageServiceAddress: string; + l2TestContractAddress?: string; + besuNodeRpcUrl?: URL; shomeiEndpoint?: URL; shomeiFrontendEndpoint?: URL; sequencerEndpoint?: URL; + transactionExclusionEndpoint?: URL; }; export type Config = { diff --git a/e2e/src/l2.spec.ts b/e2e/src/l2.spec.ts index c064e76f..e506d145 100644 --- a/e2e/src/l2.spec.ts +++ b/e2e/src/l2.spec.ts @@ -1,5 +1,5 @@ -import { JsonRpcProvider, ethers } from "ethers"; -import { beforeAll, describe, expect, it } from "@jest/globals"; +import { ethers } from "ethers"; +import { describe, expect, it } from "@jest/globals"; import { config } from "./config/tests-config"; import { RollupGetZkEVMBlockNumberClient, etherToWei } from "./common/utils"; import { TRANSACTION_CALLDATA_LIMIT } from "./common/constants"; @@ -7,12 +7,6 @@ import { TRANSACTION_CALLDATA_LIMIT } from "./common/constants"; const l2AccountManager = config.getL2AccountManager(); describe("Layer 2 test suite", () => { - let l2Provider: JsonRpcProvider; - - beforeAll(() => { - l2Provider = config.getL2Provider(); - }); - it.concurrent("Should revert if transaction data size is above the limit", async () => { const account = await l2AccountManager.generateAccount(); const dummyContract = config.getL2DummyContract(account); @@ -35,7 +29,7 @@ describe("Layer 2 test suite", () => { it.concurrent("Should successfully send a legacy transaction", async () => { const account = await l2AccountManager.generateAccount(); - const { gasPrice } = await l2Provider.getFeeData(); + const { gasPrice } = await config.getL2Provider().getFeeData(); const receipt = await ( await account.sendTransaction({ @@ -54,7 +48,7 @@ describe("Layer 2 test suite", () => { it.concurrent("Should successfully send an EIP1559 transaction", async () => { const account = await l2AccountManager.generateAccount(); - const { maxPriorityFeePerGas, maxFeePerGas } = await l2Provider.getFeeData(); + const { maxPriorityFeePerGas, maxFeePerGas } = await config.getL2Provider().getFeeData(); const receipt = await ( await account.sendTransaction({ @@ -74,7 +68,7 @@ describe("Layer 2 test suite", () => { it.concurrent("Should successfully send an access list transaction with empty access list", async () => { const account = await l2AccountManager.generateAccount(); - const { gasPrice } = await l2Provider.getFeeData(); + const { gasPrice } = await config.getL2Provider().getFeeData(); const receipt = await ( await account.sendTransaction({ @@ -93,7 +87,7 @@ describe("Layer 2 test suite", () => { it.concurrent("Should successfully send an access list transaction with access list", async () => { const account = await l2AccountManager.generateAccount(); - const { gasPrice } = await l2Provider.getFeeData(); + const { gasPrice } = await config.getL2Provider().getFeeData(); const accessList = { "0x8D97689C9818892B700e27F316cc3E41e17fBeb9": [ "0x0000000000000000000000000000000000000000000000000000000000000000", @@ -131,7 +125,7 @@ describe("Layer 2 test suite", () => { const shomeiFrontendClient = new RollupGetZkEVMBlockNumberClient(shomeiFrontendEndpoint); for (let i = 0; i < 5; i++) { - const { maxPriorityFeePerGas, maxFeePerGas } = await l2Provider.getFeeData(); + const { maxPriorityFeePerGas, maxFeePerGas } = await config.getL2Provider().getFeeData(); await ( await account.sendTransaction({ diff --git a/e2e/src/submission-finalization.spec.ts b/e2e/src/submission-finalization.spec.ts index 2b100b64..8fa7e538 100644 --- a/e2e/src/submission-finalization.spec.ts +++ b/e2e/src/submission-finalization.spec.ts @@ -1,5 +1,4 @@ import { describe, expect, it } from "@jest/globals"; -import { JsonRpcProvider } from "ethers"; import { getMessageSentEventFromLogs, sendMessage, @@ -12,12 +11,6 @@ import { config } from "./config/tests-config"; import { LineaRollup } from "./typechain"; describe("Submission and finalization test suite", () => { - let l1Provider: JsonRpcProvider; - - beforeAll(() => { - l1Provider = config.getL1Provider(); - }); - const sendMessages = async () => { const messageFee = etherToWei("0.0001"); const messageValue = etherToWei("0.0051"); @@ -32,8 +25,8 @@ describe("Submission and finalization test suite", () => { const l1MessagesPromises = []; // eslint-disable-next-line prefer-const let [l1MessageSenderNonce, { maxPriorityFeePerGas, maxFeePerGas }] = await Promise.all([ - l1Provider.getTransactionCount(l1MessageSender.address), - l1Provider.getFeeData(), + config.getL1Provider().getTransactionCount(l1MessageSender.address), + config.getL1Provider().getFeeData(), ]); for (let i = 0; i < 5; i++) { diff --git a/e2e/src/transaction-exclusion.spec.ts b/e2e/src/transaction-exclusion.spec.ts new file mode 100644 index 00000000..15fd857f --- /dev/null +++ b/e2e/src/transaction-exclusion.spec.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "@jest/globals"; +import { config } from "./config/tests-config"; +import { etherToWei, getTransactionHash, getWallet, TransactionExclusionClient, wait } from "./common/utils"; +import { TransactionRequest } from "ethers"; + +const l2AccountManager = config.getL2AccountManager(); + +describe("Transaction exclusion test suite", () => { + it.concurrent( + "Should get the status of the rejected transaction reported from Besu RPC node", + async () => { + expect(config.getTransactionExclusionEndpoint()).toBeDefined(); + + const transactionExclusionClient = new TransactionExclusionClient(config.getTransactionExclusionEndpoint()!!); + const l2Account = await l2AccountManager.generateAccount(); + const l2AccountLocal = getWallet(l2Account.privateKey, config.getL2BesuNodeProvider()!!); + const testContract = config.getL2TestContract(l2AccountLocal)!!; + + // This shall be rejected by the Besu node due to traces module limit overflow + let rejectedTxHash; + try { + const txRequest: TransactionRequest = { + to: await testContract.getAddress(), + data: testContract.interface.encodeFunctionData("testAddmod", [13000, 31]), + maxPriorityFeePerGas: etherToWei("0.000000001"), // 1 Gwei + maxFeePerGas: etherToWei("0.00000001"), // 10 Gwei + }; + rejectedTxHash = await getTransactionHash(txRequest, l2AccountLocal); + await l2AccountLocal.sendTransaction(txRequest); + } catch (err) { + // This shall return error with traces limit overflow + console.debug(`sendTransaction expected err: ${JSON.stringify(err)}`); + } + + expect(rejectedTxHash).toBeDefined(); + console.log(`rejectedTxHash (RPC): ${rejectedTxHash}`); + + let getResponse; + do { + await wait(1_000); + getResponse = await transactionExclusionClient.getTransactionExclusionStatusV1(rejectedTxHash!); + } while (!getResponse?.result); + + expect(getResponse.result.txHash).toStrictEqual(rejectedTxHash); + expect(getResponse.result.txRejectionStage).toStrictEqual("RPC"); + expect(getResponse.result.from.toLowerCase()).toStrictEqual(l2AccountLocal.address.toLowerCase()); + }, + 120_000, + ); + + it.skip("Should get the status of the rejected transaction reported from Besu SEQUENCER node", async () => { + expect(config.getTransactionExclusionEndpoint()).toBeDefined(); + + const transactionExclusionClient = new TransactionExclusionClient(config.getTransactionExclusionEndpoint()!!); + const l2Account = await l2AccountManager.generateAccount(); + const l2AccountLocal = getWallet(l2Account.privateKey, config.getL2SequencerProvider()!!); + const testContract = config.getL2TestContract(l2AccountLocal); + + // This shall be rejected by sequencer due to traces module limit overflow + const tx = await testContract!!.connect(l2AccountLocal).testAddmod(13000, 31); + const rejectedTxHash = tx.hash; + console.log(`rejectedTxHash (SEQUENCER): ${rejectedTxHash}`); + + let getResponse; + do { + await wait(1_000); + getResponse = await transactionExclusionClient.getTransactionExclusionStatusV1(rejectedTxHash); + } while (!getResponse?.result); + + expect(getResponse.result.txHash).toStrictEqual(rejectedTxHash); + expect(getResponse.result.txRejectionStage).toStrictEqual("SEQUENCER"); + expect(getResponse.result.from.toLowerCase()).toStrictEqual(l2AccountLocal.address.toLowerCase()); + }, 120_000); +}); diff --git a/coordinator/persistence/db/build.gradle b/jvm-libs/generic/persistence/db/build.gradle similarity index 77% rename from coordinator/persistence/db/build.gradle rename to jvm-libs/generic/persistence/db/build.gradle index b5ae0983..a8301a9a 100644 --- a/coordinator/persistence/db/build.gradle +++ b/jvm-libs/generic/persistence/db/build.gradle @@ -4,8 +4,9 @@ plugins { } dependencies { - api(project(":coordinator:core")) - api("io.vertx:vertx-pg-client") + implementation(project(":jvm-libs:generic:extensions:futures")) + implementation(project(":jvm-libs:generic:extensions:kotlin")) + api("io.vertx:vertx-pg-client:${libs.versions.vertx.get()}") api("com.ongres.scram:common:2.1") { because("Vertx pg client fails without it") } @@ -19,8 +20,9 @@ dependencies { } testImplementation testFixtures(project(':jvm-libs:generic:extensions:kotlin')) - testFixturesApi(platform("org.junit:junit-bom:${libs.versions.junit.get()}")) - testFixturesApi("io.vertx:vertx-junit5:${libs.versions.vertx.get()}") + + testFixturesImplementation(platform("org.junit:junit-bom:${libs.versions.junit.get()}")) + testFixturesImplementation("io.vertx:vertx-junit5:${libs.versions.vertx.get()}") } sourceSets { diff --git a/coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/Db.kt b/jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/Db.kt similarity index 92% rename from coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/Db.kt rename to jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/Db.kt index 8233ee0e..1cbe4fdc 100644 --- a/coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/Db.kt +++ b/jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/Db.kt @@ -27,7 +27,8 @@ object Db { database: String, target: String, username: String, - password: String + password: String, + migrationLocations: String = "classpath:db/" ) { val dataSource = PGSimpleDataSource().apply { @@ -37,7 +38,7 @@ object Db { this.password = password this.databaseName = database } - applyDbMigrations(dataSource, target) + applyDbMigrations(dataSource, target, migrationLocations) } /** @@ -45,11 +46,15 @@ object Db { * * @param dataSource datasource */ - fun applyDbMigrations(dataSource: DataSource, target: String) { + fun applyDbMigrations( + dataSource: DataSource, + target: String, + migrationLocations: String = "classpath:db/" + ) { LOG.info("Migrating coordinator database") Flyway.configure() .dataSource(dataSource) - .locations("classpath:db/") + .locations(migrationLocations) .baselineDescription("Migration baseline from legacy database generated ${Instant.now()}") .table("schema_version") .target(target) diff --git a/coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/DbHelper.kt b/jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/DbHelper.kt similarity index 98% rename from coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/DbHelper.kt rename to jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/DbHelper.kt index 71d03d44..8e38fd76 100644 --- a/coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/DbHelper.kt +++ b/jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/DbHelper.kt @@ -12,7 +12,7 @@ object DbHelper { private var formatter: DateTimeFormatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss") fun generateUniqueDbName( - prefix: String = "coordinator_test", + prefix: String = "test", clock: Clock = Clock.systemUTC() ): String { // Just time is not enough, as we can have multiple tests running in parallel diff --git a/coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/PersistenceRetryer.kt b/jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/PersistenceRetryer.kt similarity index 78% rename from coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/PersistenceRetryer.kt rename to jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/PersistenceRetryer.kt index 0e3f8788..c9386d4f 100644 --- a/coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/PersistenceRetryer.kt +++ b/jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/PersistenceRetryer.kt @@ -20,8 +20,21 @@ open class PersistenceRetryer( fun retryQuery( action: () -> SafeFuture, - stopRetriesOnErrorPredicate: (Throwable) -> Boolean = ::stopRetriesOnErrorPredicate - + stopRetriesOnErrorPredicate: (Throwable) -> Boolean = Companion::stopRetriesOnErrorPredicate, + exceptionConsumer: (Throwable) -> Unit = { error -> + when { + isDuplicateKeyException(error) -> log.info( + "Persistence errorMessage={}", + error.message + ) + else -> log.info( + "Persistence errorMessage={}, it will retry again in {}", + error.message, + config.backoffDelay, + error + ) + } + } ): SafeFuture { return AsyncRetryer.retry( vertx = vertx, @@ -29,14 +42,7 @@ open class PersistenceRetryer( backoffDelay = config.backoffDelay, maxRetries = config.maxRetries, stopRetriesOnErrorPredicate = stopRetriesOnErrorPredicate, - exceptionConsumer = { error -> - log.info( - "Persistence errorMessage={}, it will retry again in {}", - error.message, - config.backoffDelay, - error - ) - }, + exceptionConsumer = exceptionConsumer, action = action ) } diff --git a/coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/PostgresHelperFunctions.kt b/jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/PostgresHelperFunctions.kt similarity index 100% rename from coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/PostgresHelperFunctions.kt rename to jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/PostgresHelperFunctions.kt diff --git a/coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/SQLQueryLogger.kt b/jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/SQLQueryLogger.kt similarity index 100% rename from coordinator/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/SQLQueryLogger.kt rename to jvm-libs/generic/persistence/db/src/main/kotlin/net/consensys/zkevm/persistence/db/SQLQueryLogger.kt diff --git a/coordinator/persistence/db/src/test/kotlin/net/consensys/zkevm/persistence/db/DbHelperTest.kt b/jvm-libs/generic/persistence/db/src/test/kotlin/net/consensys/zkevm/persistence/db/DbHelperTest.kt similarity index 100% rename from coordinator/persistence/db/src/test/kotlin/net/consensys/zkevm/persistence/db/DbHelperTest.kt rename to jvm-libs/generic/persistence/db/src/test/kotlin/net/consensys/zkevm/persistence/db/DbHelperTest.kt diff --git a/coordinator/persistence/db/src/test/kotlin/net/consensys/zkevm/persistence/db/PersistenceRetryerTest.kt b/jvm-libs/generic/persistence/db/src/test/kotlin/net/consensys/zkevm/persistence/db/PersistenceRetryerTest.kt similarity index 100% rename from coordinator/persistence/db/src/test/kotlin/net/consensys/zkevm/persistence/db/PersistenceRetryerTest.kt rename to jvm-libs/generic/persistence/db/src/test/kotlin/net/consensys/zkevm/persistence/db/PersistenceRetryerTest.kt diff --git a/coordinator/persistence/db/src/testFixtures/kotlin/net/consensys/zkevm/persistence/test/CleanDbTestSuiteParallel.kt b/jvm-libs/generic/persistence/db/src/testFixtures/kotlin/net/consensys/zkevm/persistence/db/test/CleanDbTestSuiteParallel.kt similarity index 91% rename from coordinator/persistence/db/src/testFixtures/kotlin/net/consensys/zkevm/persistence/test/CleanDbTestSuiteParallel.kt rename to jvm-libs/generic/persistence/db/src/testFixtures/kotlin/net/consensys/zkevm/persistence/db/test/CleanDbTestSuiteParallel.kt index 56af55f2..0740c0c6 100644 --- a/coordinator/persistence/db/src/testFixtures/kotlin/net/consensys/zkevm/persistence/test/CleanDbTestSuiteParallel.kt +++ b/jvm-libs/generic/persistence/db/src/testFixtures/kotlin/net/consensys/zkevm/persistence/db/test/CleanDbTestSuiteParallel.kt @@ -1,4 +1,4 @@ -package net.consensys.zkevm.persistence.test +package net.consensys.zkevm.persistence.db.test import io.vertx.core.Vertx import io.vertx.sqlclient.Pool @@ -15,13 +15,14 @@ import javax.sql.DataSource abstract class CleanDbTestSuiteParallel { private val host = "localhost" private val port = 5432 - abstract val databaseName: String private val username = "postgres" private val password = "postgres" + abstract val databaseName: String private lateinit var dataSource: DataSource lateinit var pool: Pool lateinit var sqlClient: SqlClient - val target: String = "4" + var target: String = "1" + var migrationLocations: String = "classpath:db/" private fun createDataSource(databaseName: String): DataSource { return PGSimpleDataSource().also { @@ -43,7 +44,7 @@ abstract class CleanDbTestSuiteParallel { sqlClient = vertxSqlClient(vertx, host, port, databaseName, username, password) // drop flyway db metadata table as well // to recreate new db tables; - Db.applyDbMigrations(dataSource, target) + Db.applyDbMigrations(dataSource, target, migrationLocations) } @AfterEach diff --git a/jvm-libs/linea/core/metrics/src/main/kotlin/net/consensys/linea/metrics/MetricsFacade.kt b/jvm-libs/linea/core/metrics/src/main/kotlin/net/consensys/linea/metrics/MetricsFacade.kt index 730d340f..311ab9ba 100644 --- a/jvm-libs/linea/core/metrics/src/main/kotlin/net/consensys/linea/metrics/MetricsFacade.kt +++ b/jvm-libs/linea/core/metrics/src/main/kotlin/net/consensys/linea/metrics/MetricsFacade.kt @@ -9,7 +9,8 @@ enum class LineaMetricsCategory { BATCH, BLOB, AGGREGATION, - GAS_PRICE_CAP; + GAS_PRICE_CAP, + TX_EXCLUSION_API; override fun toString(): String { return this.name.replace('_', '.').lowercase() diff --git a/settings.gradle b/settings.gradle index ac007403..bacee433 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,6 +8,7 @@ include 'jvm-libs:generic:logging' include 'jvm-libs:generic:vertx-helper' include 'jvm-libs:generic:extensions:futures' include 'jvm-libs:generic:errors' +include 'jvm-libs:generic:persistence:db' include 'jvm-libs:linea:core:domain-models' include 'jvm-libs:linea:core:metrics' @@ -41,11 +42,11 @@ include 'coordinator:ethereum:finalization-monitor' include 'coordinator:ethereum:blob-submitter' include 'coordinator:ethereum:message-anchoring' include 'coordinator:ethereum:test-utils' -include 'coordinator:persistence:db' include 'coordinator:persistence:blob' include 'coordinator:persistence:aggregation' include 'coordinator:persistence:batch' include 'coordinator:persistence:feehistory' +include 'coordinator:persistence:db-common' include 'coordinator:ethereum:gas-pricing' include 'coordinator:ethereum:gas-pricing:static-cap' include 'coordinator:ethereum:gas-pricing:dynamic-cap' @@ -56,3 +57,6 @@ include 'traces-api-facade:app' include 'traces-api-facade:core' include 'traces-api-facade:conflation' include 'transaction-decoder-tool' +include 'transaction-exclusion-api:app' +include 'transaction-exclusion-api:core' +include 'transaction-exclusion-api:persistence:rejectedtransaction' diff --git a/transaction-exclusion-api/Dockerfile b/transaction-exclusion-api/Dockerfile new file mode 100644 index 00000000..7eac8059 --- /dev/null +++ b/transaction-exclusion-api/Dockerfile @@ -0,0 +1,33 @@ +# BUILDER image +FROM eclipse-temurin:21-jre-alpine AS builder +WORKDIR /opt/consensys/linea/transaction-exclusion-api + +RUN apk add --no-cache unzip + +# copy application +COPY --from=zip ./transaction-exclusion-api.zip libs/ +RUN unzip libs/transaction-exclusion-api.zip -d libs/ \ + && mv libs/transaction-exclusion-api/lib/** libs/ \ + && rm -R libs/transaction-exclusion-api/ + +# FINAL image +FROM eclipse-temurin:21-jre-alpine +WORKDIR /opt/consensys/linea/transaction-exclusion-api/ + +RUN mkdir -p /opt/consensys/linea/transaction-exclusion-api/logs/ + +COPY --from=builder /opt/consensys/linea/transaction-exclusion-api/libs libs/ + +# Build-time metadata as defined at http://label-schema.org +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="transaction-exclusion-api" \ + org.label-schema.description="Linea Transaction Exclusion API" \ + org.label-schema.url="https://consensys.io/" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.vcs-url="https://github.com/ConsenSys/linea-monorepo" \ + org.label-schema.vendor="ConsenSys" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" diff --git a/transaction-exclusion-api/README.md b/transaction-exclusion-api/README.md new file mode 100644 index 00000000..8ca89b01 --- /dev/null +++ b/transaction-exclusion-api/README.md @@ -0,0 +1,163 @@ +# Transaction Exclusion API Service +This micro-service will receive the transactions rejected by sequencer or other Linea Besu nodes, +persist them into a local database and expose a JSON-RPC v2 API to allow Linea users to query +why their transactions were not included. + +## V1 API Methods +### linea_saveRejectedTransactionV1 +```bash +curl -H 'content-type:application/json' --data '{ + "id": "1", + "jsonrpc": "2.0", + "method": "linea_saveRejectedTransactionV1", + "params": { + "txRejectionStage": "SEQUENCER", + "timestamp": "2024-08-22T09:18:51Z", + "blockNumber": 12345, + "transactionRLP": "0x02f8388204d2648203e88203e88203e8941195cf65f83b3a5768f3c496d3a05ad6412c64b38203e88c666d93e9cc5f73748162cea9c0017b8201c8", + "reasonMessage": "Transaction line count for module ADD=402 is above the limit 70", + "overflows": [ + { + "module": "ADD", + "count": 402, + "limit": 70 + }, + { + "module": "MUL", + "count": 587, + "limit": 400 + } + ] + } +}' http://127.0.0.1:8082 +``` + + +### linea_getTransactionExclusionStatusV1 +```bash +curl -H 'content-type:application/json' --data '{ + "jsonrpc": "2.0", + "id": "53", + "method": "linea_getTransactionExclusionStatusV1", + "params": [ + "0xf5bf951edfefbaa6d9ed78c88942147cf98c8ef1f3d3416f99d2534675096569" + ] +}' http://127.0.0.1:8082 +``` + + +### linea_saveRejectedTransactionV1 Response Examples: +The rejected transaction was successfully saved: +```json +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "status": "SAVED", + "txHash": "0x526e56101cf39c1e717cef9cedf6fdddb42684711abda35bae51136dbb350ad7" + } +} +``` +The rejected transaction with the same tx hash and reason message was saved before: +```json +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "status": "DUPLICATE_ALREADY_SAVED_BEFORE", + "txHash": "0x526e56101cf39c1e717cef9cedf6fdddb42684711abda35bae51136dbb350ad7" + } +} +``` +Invalid request params errors: +```json +{ + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32602, + "message": "Missing [timestamp] from the given request params" + } +} +``` +```json +{ + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32602, + "message": "Timestamp is not in ISO-8601: Text '2024-09-08T09:23:56Zdd' could not be parsed, unparsed text found at index 20" + } +} +``` +Other error: +```json +{ + "jsonrpc": "2.0", + "id": 1, + "error": { + "code": -32000, + "message": "Server error", + "data": "Database connection refused: localhost/127.0.0.1:5432" + } +} +``` + +### linea_getTransactionExclusionStatusV1 Response Examples: +The rejected transaction was successfully found: +```json +{ + "jsonrpc": "2.0", + "id": 53, + "result": { + "txHash": "0x526e56101cf39c1e717cef9cedf6fdddb42684711abda35bae51136dbb350ad7", + "from": "0x4d144d7b9c96b26361d6ac74dd1d8267edca4fc2", + "nonce": "0x64", + "txRejectionStage": "SEQUENCER", + "reasonMessage": "Transaction line count for module ADD=402 is above the limit 70", + "blockNumber": "0x3039", + "timestamp": "2024-08-22T09:18:51Z" + } +} +``` +The rejected transaction was not found: +```json +{ + "jsonrpc": "2.0", + "id": 53, + "result": null +} +``` +Invalid request params errors: +```json +{ + "jsonrpc": "2.0", + "id": 53, + "error": { + "code": -32602, + "message": "Hex string of transaction hash cannot be parsed: expected to have 32 bytes, but got 33" + } +} +``` +```json +{ + "jsonrpc": "2.0", + "id": 53, + "error": { + "code": -32602, + "message": "Hex string of transaction hash cannot be parsed: For input string: \"tt\" under radix 16" + } +} +``` +Other error: +```json +{ + "jsonrpc": "2.0", + "id": 53, + "error": { + "code": -32000, + "message": "Server error", + "data": "Database connection refused: localhost/127.0.0.1:5432" + } +} +``` diff --git a/transaction-exclusion-api/app/build.gradle b/transaction-exclusion-api/app/build.gradle new file mode 100644 index 00000000..f7fa9c71 --- /dev/null +++ b/transaction-exclusion-api/app/build.gradle @@ -0,0 +1,112 @@ +plugins { + id 'net.consensys.zkevm.kotlin-application-conventions' +} + +dependencies { + implementation project(':transaction-exclusion-api:core') + implementation project(':transaction-exclusion-api:persistence:rejectedtransaction') + implementation project(':jvm-libs:generic:extensions:futures') + implementation project(':jvm-libs:generic:extensions:kotlin') + implementation project(':jvm-libs:generic:json-rpc') + implementation project(':jvm-libs:generic:persistence:db') + implementation project(':jvm-libs:generic:vertx-helper') + implementation project(':jvm-libs:linea:core:long-running-service') + implementation project(':jvm-libs:linea:core:metrics') + implementation project(':jvm-libs:linea:metrics:micrometer') + + implementation "com.github.ben-manes.caffeine:caffeine:${libs.versions.caffeine.get()}" + implementation "io.vertx:vertx-core" + implementation "io.vertx:vertx-web" + implementation "io.vertx:vertx-health-check" + implementation "io.vertx:vertx-lang-kotlin" + implementation "io.vertx:vertx-config" + implementation "io.vertx:vertx-micrometer-metrics" + implementation "info.picocli:picocli:${libs.versions.picoli.get()}" + implementation "com.sksamuel.hoplite:hoplite-core:${libs.versions.hoplite.get()}" + implementation "com.sksamuel.hoplite:hoplite-toml:${libs.versions.hoplite.get()}" + implementation "io.micrometer:micrometer-registry-prometheus:${libs.versions.micrometer.get()}" + implementation "com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}" + implementation "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}" + implementation "com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}" + + implementation "org.hyperledger.besu:besu-datatypes:${libs.versions.besu.get()}" + implementation "org.hyperledger.besu:evm:${libs.versions.besu.get()}" + implementation "org.hyperledger.besu.internal:rlp:${libs.versions.besu.get()}" + implementation "org.hyperledger.besu.internal:core:${libs.versions.besu.get()}" + implementation "org.hyperledger.besu.internal:crypto:${libs.versions.besu.get()}" + + api("io.netty:netty-transport-native-epoll:${libs.versions.netty.get()}:linux-x86_64") { + because "It enables native transport for Linux." + // Note that its version should match netty version used in Vertx + } + api("io.netty:netty-transport-native-kqueue:${libs.versions.netty.get()}:osx-x86_64") { + because "It enables native transport for Mac OSX." + // Note that its version should match netty version used in Vertx + } + + testImplementation "io.vertx:vertx-junit5" + testImplementation "io.rest-assured:rest-assured:${libs.versions.restassured.get()}" + testImplementation "io.rest-assured:json-schema-validator:${libs.versions.restassured.get()}" + testImplementation "net.javacrumbs.json-unit:json-unit-assertj:${libs.versions.jsonUnit.get()}" + testImplementation testFixtures(project(":jvm-libs:generic:extensions:kotlin")) + testImplementation testFixtures(project(":jvm-libs:generic:persistence:db")) + testImplementation testFixtures(project(":transaction-exclusion-api:core")) +} + +application { + mainClass = 'net.consensys.linea.transactionexclusion.app.TransactionExclusionAppMain' +} + +jar { + archiveBaseName = 'transaction-exclusion-api' + manifest { + attributes( + 'Class-Path': configurations.runtimeClasspath.collect { it.getName() }.findAll {it.endsWith('jar') }.join(' '), + 'Main-Class': 'net.consensys.linea.transactionexclusion.app.TransactionExclusionAppMain', + 'Multi-Release': 'true' + ) + } +} + +distributions { + main { + distributionBaseName = 'transaction-exclusion-api' + } +} + +run { + workingDir = rootProject.projectDir + jvmArgs = [ + "-Dvertx.configurationFile=config/transaction-exclusion-api/vertx.json", + "-Dlog4j2.configurationFile=config/transaction-exclusion-api/log4j2-dev.xml" + ] + System.properties.entrySet() + .findAll { it.key.startsWith("config") } + .collect { "-D${it.key}=${it.value}" } + args = ["config/transaction-exclusion-api/transaction-exclusion-app-docker.config.toml", + "config/transaction-exclusion-api/transaction-exclusion-app-local-dev.config.overrides.toml"] +} + +sourceSets { + integrationTest { + kotlin { + compileClasspath += main.output + runtimeClasspath += main.output + } + compileClasspath += sourceSets.main.output + sourceSets.main.compileClasspath + sourceSets.test.compileClasspath + runtimeClasspath += sourceSets.main.output + sourceSets.main.runtimeClasspath + sourceSets.test.runtimeClasspath + } +} + +task integrationTest(type: Test) { + test -> + systemProperty "vertx.configurationFile", "vertx-options.json" + + description = "Runs integration tests." + group = "verification" + useJUnitPlatform() + + classpath = sourceSets.integrationTest.runtimeClasspath + testClassesDirs = sourceSets.integrationTest.output.classesDirs + + dependsOn(":localStackComposeUp") +} diff --git a/transaction-exclusion-api/app/src/integrationTest/kotlin/net/consensys/linea/transactionexclusion/TransactionExclusionAppTest.kt b/transaction-exclusion-api/app/src/integrationTest/kotlin/net/consensys/linea/transactionexclusion/TransactionExclusionAppTest.kt new file mode 100644 index 00000000..82bbc70e --- /dev/null +++ b/transaction-exclusion-api/app/src/integrationTest/kotlin/net/consensys/linea/transactionexclusion/TransactionExclusionAppTest.kt @@ -0,0 +1,291 @@ +package net.consensys.linea.transactionexclusion + +import com.sksamuel.hoplite.Masked +import io.restassured.RestAssured +import io.restassured.builder.RequestSpecBuilder +import io.restassured.http.ContentType +import io.restassured.specification.RequestSpecification +import io.vertx.junit5.VertxExtension +import kotlinx.datetime.Clock +import net.consensys.encodeHex +import net.consensys.linea.async.get +import net.consensys.linea.transactionexclusion.app.AppConfig +import net.consensys.linea.transactionexclusion.app.DatabaseConfig +import net.consensys.linea.transactionexclusion.app.DbCleanupConfig +import net.consensys.linea.transactionexclusion.app.DbConnectionConfig +import net.consensys.linea.transactionexclusion.app.PersistenceRetryConfig +import net.consensys.linea.transactionexclusion.app.TransactionExclusionApp +import net.consensys.linea.transactionexclusion.app.api.ApiConfig +import net.consensys.trimToMillisecondPrecision +import net.consensys.zkevm.persistence.db.DbHelper +import net.consensys.zkevm.persistence.db.test.CleanDbTestSuiteParallel +import net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import java.time.Duration +import kotlin.random.Random + +@ExtendWith(VertxExtension::class) +class TransactionExclusionAppTest : CleanDbTestSuiteParallel() { + init { + target = "1" + } + + override var databaseName = DbHelper.generateUniqueDbName("tx-exclusion-api-app-tests") + + private val dbConfig = DatabaseConfig( + read = DbConnectionConfig( + host = "localhost", + port = 5432, + username = "postgres", + password = Masked("postgres") + ), + write = DbConnectionConfig( + host = "localhost", + port = 5432, + username = "postgres", + password = Masked("postgres") + ), + cleanup = DbCleanupConfig( + pollingInterval = Duration.parse("PT60S"), + storagePeriod = Duration.parse("P7D") + ), + persistenceRetry = PersistenceRetryConfig( + backoffDelay = Duration.parse("PT5S"), + timeout = Duration.parse("PT20S") + ), + schema = databaseName + ) + + private lateinit var requestSpecification: RequestSpecification + private lateinit var app: TransactionExclusionApp + + @BeforeEach() + fun beforeEach() { + app = TransactionExclusionApp( + config = AppConfig( + api = ApiConfig( + port = 0, // port will be assigned under os + observabilityPort = 0, // port will be assigned under os + numberOfVerticles = 1 + ), + database = dbConfig, + dataQueryableWindowSinceRejectedTimestamp = Duration.parse("P7D") + ) + ) + app.start().get() + + requestSpecification = RequestSpecBuilder() + .setBaseUri("http://localhost:${app.apiBindedPort}/") + .build() + } + + @AfterEach + fun afterEach() { + app.stop().get() + } + + private fun makeRequestJsonResponse(request: String): String { + return RestAssured.given() + .spec(requestSpecification) + .accept(ContentType.JSON) + .body(request) + .`when`() + .post("/") + .then() + .statusCode(200) + .contentType("application/json") + .extract() + .asString() + } + + private fun saveFirstRejectedTransaction() { + val saveTxJonRequest = """{ + "jsonrpc": "2.0", + "id": 123, + "method": "linea_saveRejectedTransactionV1", + "params": { + "txRejectionStage": "P2P", + "timestamp": "${Clock.System.now().trimToMillisecondPrecision()}", + "transactionRLP": "0x02f8388204d2648203e88203e88203e8941195cf65f83b3a5768f3c496d3a05ad6412c64b38203e88c666d93e9cc5f73748162cea9c0017b8201c8", + "reasonMessage": "Transaction line count for module ADD=402 is above the limit 70", + "overflows": [ + { "module": "ADD", "count": 402, "limit": 70 }, + { "module": "MUL", "count": 587, "limit": 401 } + ] + } + } + """.trimIndent() + + // Check the save response and ensure the rejected txn was saved + assertThatJson(makeRequestJsonResponse(saveTxJonRequest)) + .isEqualTo( + """{ + "jsonrpc": "2.0", + "id": 123, + "result": {"status":"SAVED","txHash":"0x526e56101cf39c1e717cef9cedf6fdddb42684711abda35bae51136dbb350ad7"} + }""" + ) + } + + @Test + fun `Should save the rejected tx from P2P and then from SEQUENCER with same txHash but different reason message`() { + // Save the first rejected tx from P2P without rejected block number + saveFirstRejectedTransaction() + + // Save the rejected tx from SEQUENCER with rejected block number and different + // rejected reason and a more recent rejected timestamp + val rejectionTimeStamp = Clock.System.now() + .trimToMillisecondPrecision() + .toString() + + val saveTxJonRequest = """{ + "jsonrpc": "2.0", + "id": 124, + "method": "linea_saveRejectedTransactionV1", + "params": [{ + "txRejectionStage": "SEQUENCER", + "timestamp": "$rejectionTimeStamp", + "transactionRLP": "0x02f8388204d2648203e88203e88203e8941195cf65f83b3a5768f3c496d3a05ad6412c64b38203e88c666d93e9cc5f73748162cea9c0017b8201c8", + "blockNumber": "10000", + "reasonMessage": "Transaction line count for module ADD=402 is above the limit 70 (from Sequencer)", + "overflows": [ + { "module": "ADD", "count": 402, "limit": 70 }, + { "module": "MUL", "count": 587, "limit": 401 } + ] + }] + } + """.trimIndent() + + // Check the save response and ensure the rejected txn was saved + assertThatJson(makeRequestJsonResponse(saveTxJonRequest)) + .isEqualTo( + """{ + "jsonrpc": "2.0", + "id": 124, + "result": {"status":"SAVED","txHash":"0x526e56101cf39c1e717cef9cedf6fdddb42684711abda35bae51136dbb350ad7"} + }""" + ) + + // Send the get request for the rejected transaction + val getTxJsonRequest = """{ + "jsonrpc": "2.0", + "id": 125, + "method": "linea_getTransactionExclusionStatusV1", + "params": ["0x526e56101cf39c1e717cef9cedf6fdddb42684711abda35bae51136dbb350ad7"] + } + """.trimIndent() + + // Check the get response is corresponding to the rejected txn from SEQUENCER + assertThatJson(makeRequestJsonResponse(getTxJsonRequest)) + .isEqualTo( + """{ + "jsonrpc": "2.0", + "id": 125, + "result": { + "txHash": "0x526e56101cf39c1e717cef9cedf6fdddb42684711abda35bae51136dbb350ad7", + "from": "0x4d144d7b9c96b26361d6ac74dd1d8267edca4fc2", + "nonce": "0x64", + "txRejectionStage": "SEQUENCER", + "reasonMessage": "Transaction line count for module ADD=402 is above the limit 70 (from Sequencer)", + "timestamp": "$rejectionTimeStamp", + "blockNumber": "0x2710" + } + }""" + ) + } + + @Test + fun `Should return DUPLICATE_ALREADY_SAVED_BEFORE when saving rejected tx with same txHash and reason message`() { + // Save the first rejected tx from P2P without rejected block number + saveFirstRejectedTransaction() + + // Save the same rejected tx from SEQUENCER with rejected block number and a more recent rejected timestamp + val rejectionTimeStamp = Clock.System.now() + .trimToMillisecondPrecision() + .toString() + + val saveTxJonRequest = """{ + "jsonrpc": "2.0", + "id": 124, + "method": "linea_saveRejectedTransactionV1", + "params": [{ + "txRejectionStage": "SEQUENCER", + "timestamp": "$rejectionTimeStamp", + "transactionRLP": "0x02f8388204d2648203e88203e88203e8941195cf65f83b3a5768f3c496d3a05ad6412c64b38203e88c666d93e9cc5f73748162cea9c0017b8201c8", + "blockNumber": "10000", + "reasonMessage": "Transaction line count for module ADD=402 is above the limit 70", + "overflows": [ + { "module": "ADD", "count": 402, "limit": 70 }, + { "module": "MUL", "count": 587, "limit": 401 } + ] + }] + } + """.trimIndent() + + // Check the save response and ensure the status is "duplicated already saved before" + assertThatJson(makeRequestJsonResponse(saveTxJonRequest)) + .isEqualTo( + """{ + "jsonrpc": "2.0", + "id": 124, + "result": {"status":"DUPLICATE_ALREADY_SAVED_BEFORE","txHash":"0x526e56101cf39c1e717cef9cedf6fdddb42684711abda35bae51136dbb350ad7"} + }""" + ) + } + + @Test + fun `Should return result as null when getting the rejected tx with random transaction hash`() { + // Save the first rejected tx from P2P without rejected block number + saveFirstRejectedTransaction() + + // Send the get request with a random txn hash + val getTxJsonRequest = """{ + "jsonrpc": "2.0", + "id": 124, + "method": "linea_getTransactionExclusionStatusV1", + "params": ["${Random.nextBytes(32).encodeHex()}"] + } + """.trimIndent() + + // Check the get response and ensure the result is null + assertThatJson(makeRequestJsonResponse(getTxJsonRequest)) + .isEqualTo( + """{ + "jsonrpc": "2.0", + "id": 124, + "result": null + }""" + ) + } + + @Test + fun `when transaction request is invalid shall return error`() { + val saveTxJonRequest = """{ + "jsonrpc": "2.0", + "id": 123, + "method": "linea_saveRejectedTransactionV1", + "params": [{ + "txRejectionStage": "SEQUENCER", + "transactionRLP": "0x02f8388204d2648203e88203e88203e8941195cf65f83b3a5768f3c496d3a05ad6412c64b38203e88c666d93e9cc5f73748162cea9c0017b8201c8", + "blockNumber": "10000", + "reasonMessage": "Transaction line count for module ADD=402 is above the limit 70" + }] + } + """.trimIndent() + + assertThatJson(makeRequestJsonResponse(saveTxJonRequest)) + .isEqualTo( + """{ + "jsonrpc": "2.0", + "id": 123, + "error": { + "code": -32602, + "message": "Missing [timestamp,overflows] from the given request params" + } + }""" + ) + } +} diff --git a/transaction-exclusion-api/app/src/integrationTest/resources/log4j2.xml b/transaction-exclusion-api/app/src/integrationTest/resources/log4j2.xml new file mode 100644 index 00000000..63c9742f --- /dev/null +++ b/transaction-exclusion-api/app/src/integrationTest/resources/log4j2.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/transaction-exclusion-api/app/src/integrationTest/resources/vertx-options.json b/transaction-exclusion-api/app/src/integrationTest/resources/vertx-options.json new file mode 100644 index 00000000..bc9f8a2a --- /dev/null +++ b/transaction-exclusion-api/app/src/integrationTest/resources/vertx-options.json @@ -0,0 +1,10 @@ +{ + "metricsOptions": { + "enabled": true, + "jvmMetricsEnabled": true, + "prometheusOptions": { + "publishQuantiles": true, + "enabled": true + } + } +} diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/TransactionExclusionApp.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/TransactionExclusionApp.kt new file mode 100644 index 00000000..d58f0bfe --- /dev/null +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/TransactionExclusionApp.kt @@ -0,0 +1,184 @@ +package net.consensys.linea.transactionexclusion.app + +import com.sksamuel.hoplite.Masked +import io.micrometer.core.instrument.MeterRegistry +import io.vertx.core.Future +import io.vertx.core.Vertx +import io.vertx.micrometer.backends.BackendRegistries +import io.vertx.sqlclient.SqlClient +import net.consensys.linea.async.toSafeFuture +import net.consensys.linea.async.toVertxFuture +import net.consensys.linea.metrics.micrometer.MicrometerMetricsFacade +import net.consensys.linea.transactionexclusion.TransactionExclusionServiceV1 +import net.consensys.linea.transactionexclusion.app.api.Api +import net.consensys.linea.transactionexclusion.app.api.ApiConfig +import net.consensys.linea.transactionexclusion.service.RejectedTransactionCleanupService +import net.consensys.linea.transactionexclusion.service.TransactionExclusionServiceV1Impl +import net.consensys.linea.vertx.loadVertxConfig +import net.consensys.zkevm.persistence.dao.rejectedtransaction.RejectedTransactionsDao +import net.consensys.zkevm.persistence.dao.rejectedtransaction.RejectedTransactionsPostgresDao +import net.consensys.zkevm.persistence.dao.rejectedtransaction.RetryingRejectedTransactionsPostgresDao +import net.consensys.zkevm.persistence.db.Db +import net.consensys.zkevm.persistence.db.PersistenceRetryer +import org.apache.logging.log4j.LogManager +import java.time.Duration +import kotlin.time.Duration.Companion.seconds +import kotlin.time.toJavaDuration +import kotlin.time.toKotlinDuration + +data class DbConnectionConfig( + val host: String, + val port: Int, + val username: String, + val password: Masked +) + +data class DbCleanupConfig( + val pollingInterval: Duration, + val storagePeriod: Duration +) + +data class DatabaseConfig( + val read: DbConnectionConfig, + val write: DbConnectionConfig, + val cleanup: DbCleanupConfig, + val persistenceRetry: PersistenceRetryConfig, + val schema: String = "linea_transaction_exclusion", + val readPoolSize: Int = 10, + val readPipeliningLimit: Int = 10, + val transactionalPoolSize: Int = 10 +) + +data class AppConfig( + val api: ApiConfig, + val database: DatabaseConfig, + val dataQueryableWindowSinceRejectedTimestamp: Duration +) + +data class PersistenceRetryConfig( + val maxRetries: Int? = null, + val backoffDelay: Duration = 1.seconds.toJavaDuration(), + val timeout: Duration? = 20.seconds.toJavaDuration() +) + +class TransactionExclusionApp(config: AppConfig) { + private val log = LogManager.getLogger(TransactionExclusionApp::class.java) + private val meterRegistry: MeterRegistry + private val vertx: Vertx + private val api: Api + private val sqlReadClient: SqlClient + private val sqlWriteClient: SqlClient + private val rejectedTransactionsRepository: RejectedTransactionsDao + private val transactionExclusionService: TransactionExclusionServiceV1 + private val rejectedTransactionCleanupService: RejectedTransactionCleanupService + private val micrometerMetricsFacade: MicrometerMetricsFacade + val apiBindedPort: Int + get() = api.bindedPort + + init { + log.debug("System properties: {}", System.getProperties()) + val vertxConfig = loadVertxConfig() + log.debug("Vertx full configs: {}", vertxConfig) + log.info("App configs: {}", config) + this.vertx = Vertx.vertx(vertxConfig) + this.meterRegistry = BackendRegistries.getDefaultNow() + this.micrometerMetricsFacade = MicrometerMetricsFacade(meterRegistry, "linea") + this.sqlReadClient = initDb( + connectionConfig = config.database.read, + schema = config.database.schema, + transactionalPoolSize = config.database.transactionalPoolSize, + readPipeliningLimit = config.database.readPipeliningLimit, + skipMigration = true + ) + this.sqlWriteClient = initDb( + connectionConfig = config.database.write, + schema = config.database.schema, + transactionalPoolSize = config.database.transactionalPoolSize, + readPipeliningLimit = config.database.readPipeliningLimit + ) + this.rejectedTransactionsRepository = RetryingRejectedTransactionsPostgresDao( + delegate = RejectedTransactionsPostgresDao( + readConnection = this.sqlReadClient, + writeConnection = this.sqlWriteClient + ), + persistenceRetryer = PersistenceRetryer( + vertx = vertx, + config = PersistenceRetryer.Config( + backoffDelay = config.database.persistenceRetry.backoffDelay.toKotlinDuration(), + maxRetries = config.database.persistenceRetry.maxRetries, + timeout = config.database.persistenceRetry.timeout?.toKotlinDuration() + ) + ) + ) + this.transactionExclusionService = TransactionExclusionServiceV1Impl( + config = TransactionExclusionServiceV1Impl.Config( + config.dataQueryableWindowSinceRejectedTimestamp.toKotlinDuration() + ), + repository = this.rejectedTransactionsRepository, + metricsFacade = this.micrometerMetricsFacade + ) + this.rejectedTransactionCleanupService = RejectedTransactionCleanupService( + config = RejectedTransactionCleanupService.Config( + pollingInterval = config.database.cleanup.pollingInterval.toKotlinDuration(), + storagePeriod = config.database.cleanup.storagePeriod.toKotlinDuration() + ), + repository = this.rejectedTransactionsRepository, + vertx = this.vertx + ) + this.api = + Api( + configs = config.api, + vertx = vertx, + meterRegistry = meterRegistry, + transactionExclusionService = transactionExclusionService + ) + } + + fun start(): Future<*> { + log.info("Starting up app..") + return api.start().toSafeFuture() + .thenCompose { rejectedTransactionCleanupService.start() } + .thenPeek { + log.info("App successfully started") + }.toVertxFuture() + } + + fun stop(): Future<*> { + log.info("Shooting down app..") + return api.stop().toSafeFuture() + .thenCompose { rejectedTransactionCleanupService.stop() } + .thenPeek { + log.info("App successfully stopped") + }.toVertxFuture() + } + + private fun initDb( + connectionConfig: DbConnectionConfig, + schema: String, + transactionalPoolSize: Int, + readPipeliningLimit: Int, + skipMigration: Boolean = false + ): SqlClient { + val dbVersion = "1" + if (!skipMigration) { + Db.applyDbMigrations( + host = connectionConfig.host, + port = connectionConfig.port, + database = schema, + target = dbVersion, + username = connectionConfig.username, + password = connectionConfig.password.value + ) + } + return Db.vertxSqlClient( + vertx = vertx, + host = connectionConfig.host, + port = connectionConfig.port, + database = schema, + username = connectionConfig.username, + password = connectionConfig.password.value, + maxPoolSize = transactionalPoolSize, + pipeliningLimit = readPipeliningLimit + ) + } +} diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/TransactionExclusionAppCli.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/TransactionExclusionAppCli.kt new file mode 100644 index 00000000..ce0f08b1 --- /dev/null +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/TransactionExclusionAppCli.kt @@ -0,0 +1,114 @@ +package net.consensys.linea.transactionexclusion.app + +import com.sksamuel.hoplite.ConfigFailure +import com.sksamuel.hoplite.ConfigLoaderBuilder +import com.sksamuel.hoplite.addFileSource +import com.sksamuel.hoplite.fp.Validated +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import picocli.CommandLine.Command +import picocli.CommandLine.Parameters +import java.io.File +import java.io.PrintWriter +import java.nio.charset.Charset +import java.util.concurrent.Callable + +@Command( + name = TransactionExclusionAppCli.COMMAND_NAME, + showDefaultValues = true, + abbreviateSynopsis = true, + description = ["Runs Transaction Exclusion API Service"], + version = ["0.0.1"], + synopsisHeading = "%n", + descriptionHeading = "%nDescription:%n%n", + optionListHeading = "%nOptions:%n", + footerHeading = "%n" +) +class TransactionExclusionAppCli +internal constructor(private val errorWriter: PrintWriter, private val startAction: StartAction) : + Callable { + @Parameters(paramLabel = "CONFIG.toml", description = ["Configuration files"]) + private val configFiles: List? = null + + override fun call(): Int { + return try { + if (configFiles == null) { + errorWriter.println("Please provide a configuration file!") + printUsage(errorWriter) + return 1 + } + for (configFile in configFiles) { + if (!canReadFile(configFile)) { + return 1 + } + } + val configs: Validated = configs(configFiles) + if (configs.isInvalid()) { + errorWriter.println(configs.getInvalidUnsafe().description()) + return 1 + } + startAction.start(configs.getUnsafe()) + 0 + } catch (e: Exception) { + reportUserError(e) + 1 + } + } + + private fun canReadFile(file: File): Boolean { + if (!file.canRead()) { + errorWriter.println("Cannot read configuration file '${file.absolutePath}'") + return false + } + return true + } + + fun configs(configFiles: List): Validated { + val confBuilder: ConfigLoaderBuilder = ConfigLoaderBuilder.Companion.empty().addDefaults() + for (i in configFiles.indices.reversed()) { + // files must be added in reverse order for overriding + + // files must be added in reverse order for overriding + confBuilder.addFileSource(configFiles[i], false) + } + val config: Validated = + confBuilder.build().loadConfig(emptyList()) + + return config + } + + fun reportUserError(ex: Throwable) { + logger.fatal(ex.message, ex) + errorWriter.println(ex.message) + printUsage(errorWriter) + } + + private fun printUsage(outputWriter: PrintWriter) { + outputWriter.println() + outputWriter.println("To display full help:") + outputWriter.println(COMMAND_NAME + " --help") + } + + /** + * Not using a static field for this log instance because some code in this class executes prior + * to the logging configuration being applied so it's not always safe to use the logger. + * + * Where this is used we also ensure the messages are printed to the error writer so they will be + * printed even if logging is not yet configured. + * + * @return the logger for this class + */ + private val logger: Logger = LogManager.getLogger() + + fun interface StartAction { + fun start(configs: AppConfig) + } + + companion object { + const val COMMAND_NAME = "transaction-exclusion" + fun withAction(startAction: StartAction): TransactionExclusionAppCli { + val errorWriter = PrintWriter(System.err, true, Charset.defaultCharset()) + return TransactionExclusionAppCli(errorWriter, startAction) + } + } +} diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/TransactionExclusionAppMain.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/TransactionExclusionAppMain.kt new file mode 100644 index 00000000..5668e163 --- /dev/null +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/TransactionExclusionAppMain.kt @@ -0,0 +1,40 @@ +package net.consensys.linea.transactionexclusion.app + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.core.LoggerContext +import org.apache.logging.log4j.core.config.Configurator +import picocli.CommandLine +import kotlin.system.exitProcess + +class TransactionExclusionAppMain { + companion object { + private val log = LogManager.getLogger(TransactionExclusionAppMain::class) + + @JvmStatic + fun main(args: Array) { + val cmd = CommandLine(TransactionExclusionAppCli.withAction(::startApp)) + cmd.execute(*args) + } + + private fun startApp(configs: AppConfig) { + try { + val app = TransactionExclusionApp(configs) + Runtime.getRuntime() + .addShutdownHook( + Thread { + app.stop() + if (LogManager.getContext() is LoggerContext) { + // Disable log4j auto shutdown hook is not used otherwise + // Messages in App.stop won't appear in the logs + Configurator.shutdown(LogManager.getContext() as LoggerContext) + } + } + ) + app.start() + } catch (t: Throwable) { + log.error("Startup failure: ", t) + exitProcess(1) + } + } + } +} diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/Api.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/Api.kt new file mode 100644 index 00000000..13a9ed71 --- /dev/null +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/Api.kt @@ -0,0 +1,93 @@ +package net.consensys.linea.transactionexclusion.app.api + +import io.micrometer.core.instrument.MeterRegistry +import io.vertx.core.DeploymentOptions +import io.vertx.core.Future +import io.vertx.core.Vertx +import net.consensys.linea.jsonrpc.HttpRequestHandler +import net.consensys.linea.jsonrpc.JsonRpcMessageHandler +import net.consensys.linea.jsonrpc.JsonRpcMessageProcessor +import net.consensys.linea.jsonrpc.JsonRpcRequestRouter +import net.consensys.linea.transactionexclusion.TransactionExclusionServiceV1 +import net.consensys.linea.vertx.ObservabilityServer + +data class ApiConfig( + val port: Int = 0, + val observabilityPort: Int = 0, + val numberOfVerticles: Int = 0, + val path: String = "/" +) + +class Api( + private val configs: ApiConfig, + private val vertx: Vertx, + private val meterRegistry: MeterRegistry, + private val transactionExclusionService: TransactionExclusionServiceV1 +) { + private var jsonRpcServerId: String? = null + private var observabilityServerId: String? = null + private var serverPort: Int = -1 + val bindedPort: Int + get() = if (serverPort > 0) { + serverPort + } else { + throw IllegalStateException("Http server not started") + } + + fun start(): Future<*> { + val requestHandlersV1 = + mapOf( + ApiMethod.LINEA_SAVE_REJECTED_TRANSACTION_V1.method to + SaveRejectedTransactionRequestHandlerV1( + transactionExclusionService = transactionExclusionService + ), + ApiMethod.LINEA_GET_TRANSACTION_EXCLUSION_STATUS_V1.method to + GetTransactionExclusionStatusRequestHandlerV1( + transactionExclusionService = transactionExclusionService + ) + ) + + val messageHandler: JsonRpcMessageHandler = + JsonRpcMessageProcessor(JsonRpcRequestRouter(requestHandlersV1), meterRegistry) + + val numberOfVerticles: Int = + if (configs.numberOfVerticles > 0) { + configs.numberOfVerticles + } else { + Runtime.getRuntime().availableProcessors() + } + + val observabilityServer = + ObservabilityServer( + ObservabilityServer.Config( + "transaction-exclusion-api", + configs.observabilityPort + ) + ) + var httpServer: HttpJsonRpcServer? = null + return vertx + .deployVerticle( + { + HttpJsonRpcServer(configs.port.toUInt(), configs.path, HttpRequestHandler(messageHandler)) + .also { + httpServer = it + } + }, + DeploymentOptions().setInstances(numberOfVerticles) + ) + .compose { verticleId: String -> + jsonRpcServerId = verticleId + serverPort = httpServer!!.bindedPort + vertx.deployVerticle(observabilityServer).onSuccess { monitorVerticleId -> + this.observabilityServerId = monitorVerticleId + } + } + } + + fun stop(): Future<*> { + return Future.all( + this.jsonRpcServerId?.let { vertx.undeploy(it) } ?: Future.succeededFuture(null), + this.observabilityServerId?.let { vertx.undeploy(it) } ?: Future.succeededFuture(null) + ) + } +} diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/ApiMethod.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/ApiMethod.kt new file mode 100644 index 00000000..78e88e47 --- /dev/null +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/ApiMethod.kt @@ -0,0 +1,6 @@ +package net.consensys.linea.transactionexclusion.app.api + +enum class ApiMethod(val method: String) { + LINEA_SAVE_REJECTED_TRANSACTION_V1("linea_saveRejectedTransactionV1"), + LINEA_GET_TRANSACTION_EXCLUSION_STATUS_V1("linea_getTransactionExclusionStatusV1") +} diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/ArgumentParser.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/ArgumentParser.kt new file mode 100644 index 00000000..8e44eac6 --- /dev/null +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/ArgumentParser.kt @@ -0,0 +1,88 @@ +package net.consensys.linea.transactionexclusion.app.api + +import kotlinx.datetime.Instant +import net.consensys.assertIs32Bytes +import net.consensys.decodeHex +import net.consensys.linea.transactionexclusion.ModuleOverflow +import net.consensys.linea.transactionexclusion.RejectedTransaction +import net.consensys.linea.transactionexclusion.TransactionInfo +import net.consensys.linea.transactionexclusion.dto.ModuleOverflowJsonDto +import org.apache.tuweni.bytes.Bytes +import org.hyperledger.besu.ethereum.core.encoding.TransactionDecoder +import java.time.format.DateTimeFormatter +import java.time.format.DateTimeParseException + +object ArgumentParser { + fun getTransactionRLPInRawBytes(transactionRLP: String): ByteArray { + try { + return transactionRLP.decodeHex() + } catch (ex: Exception) { + throw IllegalArgumentException("Hex string of RLP-encoded transaction cannot be parsed: ${ex.message}") + } + } + + fun getTxHashInRawBytes(txHash: String): ByteArray { + try { + return txHash.decodeHex().assertIs32Bytes() + } catch (ex: Exception) { + throw IllegalArgumentException("Hex string of transaction hash cannot be parsed: ${ex.message}") + } + } + + fun getTransactionInfoFromRLP(rlp: ByteArray): TransactionInfo { + try { + return TransactionDecoder.decodeOpaqueBytes( + Bytes.wrap(rlp) + ).run { + TransactionInfo( + hash = this.hash.toArray(), + to = this.to.get().toArray(), + from = this.sender.toArray(), + nonce = this.nonce.toULong() + ) + } + } catch (ex: Exception) { + throw IllegalArgumentException("RLP-encoded transaction cannot be parsed: ${ex.message}") + } + } + + fun getOverflows(target: Any): List { + try { + return ModuleOverflowJsonDto.parseListFrom(target).map { it.toDomainObject() } + } catch (ex: Exception) { + throw IllegalArgumentException("Overflows cannot be parsed: ${ex.message}") + } + } + + fun getReasonMessage(reasonMessage: String): String { + if (reasonMessage.length > 256) { + throw IllegalArgumentException("Reason message should not be more than 256 characters: $reasonMessage") + } + return reasonMessage + } + + fun getBlockNumber(blockNumberStr: String?): ULong? { + try { + return blockNumberStr?.toULong() + } catch (ex: NumberFormatException) { + throw IllegalArgumentException("Block number cannot be parsed to an unsigned number: ${ex.message}") + } + } + + fun getTimestampFromISO8601(timestamp: String): Instant { + try { + DateTimeFormatter.ISO_DATE_TIME.parse(timestamp) + return Instant.parse(timestamp) + } catch (ex: DateTimeParseException) { + throw IllegalArgumentException("Timestamp is not in ISO-8601: ${ex.message}") + } + } + + fun getTxRejectionStage(txRejectionStage: String): RejectedTransaction.Stage { + try { + return RejectedTransaction.Stage.valueOf(txRejectionStage) + } catch (ex: IllegalArgumentException) { + throw IllegalArgumentException("Unsupported transaction rejection stage: $txRejectionStage") + } + } +} diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/HttpJsonRpcServer.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/HttpJsonRpcServer.kt new file mode 100644 index 00000000..a37143ae --- /dev/null +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/HttpJsonRpcServer.kt @@ -0,0 +1,53 @@ +package net.consensys.linea.transactionexclusion.app.api + +import io.vertx.core.AbstractVerticle +import io.vertx.core.AsyncResult +import io.vertx.core.Handler +import io.vertx.core.Promise +import io.vertx.core.http.HttpServer +import io.vertx.core.http.HttpServerOptions +import io.vertx.ext.web.Router +import io.vertx.ext.web.RoutingContext +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger + +class HttpJsonRpcServer( + private val port: UInt, + private val path: String, + private val requestHandler: Handler +) : AbstractVerticle() { + private val log: Logger = LogManager.getLogger(this.javaClass) + private lateinit var httpServer: HttpServer + val bindedPort: Int + get() = if (this::httpServer.isInitialized) { + httpServer.actualPort() + } else { + throw IllegalStateException("Http server not started") + } + + override fun start(startPromise: Promise) { + val options = HttpServerOptions().setPort(port.toInt()).setReusePort(true) + log.debug("Creating Http server on port {}", port) + httpServer = vertx.createHttpServer(options) + httpServer.requestHandler(buildRouter()) + httpServer.listen { res: AsyncResult -> + if (res.succeeded()) { + log.info("Http server started and listening on port {}", res.result().actualPort()) + startPromise.complete() + } else { + log.error("Creating Http server: {}", res.cause()) + startPromise.fail(res.cause()) + } + } + } + + private fun buildRouter(): Router { + val router = Router.router(vertx) + router.route(path).produces("application/json").handler(requestHandler) + return router + } + + override fun stop(endFuture: Promise) { + httpServer.close(endFuture) + } +} diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/RequestHandlersV1.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/RequestHandlersV1.kt new file mode 100644 index 00000000..c7eb4fb2 --- /dev/null +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/RequestHandlersV1.kt @@ -0,0 +1,211 @@ +package net.consensys.linea.transactionexclusion.app.api + +import com.github.michaelbull.result.Err +import com.github.michaelbull.result.Ok +import com.github.michaelbull.result.Result +import com.github.michaelbull.result.flatMap +import com.github.michaelbull.result.get +import com.github.michaelbull.result.map +import com.github.michaelbull.result.mapError +import io.vertx.core.Future +import io.vertx.core.json.JsonObject +import io.vertx.ext.auth.User +import net.consensys.encodeHex +import net.consensys.linea.async.toVertxFuture +import net.consensys.linea.jsonrpc.JsonRpcErrorResponse +import net.consensys.linea.jsonrpc.JsonRpcRequest +import net.consensys.linea.jsonrpc.JsonRpcRequestHandler +import net.consensys.linea.jsonrpc.JsonRpcRequestListParams +import net.consensys.linea.jsonrpc.JsonRpcRequestMapParams +import net.consensys.linea.jsonrpc.JsonRpcSuccessResponse +import net.consensys.linea.transactionexclusion.RejectedTransaction +import net.consensys.linea.transactionexclusion.TransactionExclusionServiceV1 +import net.consensys.linea.transactionexclusion.dto.RejectedTransactionJsonDto +import net.consensys.toHexString + +private fun validateIsMapOrListParams(request: JsonRpcRequest): Result { + if (request.params !is Map<*, *> && request.params !is List<*>) { + return Err( + JsonRpcErrorResponse.invalidParams( + request.id, + "params should be either an object or a list" + ) + ) + } + return Ok(request) +} + +private fun validateIsListParams(request: JsonRpcRequest): Result { + if (request.params !is List<*>) { + return Err( + JsonRpcErrorResponse.invalidParams( + request.id, + "params should be a list" + ) + ) + } + return Ok(request) +} + +class SaveRejectedTransactionRequestHandlerV1( + private val transactionExclusionService: TransactionExclusionServiceV1 +) : JsonRpcRequestHandler { + enum class RequestParams(val paramName: String) { + TX_REJECTION_STAGE("txRejectionStage"), + TIMESTAMP("timestamp"), + REASON_MESSAGE("reasonMessage"), + TRANSACTION_RLP("transactionRLP"), + BLOCK_NUMBER("blockNumber"), + OVERFLOWS("overflows") + } + + private fun validateMapParamsPresence(requestMapParams: Map<*, *>) { + RequestParams.entries + .filter { requestParam -> + requestParam != RequestParams.BLOCK_NUMBER && requestMapParams[requestParam.paramName] == null + } + .run { + if (this.isNotEmpty()) { + throw IllegalArgumentException( + "Missing ${this.joinToString(",", "[", "]") { it.paramName }} " + + "from the given request params" + ) + } + } + } + + private fun parseMapParamsToRejectedTransaction(requestMapParams: Map<*, *>): RejectedTransaction { + return validateMapParamsPresence(requestMapParams).run { + RejectedTransactionJsonDto.parseFrom(requestMapParams).toDomainObject() + } + } + + private fun parseListParamsToRejectedTransaction(requestListParams: List): RejectedTransaction { + if (requestListParams.size != 1) { + throw IllegalArgumentException( + "The given request params list should have one argument" + ) + } else if (requestListParams.first() !is Map<*, *>) { + throw IllegalArgumentException( + "The argument in the request params list should be an object" + ) + } + return parseMapParamsToRejectedTransaction(requestListParams[0] as Map<*, *>) + } + + override fun invoke( + user: User?, + request: JsonRpcRequest, + requestJson: JsonObject + ): Future> { + val rejectedTransaction = try { + val parsingResult = validateIsMapOrListParams(request).flatMap { validatedRequest -> + val parsedRejectedTransaction = + when (validatedRequest) { + is JsonRpcRequestMapParams -> parseMapParamsToRejectedTransaction(validatedRequest.params) + is JsonRpcRequestListParams -> parseListParamsToRejectedTransaction(validatedRequest.params) + else -> throw IllegalStateException( + "JsonRpcRequest should be as JsonRpcRequestMapParams or JsonRpcRequestListParams" + ) + } + Ok(parsedRejectedTransaction) + } + if (parsingResult is Err) { + return Future.succeededFuture(parsingResult) + } else { + parsingResult.get()!! + } + } catch (e: Exception) { + return Future.succeededFuture( + Err( + JsonRpcErrorResponse.invalidParams( + request.id, + e.message + ) + ) + ) + } + + return transactionExclusionService + .saveRejectedTransaction(rejectedTransaction) + .thenApply { result -> + result.map { + val rpcResult = + JsonObject() + .put("status", it.name) + .put("txHash", rejectedTransaction.transactionInfo.hash.encodeHex()) + JsonRpcSuccessResponse(request.id, rpcResult) + }.mapError { error -> + JsonRpcErrorResponse(request.id, jsonRpcError(error)) + } + }.toVertxFuture() + } +} + +class GetTransactionExclusionStatusRequestHandlerV1( + private val transactionExclusionService: TransactionExclusionServiceV1 +) : JsonRpcRequestHandler { + private fun parseListParamsToTxHash(requestListParams: List): ByteArray { + if (requestListParams.size != 1) { + throw IllegalArgumentException( + "The given request params list should have one argument" + ) + } + return ArgumentParser.getTxHashInRawBytes(requestListParams[0].toString()) + } + + override fun invoke( + user: User?, + request: JsonRpcRequest, + requestJson: JsonObject + ): Future> { + val txHash = try { + val parsingResult = validateIsListParams(request).flatMap { validatedRequest -> + val parsedTxHash = + when (validatedRequest) { + is JsonRpcRequestListParams -> parseListParamsToTxHash(validatedRequest.params) + else -> throw IllegalStateException("JsonRpcRequest should be as JsonRpcRequestListParams") + } + Ok(parsedTxHash) + } + if (parsingResult is Err) { + return Future.succeededFuture(parsingResult) + } else { + parsingResult.get()!! + } + } catch (e: Exception) { + return Future.succeededFuture( + Err( + JsonRpcErrorResponse.invalidParams( + request.id, + e.message + ) + ) + ) + } + + return transactionExclusionService + .getTransactionExclusionStatus(txHash) + .thenApply { result -> + result.map { + val rpcResult = if (it == null) { null } else { + JsonObject() + .put("txHash", it.transactionInfo.hash.encodeHex()) + .put("from", it.transactionInfo.from.encodeHex()) + .put("nonce", it.transactionInfo.nonce.toHexString()) + .put("txRejectionStage", it.txRejectionStage.name) + .put("reasonMessage", it.reasonMessage) + .put("timestamp", it.timestamp.toString()) + .also { jsonObject -> + if (it.blockNumber != null) { + jsonObject.put("blockNumber", it.blockNumber!!.toHexString()) + } + } + } + JsonRpcSuccessResponse(request.id, rpcResult) + }.mapError { error -> + JsonRpcErrorResponse(request.id, jsonRpcError(error)) + } + }.toVertxFuture() + } +} diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/TransactionExclusionErrorCodes.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/TransactionExclusionErrorCodes.kt new file mode 100644 index 00000000..c9baffaa --- /dev/null +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/TransactionExclusionErrorCodes.kt @@ -0,0 +1,21 @@ +package net.consensys.linea.transactionexclusion.app.api + +import net.consensys.linea.jsonrpc.JsonRpcError +import net.consensys.linea.transactionexclusion.ErrorType +import net.consensys.linea.transactionexclusion.TransactionExclusionError + +enum class TransactionExclusionErrorCodes(val code: Int, val message: String) { + // App/System/Server' error codes + SERVER_ERROR(-32000, "Server error"); + + fun toErrorObject(data: Any? = null): JsonRpcError { + return JsonRpcError(this.code, this.message, data) + } +} + +fun jsonRpcError(appError: TransactionExclusionError): JsonRpcError { + return when (appError.errorType) { + ErrorType.SERVER_ERROR -> + TransactionExclusionErrorCodes.SERVER_ERROR.toErrorObject(appError.errorDetail) + } +} diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/dto/RejectedTransactionJsonDto.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/dto/RejectedTransactionJsonDto.kt new file mode 100644 index 00000000..ecdbf9a3 --- /dev/null +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/dto/RejectedTransactionJsonDto.kt @@ -0,0 +1,92 @@ +package net.consensys.linea.transactionexclusion.dto + +import com.fasterxml.jackson.databind.ObjectMapper +import net.consensys.linea.transactionexclusion.ModuleOverflow +import net.consensys.linea.transactionexclusion.RejectedTransaction +import net.consensys.linea.transactionexclusion.app.api.ArgumentParser + +data class ModuleOverflowJsonDto( + val count: Long, + val limit: Long, + val module: String +) { + // Jackson ObjectMapper requires a default constructor + constructor() : this(0L, 0L, "") + + companion object { + fun parseListFrom(target: Any): List { + return ObjectMapper().readValue( + ObjectMapper().writeValueAsString(target), + Array::class.java + ).toList() + } + } + + fun toDomainObject(): ModuleOverflow { + return ModuleOverflow( + count = count, + limit = limit, + module = module + ) + } +} + +data class RejectedTransactionJsonDto( + val txRejectionStage: String, + val timestamp: String, + val blockNumber: String?, + val transactionRLP: String, + val reasonMessage: String, + val overflows: Any +) { + // Jackson ObjectMapper requires a default constructor + constructor() : this("", "", null, "", "", Any()) + + companion object { + fun parseFrom(target: Any): RejectedTransactionJsonDto { + return ObjectMapper().readValue( + ObjectMapper().writeValueAsString(target), + RejectedTransactionJsonDto::class.java + ) + } + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as RejectedTransactionJsonDto + + if (txRejectionStage != other.txRejectionStage) return false + if (timestamp != other.timestamp) return false + if (blockNumber != other.blockNumber) return false + if (transactionRLP != other.transactionRLP) return false + if (reasonMessage != other.reasonMessage) return false + return overflows == other.overflows + } + + override fun hashCode(): Int { + var result = txRejectionStage.hashCode() + result = 31 * result + timestamp.hashCode() + result = 31 * result + (blockNumber?.hashCode() ?: 0) + result = 31 * result + transactionRLP.hashCode() + result = 31 * result + reasonMessage.hashCode() + result = 31 * result + overflows.hashCode() + return result + } + + fun toDomainObject(): RejectedTransaction { + return ArgumentParser.getTransactionRLPInRawBytes(transactionRLP) + .let { parsedTransactionRLP -> + RejectedTransaction( + txRejectionStage = ArgumentParser.getTxRejectionStage(txRejectionStage), + timestamp = ArgumentParser.getTimestampFromISO8601(timestamp), + blockNumber = ArgumentParser.getBlockNumber(blockNumber), + transactionRLP = parsedTransactionRLP, + reasonMessage = ArgumentParser.getReasonMessage(reasonMessage), + overflows = ArgumentParser.getOverflows(overflows), + transactionInfo = ArgumentParser.getTransactionInfoFromRLP(parsedTransactionRLP) + ) + } + } +} diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/service/RejectedTransactionCleanupService.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/service/RejectedTransactionCleanupService.kt new file mode 100644 index 00000000..be86184f --- /dev/null +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/service/RejectedTransactionCleanupService.kt @@ -0,0 +1,45 @@ +package net.consensys.linea.transactionexclusion.service + +import io.vertx.core.Vertx +import kotlinx.datetime.Clock +import net.consensys.zkevm.PeriodicPollingService +import net.consensys.zkevm.persistence.dao.rejectedtransaction.RejectedTransactionsDao +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import tech.pegasys.teku.infrastructure.async.SafeFuture +import kotlin.time.Duration + +class RejectedTransactionCleanupService( + vertx: Vertx, + private val config: Config, + private val repository: RejectedTransactionsDao, + private val clock: Clock = Clock.System, + private val log: Logger = LogManager.getLogger(RejectedTransactionCleanupService::class.java) +) : PeriodicPollingService( + vertx = vertx, + pollingIntervalMs = config.pollingInterval.inWholeMilliseconds, + log = log +) { + data class Config( + val pollingInterval: Duration, + val storagePeriod: Duration + ) + + override fun action(): SafeFuture<*> { + return this.repository.deleteRejectedTransactions( + clock.now().minus(config.storagePeriod) + ).thenPeek { deletedRows -> + if (deletedRows > 0) { + log.debug("deletedRows=$deletedRows") + } + } + } + + override fun handleError(error: Throwable) { + log.error( + "Error with rejected transaction cleanup service: errorMessage={}", + error.message, + error + ) + } +} diff --git a/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/service/TransactionExclusionServiceV1Impl.kt b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/service/TransactionExclusionServiceV1Impl.kt new file mode 100644 index 00000000..afedd998 --- /dev/null +++ b/transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/service/TransactionExclusionServiceV1Impl.kt @@ -0,0 +1,76 @@ +package net.consensys.linea.transactionexclusion.service + +import com.github.michaelbull.result.Err +import com.github.michaelbull.result.Ok +import com.github.michaelbull.result.Result +import kotlinx.datetime.Clock +import net.consensys.linea.metrics.LineaMetricsCategory +import net.consensys.linea.metrics.MetricsFacade +import net.consensys.linea.transactionexclusion.ErrorType +import net.consensys.linea.transactionexclusion.RejectedTransaction +import net.consensys.linea.transactionexclusion.TransactionExclusionError +import net.consensys.linea.transactionexclusion.TransactionExclusionServiceV1 +import net.consensys.linea.transactionexclusion.TransactionExclusionServiceV1.SaveRejectedTransactionStatus +import net.consensys.zkevm.persistence.dao.rejectedtransaction.RejectedTransactionsDao +import net.consensys.zkevm.persistence.db.DuplicatedRecordException +import tech.pegasys.teku.infrastructure.async.SafeFuture +import kotlin.time.Duration + +class TransactionExclusionServiceV1Impl( + private val config: Config, + private val repository: RejectedTransactionsDao, + metricsFacade: MetricsFacade, + private val clock: Clock = Clock.System +) : TransactionExclusionServiceV1 { + data class Config( + val rejectedTimestampWithinDuration: Duration + ) + + private val txRejectionCounter = metricsFacade.createCounter( + LineaMetricsCategory.TX_EXCLUSION_API, + "transactions.rejected", + "Counter of rejected transactions reported to Transaction Exclusion API service" + ) + + override fun saveRejectedTransaction( + rejectedTransaction: RejectedTransaction + ): SafeFuture< + Result + > { + return this.repository.saveNewRejectedTransaction(rejectedTransaction) + .handleComposed { _, error -> + if (error != null) { + if (error is DuplicatedRecordException) { + SafeFuture.completedFuture( + Ok(SaveRejectedTransactionStatus.DUPLICATE_ALREADY_SAVED_BEFORE) + ) + } else { + SafeFuture.completedFuture( + Err(TransactionExclusionError(ErrorType.SERVER_ERROR, error.message ?: "")) + ) + } + } else { + txRejectionCounter.increment() + SafeFuture.completedFuture(Ok(SaveRejectedTransactionStatus.SAVED)) + } + } + } + + override fun getTransactionExclusionStatus( + txHash: ByteArray + ): SafeFuture> { + return this.repository.findRejectedTransactionByTxHash( + txHash = txHash, + notRejectedBefore = clock.now().minus(config.rejectedTimestampWithinDuration) + ) + .handleComposed { result, error -> + if (error != null) { + SafeFuture.completedFuture( + Err(TransactionExclusionError(ErrorType.SERVER_ERROR, error.message ?: "")) + ) + } else { + SafeFuture.completedFuture(Ok(result)) + } + } + } +} diff --git a/transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/app/api/ArgumentParserTest.kt b/transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/app/api/ArgumentParserTest.kt new file mode 100644 index 00000000..1ba690df --- /dev/null +++ b/transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/app/api/ArgumentParserTest.kt @@ -0,0 +1,358 @@ +package net.consensys.linea.transactionexclusion.app.api + +import io.vertx.core.json.JsonObject +import kotlinx.datetime.Instant +import net.consensys.decodeHex +import net.consensys.encodeHex +import net.consensys.linea.transactionexclusion.ModuleOverflow +import net.consensys.linea.transactionexclusion.RejectedTransaction +import net.consensys.linea.transactionexclusion.TransactionInfo +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import kotlin.random.Random + +class ArgumentParserTest { + @Test + fun getTransactionRLPInRawBytes_should_return_correct_byte_array() { + val transactionRLPInHexStr = + "0x02f8388204d2648203e88203e88203e8941195cf65f83b3a5768f3c4" + + "96d3a05ad6412c64b38203e88c666d93e9cc5f73748162cea9c0017b8201c8" + Assertions.assertTrue( + ArgumentParser.getTransactionRLPInRawBytes(transactionRLPInHexStr) + .contentEquals(transactionRLPInHexStr.decodeHex()) + ) + } + + @Test + fun getTransactionRLPInRawBytes_should_throw_error_for_invalid_hex_string() { + // odd number of hex character + assertThrows { + ArgumentParser.getTransactionRLPInRawBytes( + "0x02f8388204d2648203e88203e88203e8941195cf65f83b3a5768f3c4" + + "96d3a05ad6412c64b38203e88c666d93e9cc5f73748162cea9c0017b820" + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("RLP-encoded transaction cannot be parsed") + ) + } + + // invalid hex character + assertThrows { + ArgumentParser.getTransactionRLPInRawBytes( + "yyf8388204d2648203e88203e88203e8941195cf65f83b3a5768f3c4" + + "96d3a05ad6412c64b38203e88c666d93e9cc5f73748162cea9c0017b8201xx" + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("RLP-encoded transaction cannot be parsed") + ) + } + } + + @Test + fun getTxHashInRawBytes_should_return_correct_byte_array() { + val txHashInHexStr = "0x526e56101cf39c1e717cef9cedf6fdddb42684711abda35bae51136dbb350ad7" + Assertions.assertTrue( + ArgumentParser.getTxHashInRawBytes(txHashInHexStr) + .contentEquals(txHashInHexStr.decodeHex()) + ) + } + + @Test + fun getTxHashInRawBytes_should_throw_error_for_invalid_hex_string() { + // hex string of less than 64 hex characters + assertThrows { + ArgumentParser.getTxHashInRawBytes( + "0x526e56101cf39c1e717cef9cedf6fdddb42684711abda35bae51136dbb350a" + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Hex string of transaction hash cannot be parsed") + ) + } + } + + @Test + fun getTransactionInfoFromRLP_should_return_correct_transactionInfo() { + val transactionRLP = + ( + "0x02f8388204d2648203e88203e88203e8941195cf65f83b3a5768f3c4" + + "96d3a05ad6412c64b38203e88c666d93e9cc5f73748162cea9c0017b8201c8" + ).decodeHex() + Assertions.assertEquals( + ArgumentParser.getTransactionInfoFromRLP(transactionRLP), + TransactionInfo( + hash = "0x526e56101cf39c1e717cef9cedf6fdddb42684711abda35bae51136dbb350ad7".decodeHex(), + from = "0x4d144d7b9c96b26361d6ac74dd1d8267edca4fc2".decodeHex(), + to = "0x1195cf65f83b3a5768f3c496d3a05ad6412c64b3".decodeHex(), + nonce = 100UL + ) + ) + } + + @Test + fun getTransactionInfoFromRLP_should_throw_error_for_invalid_transactionRLP() { + // hex string of less than 64 hex characters + assertThrows { + ArgumentParser.getTransactionInfoFromRLP( + ( + "0xaaf8388204d2648203e88203e88203e8941195cf65f83b3a5768f3c4" + + "96d3a05ad6412c64b38203e88c666d93e9cc5f73748162cea9c0017b8201c8" + ).decodeHex() + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("RLP-encoded transaction cannot be parsed") + ) + } + } + + @Test + fun getOverflows_should_return_correct_list_of_ModuleOverflow() { + val expectedModuleOverflowList = listOf( + ModuleOverflow( + module = "ADD", + count = 402, + limit = 70 + ), + ModuleOverflow( + module = "MUL", + count = 587, + limit = 400 + ) + ) + + // valid module overflow as json request params + val moduleOverflowJsonRequestParams = + listOf( + mapOf( + "module" to "ADD", + "count" to "402", + "limit" to "70" + ), + mapOf( + "module" to "MUL", + "count" to "587", + "limit" to "400" + ) + ) + + Assertions.assertEquals( + ArgumentParser.getOverflows(moduleOverflowJsonRequestParams), + expectedModuleOverflowList + ) + } + + @Test + fun getOverflows_should_throw_error_for_invalid_moduleOverflowJsonRequestParams() { + // invalid module overflow as json request params (invalid field name xxx) + assertThrows { + ArgumentParser.getOverflows( + listOf( + mapOf( + "module" to "ADD", + "count" to "402", + "xxx" to "70" + ), + mapOf( + "module" to "MUL", + "count" to "587", + "limit" to "400" + ) + ) + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Overflows cannot be parsed") + ) + } + + // invalid module overflow as json request params (invalid module value) + assertThrows { + ArgumentParser.getOverflows( + listOf( + mapOf( + "module" to null, + "count" to "402", + "limit" to "70" + ), + mapOf( + "module" to "MUL", + "count" to "587", + "limit" to "400" + ) + ) + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Overflows cannot be parsed") + ) + } + + // invalid json string (invalid input) + assertThrows { + ArgumentParser.getOverflows(JsonObject()) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Overflows cannot be parsed") + ) + } + } + + @Test + fun getReasonMessage_should_return_correct_reason_message() { + val reasonMessage = "Transaction line count for module ADD=402 is above the limit 70" + Assertions.assertEquals( + ArgumentParser.getReasonMessage(reasonMessage), + reasonMessage + ) + + val reasonMessageWithMaxLen = Random.Default.nextBytes(128).encodeHex(prefix = false) + Assertions.assertEquals( + ArgumentParser.getReasonMessage(reasonMessageWithMaxLen), + reasonMessageWithMaxLen + ) + } + + @Test + fun getReasonMessage_should_throw_error_for_string_length_longer_than_256() { + // reason message string with more than 256 characters + assertThrows { + ArgumentParser.getReasonMessage( + Random.Default.nextBytes(128).encodeHex(prefix = false) + "0" + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Reason message should not be more than 256 characters") + ) + } + } + + @Test + fun getBlockNumber_should_return_correct_unsigned_long_or_null() { + // 10-base number + val blockNumberStr = "12345" + Assertions.assertEquals( + ArgumentParser.getBlockNumber(blockNumberStr)!!, + blockNumberStr.toULong() + ) + + Assertions.assertNull( + ArgumentParser.getBlockNumber(null) + ) + } + + @Test + fun getBlockNumber_should_throw_error_for_invalid_blockNumberStr() { + // block number string with hex string + assertThrows { + ArgumentParser.getBlockNumber( + "0x12345" + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Block number cannot be parsed to an unsigned number") + ) + } + + // block number string with random characters + assertThrows { + ArgumentParser.getBlockNumber( + "xxyyzz" + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Block number cannot be parsed to an unsigned number") + ) + } + + // empty block number string + assertThrows { + ArgumentParser.getBlockNumber("") + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Block number cannot be parsed to an unsigned number") + ) + } + } + + @Test + fun getTimestampFromISO8601_should_return_correct_instant() { + // timestamp in ISO-8601 + val timestampStr = "2024-09-05T09:22:52Z" + Assertions.assertEquals( + ArgumentParser.getTimestampFromISO8601(timestampStr), + Instant.parse(timestampStr) + ) + } + + @Test + fun getTimestampFromISO8601_should_throw_error_for_invalid_timestampStr() { + // timestamp string not in ISO-8601 + assertThrows { + ArgumentParser.getTimestampFromISO8601( + "2024-09-05_09:22:52" + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Timestamp is not in ISO-8601") + ) + } + + // timestamp string in epoch time millisecond + assertThrows { + ArgumentParser.getTimestampFromISO8601( + "1725543970103" + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Timestamp is not in ISO-8601") + ) + } + + // empty timestamp string + assertThrows { + ArgumentParser.getTimestampFromISO8601("") + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Timestamp is not in ISO-8601") + ) + } + } + + @Test + fun getTxRejectionStage_should_return_correct_rejection_stage() { + val txRejectionStageStr = "SEQUENCER" + Assertions.assertEquals( + ArgumentParser.getTxRejectionStage(txRejectionStageStr), + RejectedTransaction.Stage.SEQUENCER + ) + } + + @Test + fun getTxRejectionStage_should_throw_error_for_invalid_txRejectionStageStr() { + // rejection stage string in lower case + assertThrows { + ArgumentParser.getTxRejectionStage( + "sequencer" + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Unsupported transaction rejection stage") + ) + } + + // rejection stage string in random characters + assertThrows { + ArgumentParser.getTxRejectionStage( + "helloworld" + ) + }.also { error -> + Assertions.assertTrue( + error.message!!.contains("Unsupported transaction rejection stage") + ) + } + } +} diff --git a/transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/app/api/RequestHandlersTest.kt b/transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/app/api/RequestHandlersTest.kt new file mode 100644 index 00000000..f8480331 --- /dev/null +++ b/transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/app/api/RequestHandlersTest.kt @@ -0,0 +1,456 @@ +package net.consensys.linea.transactionexclusion.app.api + +import com.github.michaelbull.result.Err +import com.github.michaelbull.result.Ok +import com.github.michaelbull.result.get +import com.github.michaelbull.result.getError +import io.vertx.core.json.JsonObject +import net.consensys.encodeHex +import net.consensys.linea.async.get +import net.consensys.linea.jsonrpc.JsonRpcErrorResponse +import net.consensys.linea.jsonrpc.JsonRpcRequestListParams +import net.consensys.linea.jsonrpc.JsonRpcRequestMapParams +import net.consensys.linea.jsonrpc.JsonRpcSuccessResponse +import net.consensys.linea.transactionexclusion.ErrorType +import net.consensys.linea.transactionexclusion.TransactionExclusionError +import net.consensys.linea.transactionexclusion.TransactionExclusionServiceV1 +import net.consensys.linea.transactionexclusion.test.defaultRejectedTransaction +import net.consensys.toHexString +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.Mockito +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import tech.pegasys.teku.infrastructure.async.SafeFuture + +class RequestHandlersTest { + private lateinit var transactionExclusionServiceMock: TransactionExclusionServiceV1 + + private val mapParams = mapOf( + "txRejectionStage" to "SEQUENCER", + "timestamp" to "2024-09-05T09:22:52Z", + "transactionRLP" to defaultRejectedTransaction.transactionRLP.encodeHex(), + "reasonMessage" to defaultRejectedTransaction.reasonMessage, + "overflows" to defaultRejectedTransaction.overflows + ) + + private val mapRequest = JsonRpcRequestMapParams( + "2.0", + "1", + "linea_saveRejectedTransactionV1", + mapParams + ) + + private val listRequest = JsonRpcRequestListParams( + "2.0", + "1", + "linea_saveRejectedTransactionV1", + listOf(mapParams) + ) + + @BeforeEach + fun beforeEach() { + transactionExclusionServiceMock = mock( + defaultAnswer = Mockito.RETURNS_DEEP_STUBS + ) + } + + @Test + fun SaveRejectedTransactionRequestHandlerV1_rejectsEmptyMap() { + val request = JsonRpcRequestMapParams("", "", "", emptyMap()) + + val saveRequestHandlerV1 = SaveRejectedTransactionRequestHandlerV1( + transactionExclusionServiceMock + ) + + val result = saveRequestHandlerV1.invoke( + user = null, + request = request, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals( + Err( + JsonRpcErrorResponse.invalidParams( + request.id, + "Missing [txRejectionStage,timestamp,reasonMessage,transactionRLP,overflows] " + + "from the given request params" + ) + ), + result + ) + } + + @Test + fun SaveRejectedTransactionRequestHandlerV1_rejectsEmptyList() { + val request = JsonRpcRequestListParams("", "", "", emptyList()) + + val saveRequestHandlerV1 = SaveRejectedTransactionRequestHandlerV1( + transactionExclusionServiceMock + ) + + val result = saveRequestHandlerV1.invoke( + user = null, + request = request, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals( + Err( + JsonRpcErrorResponse.invalidParams( + request.id, + "The given request params list should have one argument" + ) + ), + result + ) + } + + @Test + fun SaveRejectedTransactionRequestHandlerV1_rejectsListWithInvalidArgument() { + val request = JsonRpcRequestListParams("", "", "", listOf("invalid_argument")) + + val saveRequestHandlerV1 = SaveRejectedTransactionRequestHandlerV1( + transactionExclusionServiceMock + ) + + val result = saveRequestHandlerV1.invoke( + user = null, + request = request, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals( + Err( + JsonRpcErrorResponse.invalidParams( + request.id, + "The argument in the request params list should be an object" + ) + ), + result + ) + } + + @Test + fun SaveRejectedTransactionRequestHandlerV1_invoke_acceptsValidRequestMap() { + whenever(transactionExclusionServiceMock.saveRejectedTransaction(any())) + .thenReturn( + SafeFuture.completedFuture( + Ok(TransactionExclusionServiceV1.SaveRejectedTransactionStatus.SAVED) + ) + ) + + val saveRequestHandlerV1 = SaveRejectedTransactionRequestHandlerV1( + transactionExclusionServiceMock + ) + + val expectedResult = JsonObject() + .put("status", TransactionExclusionServiceV1.SaveRejectedTransactionStatus.SAVED) + .put("txHash", defaultRejectedTransaction.transactionInfo.hash.encodeHex()) + .let { + JsonRpcSuccessResponse(mapRequest.id, it) + } + + val result = saveRequestHandlerV1.invoke( + user = null, + request = mapRequest, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals(expectedResult, result.get()) + } + + @Test + fun SaveRejectedTransactionRequestHandlerV1_invoke_acceptsValidRequestList() { + whenever(transactionExclusionServiceMock.saveRejectedTransaction(any())) + .thenReturn( + SafeFuture.completedFuture( + Ok(TransactionExclusionServiceV1.SaveRejectedTransactionStatus.SAVED) + ) + ) + + val saveRequestHandlerV1 = SaveRejectedTransactionRequestHandlerV1( + transactionExclusionServiceMock + ) + + val expectedResult = JsonObject() + .put("status", TransactionExclusionServiceV1.SaveRejectedTransactionStatus.SAVED) + .put("txHash", defaultRejectedTransaction.transactionInfo.hash.encodeHex()) + .let { + JsonRpcSuccessResponse(listRequest.id, it) + } + + val result = saveRequestHandlerV1.invoke( + user = null, + request = listRequest, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals(expectedResult, result.get()) + } + + @Test + fun SaveRejectedTransactionRequestHandlerV1_invoke_acceptsValidRequestMap_without_blockNumber() { + whenever(transactionExclusionServiceMock.saveRejectedTransaction(any())) + .thenReturn( + SafeFuture.completedFuture( + Ok(TransactionExclusionServiceV1.SaveRejectedTransactionStatus.SAVED) + ) + ) + + val saveTxRequestHandlerV1 = SaveRejectedTransactionRequestHandlerV1( + transactionExclusionServiceMock + ) + + val expectedResult = JsonObject() + .put("status", TransactionExclusionServiceV1.SaveRejectedTransactionStatus.SAVED) + .put("txHash", defaultRejectedTransaction.transactionInfo.hash.encodeHex()) + .let { + JsonRpcSuccessResponse(mapRequest.id, it) + } + + val result = saveTxRequestHandlerV1.invoke( + user = null, + request = mapRequest, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals(expectedResult, result.get()) + } + + @Test + fun SaveRejectedTransactionRequestHandlerV1_invoke_return_success_result_with_duplicate_status() { + whenever(transactionExclusionServiceMock.saveRejectedTransaction(any())) + .thenReturn( + SafeFuture.completedFuture( + Ok(TransactionExclusionServiceV1.SaveRejectedTransactionStatus.DUPLICATE_ALREADY_SAVED_BEFORE) + ) + ) + + val saveTxRequestHandlerV1 = SaveRejectedTransactionRequestHandlerV1( + transactionExclusionServiceMock + ) + + val expectedResult = JsonObject() + .put("status", TransactionExclusionServiceV1.SaveRejectedTransactionStatus.DUPLICATE_ALREADY_SAVED_BEFORE) + .put("txHash", defaultRejectedTransaction.transactionInfo.hash.encodeHex()) + .let { + JsonRpcSuccessResponse(mapRequest.id, it) + } + + val result = saveTxRequestHandlerV1.invoke( + user = null, + request = mapRequest, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals(expectedResult, result.get()) + } + + @Test + fun SaveRejectedTransactionRequestHandlerV1_invoke_return_failure_result() { + whenever(transactionExclusionServiceMock.saveRejectedTransaction(any())) + .thenReturn( + SafeFuture.completedFuture( + Err( + TransactionExclusionError( + ErrorType.SERVER_ERROR, + "error for unit test" + ) + ) + ) + ) + + val saveTxRequestHandlerV1 = SaveRejectedTransactionRequestHandlerV1( + transactionExclusionServiceMock + ) + + val expectedResult = JsonRpcErrorResponse( + mapRequest.id, + jsonRpcError( + TransactionExclusionError( + ErrorType.SERVER_ERROR, + "error for unit test" + ) + ) + ) + + val result = saveTxRequestHandlerV1.invoke( + user = null, + request = mapRequest, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals(expectedResult, result.getError()) + } + + @Test + fun GetTransactionExclusionStatusRequestHandlerV1_rejectsEmptyList() { + val request = JsonRpcRequestListParams("", "", "", emptyList()) + + val getRequestHandlerV1 = GetTransactionExclusionStatusRequestHandlerV1( + transactionExclusionServiceMock + ) + + val result = getRequestHandlerV1.invoke( + user = null, + request = request, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals( + Err( + JsonRpcErrorResponse.invalidParams( + request.id, + "The given request params list should have one argument" + ) + ), + result + ) + } + + @Test + fun GetTransactionExclusionStatusRequestHandlerV1_rejectsListWithInvalidArgument() { + val request = JsonRpcRequestListParams("", "", "", listOf("0x123")) + + val getRequestHandlerV1 = GetTransactionExclusionStatusRequestHandlerV1( + transactionExclusionServiceMock + ) + + val result = getRequestHandlerV1.invoke( + user = null, + request = request, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals( + Err( + JsonRpcErrorResponse.invalidParams( + request.id, + "Hex string of transaction hash cannot be parsed: Must have an even length" + ) + ), + result + ) + } + + @Test + fun GetTransactionExclusionStatusRequestHandlerV1_invoke_acceptsValidRequestList() { + whenever(transactionExclusionServiceMock.getTransactionExclusionStatus(any())) + .thenReturn( + SafeFuture.completedFuture( + Ok(defaultRejectedTransaction) + ) + ) + + val request = JsonRpcRequestListParams( + "2.0", + "1", + "linea_getTransactionExclusionStatusV1", + listOf( + defaultRejectedTransaction.transactionInfo.hash.encodeHex() + ) + ) + + val getTxStatusRequestHandlerV1 = GetTransactionExclusionStatusRequestHandlerV1( + transactionExclusionServiceMock + ) + + val expectedResult = JsonObject() + .put("txHash", defaultRejectedTransaction.transactionInfo.hash.encodeHex()) + .put("from", defaultRejectedTransaction.transactionInfo.from.encodeHex()) + .put("nonce", defaultRejectedTransaction.transactionInfo.nonce.toHexString()) + .put("txRejectionStage", defaultRejectedTransaction.txRejectionStage.name) + .put("reasonMessage", defaultRejectedTransaction.reasonMessage) + .put("timestamp", defaultRejectedTransaction.timestamp.toString()) + .put("blockNumber", defaultRejectedTransaction.blockNumber!!.toHexString()) + .let { + JsonRpcSuccessResponse(request.id, it) + } + + val result = getTxStatusRequestHandlerV1.invoke( + user = null, + request = request, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals(expectedResult, result.get()) + } + + @Test + fun GetTransactionExclusionStatusRequestHandlerV1_invoke_return_null_result() { + whenever(transactionExclusionServiceMock.getTransactionExclusionStatus(any())) + .thenReturn(SafeFuture.completedFuture(Ok(null))) + + val request = JsonRpcRequestListParams( + "2.0", + "1", + "linea_getTransactionExclusionStatusV1", + listOf( + defaultRejectedTransaction.transactionInfo.hash.encodeHex() + ) + ) + + val getTxStatusRequestHandlerV1 = GetTransactionExclusionStatusRequestHandlerV1( + transactionExclusionServiceMock + ) + + val expectedResult = JsonRpcSuccessResponse(request.id, null) + + val result = getTxStatusRequestHandlerV1.invoke( + user = null, + request = request, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals(expectedResult, result.get()) + } + + @Test + fun GetTransactionExclusionStatusRequestHandlerV1_invoke_return_failure_result() { + whenever(transactionExclusionServiceMock.getTransactionExclusionStatus(any())) + .thenReturn( + SafeFuture.completedFuture( + Err( + TransactionExclusionError( + ErrorType.SERVER_ERROR, + "error for unit test" + ) + ) + ) + ) + + val request = JsonRpcRequestListParams( + "2.0", + "1", + "linea_getTransactionExclusionStatusV1", + listOf( + defaultRejectedTransaction.transactionInfo.hash.encodeHex() + ) + ) + + val getTxStatusRequestHandlerV1 = GetTransactionExclusionStatusRequestHandlerV1( + transactionExclusionServiceMock + ) + + val expectedResult = JsonRpcErrorResponse( + request.id, + jsonRpcError( + TransactionExclusionError( + ErrorType.SERVER_ERROR, + "error for unit test" + ) + ) + ) + + val result = getTxStatusRequestHandlerV1.invoke( + user = null, + request = request, + requestJson = JsonObject() + ).get() + + Assertions.assertEquals(expectedResult, result.getError()) + } +} diff --git a/transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/service/RejectedTransactionCleanupServiceTest.kt b/transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/service/RejectedTransactionCleanupServiceTest.kt new file mode 100644 index 00000000..dff10b83 --- /dev/null +++ b/transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/service/RejectedTransactionCleanupServiceTest.kt @@ -0,0 +1,69 @@ +package net.consensys.linea.transactionexclusion.service + +import io.vertx.core.Vertx +import io.vertx.junit5.Timeout +import io.vertx.junit5.VertxExtension +import io.vertx.junit5.VertxTestContext +import kotlinx.datetime.Clock +import net.consensys.FakeFixedClock +import net.consensys.zkevm.persistence.dao.rejectedtransaction.RejectedTransactionsDao +import org.awaitility.Awaitility +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Mockito +import org.mockito.kotlin.any +import org.mockito.kotlin.atLeastOnce +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import tech.pegasys.teku.infrastructure.async.SafeFuture +import java.util.concurrent.TimeUnit +import kotlin.time.Duration.Companion.hours +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.toJavaDuration + +@ExtendWith(VertxExtension::class) +class RejectedTransactionCleanupServiceTest { + private lateinit var rejectedTransactionCleanupService: RejectedTransactionCleanupService + private lateinit var rejectedTransactionsRepositoryMock: RejectedTransactionsDao + private var fakeClock = FakeFixedClock(Clock.System.now()) + + @BeforeEach + fun beforeEach() { + fakeClock.setTimeTo(Clock.System.now()) + rejectedTransactionsRepositoryMock = mock( + defaultAnswer = Mockito.RETURNS_DEEP_STUBS + ).also { + whenever(it.deleteRejectedTransactions(any())) + .thenReturn(SafeFuture.completedFuture(1)) + } + rejectedTransactionCleanupService = + RejectedTransactionCleanupService( + config = RejectedTransactionCleanupService.Config( + pollingInterval = 100.milliseconds, + storagePeriod = 24.hours + ), + clock = fakeClock, + vertx = Vertx.vertx(), + repository = rejectedTransactionsRepositoryMock + ) + } + + @Test + @Timeout(2, timeUnit = TimeUnit.SECONDS) + fun `when rejectedTransactionCleanupService starts, deleteRejectedTransaction should be called` + (testContext: VertxTestContext) { + rejectedTransactionCleanupService.start() + .thenApply { + Awaitility.await() + .pollInterval(50.milliseconds.toJavaDuration()) + .untilAsserted { + verify(rejectedTransactionsRepositoryMock, atLeastOnce()) + .deleteRejectedTransactions(eq(fakeClock.now().minus(24.hours))) + } + testContext.completeNow() + }.whenException(testContext::failNow) + } +} diff --git a/transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/service/TransactionExclusionServiceTest.kt b/transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/service/TransactionExclusionServiceTest.kt new file mode 100644 index 00000000..f58f792f --- /dev/null +++ b/transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/service/TransactionExclusionServiceTest.kt @@ -0,0 +1,152 @@ +package net.consensys.linea.transactionexclusion.service + +import com.github.michaelbull.result.Err +import com.github.michaelbull.result.Ok +import net.consensys.linea.metrics.MetricsFacade +import net.consensys.linea.transactionexclusion.ErrorType +import net.consensys.linea.transactionexclusion.TransactionExclusionError +import net.consensys.linea.transactionexclusion.TransactionExclusionServiceV1 +import net.consensys.linea.transactionexclusion.test.defaultRejectedTransaction +import net.consensys.zkevm.persistence.dao.rejectedtransaction.RejectedTransactionsDao +import net.consensys.zkevm.persistence.db.DuplicatedRecordException +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.Mockito +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import tech.pegasys.teku.infrastructure.async.SafeFuture +import kotlin.time.Duration.Companion.hours + +class TransactionExclusionServiceTest { + private val metricsFacadeMock = mock(defaultAnswer = Mockito.RETURNS_DEEP_STUBS) + private val config = TransactionExclusionServiceV1Impl.Config( + rejectedTimestampWithinDuration = 24.hours + ) + private lateinit var rejectedTransactionsRepositoryMock: RejectedTransactionsDao + + @BeforeEach + fun beforeEach() { + rejectedTransactionsRepositoryMock = mock( + defaultAnswer = Mockito.RETURNS_DEEP_STUBS + ) + } + + @Test + fun saveRejectedTransaction_return_success_result_with_saved_status() { + whenever(rejectedTransactionsRepositoryMock.findRejectedTransactionByTxHash(any(), any())) + .thenReturn(SafeFuture.completedFuture(defaultRejectedTransaction)) + whenever(rejectedTransactionsRepositoryMock.saveNewRejectedTransaction(any())) + .thenReturn(SafeFuture.completedFuture(Unit)) + + val transactionExclusionService = TransactionExclusionServiceV1Impl( + config = config, + repository = rejectedTransactionsRepositoryMock, + metricsFacade = metricsFacadeMock + ) + + Assertions.assertEquals( + Ok(TransactionExclusionServiceV1.SaveRejectedTransactionStatus.SAVED), + transactionExclusionService.saveRejectedTransaction(defaultRejectedTransaction).get() + ) + } + + @Test + fun saveRejectedTransaction_return_success_result_with_duplicated_already_saved_status() { + whenever(rejectedTransactionsRepositoryMock.saveNewRejectedTransaction(any())) + .thenReturn(SafeFuture.failedFuture(DuplicatedRecordException())) + + val transactionExclusionService = TransactionExclusionServiceV1Impl( + config = config, + repository = rejectedTransactionsRepositoryMock, + metricsFacade = metricsFacadeMock + ) + + Assertions.assertEquals( + Ok(TransactionExclusionServiceV1.SaveRejectedTransactionStatus.DUPLICATE_ALREADY_SAVED_BEFORE), + transactionExclusionService.saveRejectedTransaction(defaultRejectedTransaction).get() + ) + } + + @Test + fun saveRejectedTransaction_return_error_result_when_saveRejectedTransaction_failed() { + whenever(rejectedTransactionsRepositoryMock.findRejectedTransactionByTxHash(any(), any())) + .thenReturn(SafeFuture.completedFuture(defaultRejectedTransaction)) + whenever(rejectedTransactionsRepositoryMock.saveNewRejectedTransaction(any())) + .thenReturn(SafeFuture.failedFuture(RuntimeException())) + + val transactionExclusionService = TransactionExclusionServiceV1Impl( + config = config, + repository = rejectedTransactionsRepositoryMock, + metricsFacade = metricsFacadeMock + ) + + Assertions.assertEquals( + Err(TransactionExclusionError(ErrorType.SERVER_ERROR, "")), + transactionExclusionService.saveRejectedTransaction(defaultRejectedTransaction).get() + ) + } + + @Test + fun getTransactionExclusionStatus_return_success_result_with_rejected_txn() { + whenever(rejectedTransactionsRepositoryMock.findRejectedTransactionByTxHash(any(), any())) + .thenReturn(SafeFuture.completedFuture(defaultRejectedTransaction)) + + val transactionExclusionService = TransactionExclusionServiceV1Impl( + config = config, + repository = rejectedTransactionsRepositoryMock, + metricsFacade = metricsFacadeMock + ) + + Assertions.assertEquals( + Ok(defaultRejectedTransaction), + transactionExclusionService.getTransactionExclusionStatus( + defaultRejectedTransaction.transactionInfo.hash + ).get() + ) + } + + @Test + fun getTransactionExclusionStatus_return_error_result_with_transaction_unavailable() { + whenever(rejectedTransactionsRepositoryMock.findRejectedTransactionByTxHash(any(), any())) + .thenReturn(SafeFuture.completedFuture(null)) + + val transactionExclusionService = TransactionExclusionServiceV1Impl( + config = config, + repository = rejectedTransactionsRepositoryMock, + metricsFacade = metricsFacadeMock + ) + + Assertions.assertEquals( + Ok(null), + transactionExclusionService.getTransactionExclusionStatus( + defaultRejectedTransaction.transactionInfo.hash + ).get() + ) + } + + @Test + fun getTransactionExclusionStatus_return_error_result_with_other_error() { + whenever(rejectedTransactionsRepositoryMock.findRejectedTransactionByTxHash(any(), any())) + .thenReturn(SafeFuture.failedFuture(RuntimeException())) + + val transactionExclusionService = TransactionExclusionServiceV1Impl( + config = config, + repository = rejectedTransactionsRepositoryMock, + metricsFacade = metricsFacadeMock + ) + + Assertions.assertEquals( + Err( + TransactionExclusionError( + ErrorType.SERVER_ERROR, + "" + ) + ), + transactionExclusionService.getTransactionExclusionStatus( + defaultRejectedTransaction.transactionInfo.hash + ).get() + ) + } +} diff --git a/transaction-exclusion-api/core/build.gradle b/transaction-exclusion-api/core/build.gradle new file mode 100644 index 00000000..e459955b --- /dev/null +++ b/transaction-exclusion-api/core/build.gradle @@ -0,0 +1,10 @@ +plugins { + id 'net.consensys.zkevm.kotlin-library-conventions' + id 'java-test-fixtures' +} + +dependencies { + implementation project(":jvm-libs:generic:extensions:kotlin") + + testFixtures implementation(testFixtures(project(":jvm-libs:generic:extensions:kotlin"))) +} diff --git a/transaction-exclusion-api/core/src/main/kotlin/net/consensys/linea/transactionexclusion/ErrorType.kt b/transaction-exclusion-api/core/src/main/kotlin/net/consensys/linea/transactionexclusion/ErrorType.kt new file mode 100644 index 00000000..1beea363 --- /dev/null +++ b/transaction-exclusion-api/core/src/main/kotlin/net/consensys/linea/transactionexclusion/ErrorType.kt @@ -0,0 +1,8 @@ +package net.consensys.linea.transactionexclusion + +/** For simplicity, placing all error codes into single enum */ +enum class ErrorType { + SERVER_ERROR +} + +data class TransactionExclusionError(val errorType: ErrorType, val errorDetail: String) diff --git a/transaction-exclusion-api/core/src/main/kotlin/net/consensys/linea/transactionexclusion/RejectedTransaction.kt b/transaction-exclusion-api/core/src/main/kotlin/net/consensys/linea/transactionexclusion/RejectedTransaction.kt new file mode 100644 index 00000000..4d1d7395 --- /dev/null +++ b/transaction-exclusion-api/core/src/main/kotlin/net/consensys/linea/transactionexclusion/RejectedTransaction.kt @@ -0,0 +1,97 @@ +package net.consensys.linea.transactionexclusion + +import kotlinx.datetime.Instant +import net.consensys.encodeHex + +data class ModuleOverflow( + val count: Long, + val limit: Long, + val module: String +) { + // Jackson ObjectMapper requires a default constructor + constructor() : this(0L, 0L, "") + + override fun toString(): String { + return "module=$module count=$count limit=$limit" + } +} + +data class TransactionInfo( + val hash: ByteArray, + val from: ByteArray, + val to: ByteArray, + val nonce: ULong +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as TransactionInfo + + if (!hash.contentEquals(other.hash)) return false + if (!from.contentEquals(other.from)) return false + if (!to.contentEquals(other.to)) return false + return nonce == other.nonce + } + + override fun hashCode(): Int { + var result = hash.contentHashCode() + result = 31 * result + from.contentHashCode() + result = 31 * result + to.contentHashCode() + result = 31 * result + nonce.hashCode() + return result + } + + override fun toString(): String { + return "hash=${hash.encodeHex()} from=${from.encodeHex()} " + + "to=${to.encodeHex()} nonce=$nonce" + } +} + +data class RejectedTransaction( + val txRejectionStage: Stage, + val timestamp: Instant, + val blockNumber: ULong?, + val transactionRLP: ByteArray, + val reasonMessage: String, + val overflows: List, + val transactionInfo: TransactionInfo +) { + enum class Stage { + SEQUENCER, + RPC, + P2P + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as RejectedTransaction + + if (txRejectionStage != other.txRejectionStage) return false + if (timestamp != other.timestamp) return false + if (blockNumber != other.blockNumber) return false + if (!transactionRLP.contentEquals(other.transactionRLP)) return false + if (reasonMessage != other.reasonMessage) return false + return overflows == other.overflows + } + + override fun hashCode(): Int { + var result = txRejectionStage.hashCode() + result = 31 * result + timestamp.hashCode() + result = 31 * result + blockNumber.hashCode() + result = 31 * result + transactionRLP.contentHashCode() + result = 31 * result + reasonMessage.hashCode() + result = 31 * result + overflows.hashCode() + return result + } + + override fun toString(): String { + return "txRejectionStage=$txRejectionStage timestamp=${timestamp.toEpochMilliseconds()} blockNumber=$blockNumber" + + " transactionRLP=${transactionRLP.encodeHex()}" + + " transactionInfo=$transactionInfo" + + " reasonMessage=\"$reasonMessage\"" + + " overflows=[${overflows.joinToString(",") { "{$it}" }}]" + } +} diff --git a/transaction-exclusion-api/core/src/main/kotlin/net/consensys/linea/transactionexclusion/TransactionExclusionServiceV1.kt b/transaction-exclusion-api/core/src/main/kotlin/net/consensys/linea/transactionexclusion/TransactionExclusionServiceV1.kt new file mode 100644 index 00000000..c6933ab4 --- /dev/null +++ b/transaction-exclusion-api/core/src/main/kotlin/net/consensys/linea/transactionexclusion/TransactionExclusionServiceV1.kt @@ -0,0 +1,19 @@ +package net.consensys.linea.transactionexclusion + +import com.github.michaelbull.result.Result +import tech.pegasys.teku.infrastructure.async.SafeFuture + +interface TransactionExclusionServiceV1 { + enum class SaveRejectedTransactionStatus { + SAVED, + DUPLICATE_ALREADY_SAVED_BEFORE + } + + fun saveRejectedTransaction( + rejectedTransaction: RejectedTransaction + ): SafeFuture> + + fun getTransactionExclusionStatus( + txHash: ByteArray + ): SafeFuture> +} diff --git a/transaction-exclusion-api/core/src/testFixtures/kotlin/net.consensys.linea.transactionexclusion.test/Common.kt b/transaction-exclusion-api/core/src/testFixtures/kotlin/net.consensys.linea.transactionexclusion.test/Common.kt new file mode 100644 index 00000000..9301ba66 --- /dev/null +++ b/transaction-exclusion-api/core/src/testFixtures/kotlin/net.consensys.linea.transactionexclusion.test/Common.kt @@ -0,0 +1,43 @@ +package net.consensys.linea.transactionexclusion.test + +import kotlinx.datetime.Instant +import net.consensys.decodeHex +import net.consensys.linea.transactionexclusion.ModuleOverflow +import net.consensys.linea.transactionexclusion.RejectedTransaction +import net.consensys.linea.transactionexclusion.TransactionInfo + +val defaultRejectedTransaction = RejectedTransaction( + txRejectionStage = RejectedTransaction.Stage.SEQUENCER, + timestamp = Instant.parse("2024-08-31T09:18:51Z"), + blockNumber = 10000UL, + transactionRLP = + ( + "0x02f8388204d2648203e88203e88203e8941195cf65f83b3a5768f3c4" + + "96d3a05ad6412c64b38203e88c666d93e9cc5f73748162cea9c0017b8201c8" + ) + .decodeHex(), + reasonMessage = "Transaction line count for module ADD=402 is above the limit 70", + overflows = listOf( + ModuleOverflow( + module = "ADD", + count = 402, + limit = 70 + ), + ModuleOverflow( + module = "MUL", + count = 587, + limit = 401 + ), + ModuleOverflow( + module = "EXP", + count = 9000, + limit = 8192 + ) + ), + transactionInfo = TransactionInfo( + hash = "0x526e56101cf39c1e717cef9cedf6fdddb42684711abda35bae51136dbb350ad7".decodeHex(), + from = "0x4d144d7b9c96b26361d6ac74dd1d8267edca4fc2".decodeHex(), + to = "0x1195cf65f83b3a5768f3c496d3a05ad6412c64b3".decodeHex(), + nonce = 100UL + ) +) diff --git a/transaction-exclusion-api/persistence/rejectedtransaction/build.gradle b/transaction-exclusion-api/persistence/rejectedtransaction/build.gradle new file mode 100644 index 00000000..b035b961 --- /dev/null +++ b/transaction-exclusion-api/persistence/rejectedtransaction/build.gradle @@ -0,0 +1,55 @@ +import org.gradle.api.tasks.testing.logging.TestExceptionFormat +import org.gradle.api.tasks.testing.logging.TestLogEvent + +plugins { + id "net.consensys.zkevm.kotlin-library-conventions" +} + +dependencies { + implementation project(":jvm-libs:generic:extensions:futures") + implementation project(":jvm-libs:generic:extensions:kotlin") + implementation project(':jvm-libs:generic:persistence:db') + implementation project(":transaction-exclusion-api:core") + implementation "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}" + implementation "io.vertx:vertx-pg-client:${libs.versions.vertx.get()}" + + testImplementation(testFixtures(project(":transaction-exclusion-api:core"))) + testImplementation(testFixtures(project(":jvm-libs:generic:extensions:kotlin"))) + testImplementation(testFixtures(project(":jvm-libs:generic:persistence:db"))) + testImplementation("io.vertx:vertx-junit5") +} + +sourceSets { + integrationTest { + kotlin { + compileClasspath += main.output + runtimeClasspath += main.output + } + compileClasspath += sourceSets.main.output + sourceSets.main.compileClasspath + sourceSets.test.compileClasspath + runtimeClasspath += sourceSets.main.output + sourceSets.main.runtimeClasspath + sourceSets.test.runtimeClasspath + } +} + +task integrationTest(type: Test) { + test -> + testLogging { + events TestLogEvent.FAILED, + TestLogEvent.SKIPPED, + TestLogEvent.STANDARD_ERROR, + TestLogEvent.STANDARD_OUT + exceptionFormat TestExceptionFormat.FULL + showCauses true + showExceptions true + showStackTraces true + showStandardStreams false + } + description = "Runs integration tests." + group = "verification" + useJUnitPlatform() + + classpath = sourceSets.integrationTest.runtimeClasspath + testClassesDirs = sourceSets.integrationTest.output.classesDirs + + dependsOn(":localStackComposeUp") +} + diff --git a/transaction-exclusion-api/persistence/rejectedtransaction/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RejectedTransactionsPostgresDaoTest.kt b/transaction-exclusion-api/persistence/rejectedtransaction/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RejectedTransactionsPostgresDaoTest.kt new file mode 100644 index 00000000..75030e2b --- /dev/null +++ b/transaction-exclusion-api/persistence/rejectedtransaction/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RejectedTransactionsPostgresDaoTest.kt @@ -0,0 +1,319 @@ +package net.consensys.zkevm.persistence.dao.rejectedtransaction + +import com.fasterxml.jackson.databind.ObjectMapper +import io.vertx.junit5.VertxExtension +import io.vertx.sqlclient.PreparedQuery +import io.vertx.sqlclient.Row +import io.vertx.sqlclient.RowSet +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant +import net.consensys.FakeFixedClock +import net.consensys.decodeHex +import net.consensys.encodeHex +import net.consensys.linea.async.get +import net.consensys.linea.transactionexclusion.ModuleOverflow +import net.consensys.linea.transactionexclusion.RejectedTransaction +import net.consensys.linea.transactionexclusion.TransactionInfo +import net.consensys.linea.transactionexclusion.test.defaultRejectedTransaction +import net.consensys.trimToMillisecondPrecision +import net.consensys.zkevm.persistence.db.DbHelper +import net.consensys.zkevm.persistence.db.DuplicatedRecordException +import net.consensys.zkevm.persistence.db.test.CleanDbTestSuiteParallel +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.api.extension.ExtendWith +import java.util.concurrent.ExecutionException +import kotlin.time.Duration.Companion.hours +import kotlin.time.Duration.Companion.seconds + +@ExtendWith(VertxExtension::class) +class RejectedTransactionsPostgresDaoTest : CleanDbTestSuiteParallel() { + init { + target = "1" + } + + override val databaseName = DbHelper.generateUniqueDbName("tx-exclusion-api-rejectedtxns-dao-tests") + private var fakeClock = FakeFixedClock(Clock.System.now()) + private lateinit var rejectedTransactionsPostgresDao: RejectedTransactionsPostgresDao + private lateinit var notRejectedBefore: Instant + + // Helper functions + private fun createRejectedTransaction( + txRejectionStage: RejectedTransaction.Stage = defaultRejectedTransaction.txRejectionStage, + timestamp: Instant = fakeClock.now().minus(10.seconds), + blockNumber: ULong? = defaultRejectedTransaction.blockNumber, + transactionRLP: ByteArray = defaultRejectedTransaction.transactionRLP, + reasonMessage: String = defaultRejectedTransaction.reasonMessage, + overflows: List = defaultRejectedTransaction.overflows, + transactionInfo: TransactionInfo = defaultRejectedTransaction.transactionInfo + ): RejectedTransaction { + return RejectedTransaction( + txRejectionStage = txRejectionStage, + timestamp = timestamp.trimToMillisecondPrecision(), + blockNumber = blockNumber, + transactionRLP = transactionRLP, + reasonMessage = reasonMessage, + overflows = overflows, + transactionInfo = transactionInfo + ) + } + + private fun dbTableContentQuery(dbTableName: String): PreparedQuery> = + sqlClient.preparedQuery("select * from $dbTableName") + + private fun rejectedTransactionsTotalRows(): Int = + dbTableContentQuery(RejectedTransactionsPostgresDao.rejectedTransactionsTable).execute().get().size() + + private fun fullTransactionsTotalRows(): Int = + dbTableContentQuery(RejectedTransactionsPostgresDao.fullTransactionsTable).execute().get().size() + + @BeforeEach + fun beforeEach() { + fakeClock.setTimeTo(Clock.System.now()) + notRejectedBefore = fakeClock.now().minus(1.hours) + rejectedTransactionsPostgresDao = + RejectedTransactionsPostgresDao( + readConnection = sqlClient, + writeConnection = sqlClient, + clock = fakeClock + ) + } + + private fun performInsertTest( + rejectedTransaction: RejectedTransaction + ) { + rejectedTransactionsPostgresDao.saveNewRejectedTransaction(rejectedTransaction).get() + + // assert the corresponding record was inserted into the full_transactions table + val newlyInsertedFullTxnsRows = dbTableContentQuery(RejectedTransactionsPostgresDao.fullTransactionsTable) + .execute().get().filter { row -> + row.getBuffer("tx_hash").bytes.contentEquals(rejectedTransaction.transactionInfo.hash) + } + assertThat(newlyInsertedFullTxnsRows.size).isEqualTo(1) + assertThat(newlyInsertedFullTxnsRows.first().getBuffer("tx_rlp").bytes).isEqualTo( + rejectedTransaction.transactionRLP + ) + + // assert the corresponding record was inserted into the rejected_transactions table + val newlyInsertedRejectedTxnsRows = dbTableContentQuery(RejectedTransactionsPostgresDao.rejectedTransactionsTable) + .execute().get().filter { row -> + row.getBuffer("tx_hash").bytes.contentEquals(rejectedTransaction.transactionInfo.hash) && + row.getString("reject_reason") == rejectedTransaction.reasonMessage + } + assertThat(newlyInsertedRejectedTxnsRows.size).isEqualTo(1) + val insertedRow = newlyInsertedRejectedTxnsRows.first() + assertThat(insertedRow.getLong("created_epoch_milli")).isEqualTo( + fakeClock.now().toEpochMilliseconds() + ) + assertThat(insertedRow.getString("reject_stage")).isEqualTo( + RejectedTransactionsPostgresDao.rejectedStageToDbValue(rejectedTransaction.txRejectionStage) + ) + assertThat(insertedRow.getLong("block_number") == rejectedTransaction.blockNumber?.toLong()).isTrue() + assertThat(insertedRow.getJsonArray("overflows").encode()).isEqualTo( + ObjectMapper().writeValueAsString(rejectedTransaction.overflows) + ) + assertThat(insertedRow.getLong("reject_timestamp")).isEqualTo( + rejectedTransaction.timestamp.toEpochMilliseconds() + ) + assertThat(insertedRow.getBuffer("tx_from").bytes).isEqualTo( + rejectedTransaction.transactionInfo.from + ) + assertThat(insertedRow.getBuffer("tx_to").bytes).isEqualTo( + rejectedTransaction.transactionInfo.to + ) + assertThat(insertedRow.getLong("tx_nonce")).isEqualTo( + rejectedTransaction.transactionInfo.nonce.toLong() + ) + } + + @Test + fun `saveNewRejectedTransaction inserts new rejected transaction to db`() { + // insert a new rejected transaction + performInsertTest(createRejectedTransaction()) + + // assert that the total number of rows in the two tables are correct + assertThat(rejectedTransactionsTotalRows()).isEqualTo(1) + assertThat(fullTransactionsTotalRows()).isEqualTo(1) + } + + @Test + fun `saveNewRejectedTransaction inserts new rejected transactions with same txHash but different reason to db`() { + // insert a new rejected transaction + performInsertTest(createRejectedTransaction()) + + // insert another rejected transaction with same txHash but different reason + performInsertTest( + createRejectedTransaction( + txRejectionStage = RejectedTransaction.Stage.P2P, + blockNumber = null, + reasonMessage = "Transaction line count for module MUL=587 is above the limit 401", + overflows = listOf( + ModuleOverflow( + module = "MUL", + count = 587, + limit = 401 + ) + ) + ) + ) + + // assert that the total number of rows in the two tables are correct + assertThat(rejectedTransactionsTotalRows()).isEqualTo(2) + assertThat(fullTransactionsTotalRows()).isEqualTo(1) + } + + @Test + fun `saveNewRejectedTransaction throws error when inserting rejected transactions with same txHash and reason`() { + // insert a new rejected transaction + performInsertTest(createRejectedTransaction()) + + // another rejected transaction with same txHash and reason + val duplicatedRejectedTransaction = createRejectedTransaction( + txRejectionStage = RejectedTransaction.Stage.P2P, + blockNumber = null, + overflows = listOf( + ModuleOverflow( + module = "ADD", + count = 587, + limit = 401 + ) + ) + ) + + // assert that the insertion of duplicatedRejectedTransaction would trigger DuplicatedRecordException error + assertThrows { + rejectedTransactionsPostgresDao.saveNewRejectedTransaction(duplicatedRejectedTransaction).get() + }.also { executionException -> + assertThat(executionException.cause).isInstanceOf(DuplicatedRecordException::class.java) + assertThat(executionException.cause!!.message) + .isEqualTo( + "RejectedTransaction ${duplicatedRejectedTransaction.transactionInfo.hash.encodeHex()} " + + "is already persisted!" + ) + } + + // assert that the total number of rows in the two tables are correct + assertThat(rejectedTransactionsTotalRows()).isEqualTo(1) + assertThat(fullTransactionsTotalRows()).isEqualTo(1) + } + + @Test + fun `findRejectedTransactionByTxHash returns rejected transaction with most recent timestamp from db`() { + // insert a new rejected transaction + val oldestRejectedTransaction = createRejectedTransaction( + timestamp = fakeClock.now().minus(10.seconds) + ) + performInsertTest(oldestRejectedTransaction) + + // insert another rejected transaction with same txHash but different reason and + // with a more recent timestamp + performInsertTest( + createRejectedTransaction( + reasonMessage = "Transaction line count for module MUL=587 is above the limit 401", + timestamp = fakeClock.now().minus(9.seconds) + ) + ) + + // insert another rejected transaction with same txHash but different reason + // and with the most recent timestamp + val newestRejectedTransaction = createRejectedTransaction( + reasonMessage = "Transaction line count for module EXP=9000 is above the limit 8192", + timestamp = fakeClock.now().minus(8.seconds) + ) + performInsertTest(newestRejectedTransaction) + + // find the rejected transaction with the txHash + val foundRejectedTransaction = rejectedTransactionsPostgresDao.findRejectedTransactionByTxHash( + oldestRejectedTransaction.transactionInfo.hash + ).get() + + // assert that the found rejected transaction is the same as the one with most recent timestamp + assertThat(foundRejectedTransaction).isEqualTo(newestRejectedTransaction) + + // assert that the total number of rows in the two tables are correct + assertThat(rejectedTransactionsTotalRows()).isEqualTo(3) + assertThat(fullTransactionsTotalRows()).isEqualTo(1) + } + + @Test + fun `findRejectedTransactionByTxHash returns null as rejected timestamp exceeds queryable window`() { + // insert a new rejected transaction with timestamp exceeds the 1-hour queryable window + val rejectedTransaction = createRejectedTransaction( + timestamp = fakeClock.now().minus(1.hours).minus(1.seconds) + ) + performInsertTest(rejectedTransaction) + + // find the rejected transaction with the txHash + val foundRejectedTransaction = rejectedTransactionsPostgresDao.findRejectedTransactionByTxHash( + rejectedTransaction.transactionInfo.hash, + notRejectedBefore + ).get() + + // assert that null is returned from the find method + assertThat(foundRejectedTransaction).isNull() + + // assert that the total number of rows in the two tables are both one which + // implies the rejected transaction is still present in db + assertThat(rejectedTransactionsTotalRows()).isEqualTo(1) + assertThat(fullTransactionsTotalRows()).isEqualTo(1) + } + + @Test + fun `deleteRejectedTransactions returns 2 row deleted as created timestamp exceeds storage window`() { + // insert a new rejected transaction + performInsertTest( + createRejectedTransaction( + timestamp = fakeClock.now() + ) + ) + // advance the fake clock to make its created timestamp exceeds the 10-hours storage window + fakeClock.advanceBy(1.hours) + + // insert another rejected transaction with same txHash but different reason + performInsertTest( + createRejectedTransaction( + reasonMessage = "Transaction line count for module EXP=9000 is above the limit 8192", + timestamp = fakeClock.now() + ) + ) + // advance the fake clock to make its created timestamp just within the 10-hours storage window + fakeClock.advanceBy(1.hours) + + // insert another rejected transaction with different txHash and reason + performInsertTest( + createRejectedTransaction( + transactionInfo = TransactionInfo( + hash = "0x078ecd6f00bff4beca9116ca85c65ddd265971e415d7df7a96b3c10424b031e2".decodeHex(), + from = "0x4d144d7b9c96b26361d6ac74dd1d8267edca4fc2".decodeHex(), + to = "0x1195cf65f83b3a5768f3c496d3a05ad6412c64b3".decodeHex(), + nonce = 101UL + ), + reasonMessage = "Transaction line count for module EXP=10000 is above the limit 8192", + timestamp = fakeClock.now() + ) + ) + // advance the fake clock to make its created timestamp within the 10-hours storage window + fakeClock.advanceBy(9.hours) + + // assert that the total number of rows in the two tables are both three which + // implies all the rejected transactions above are present in db + assertThat(rejectedTransactionsTotalRows()).isEqualTo(3) + assertThat(fullTransactionsTotalRows()).isEqualTo(2) + + // delete the rejected transactions with storage window as 10 hours from now + val deletedRows = rejectedTransactionsPostgresDao.deleteRejectedTransactions( + fakeClock.now().minus(10.hours) + ).get() + + // assert that number of total deleted rows is just one + assertThat(deletedRows).isEqualTo(1) + + // assert that the total number of rows in the two tables are both two which + // implies only the rejected transactions with created timestamp exceeds + // the storage window was deleted + assertThat(rejectedTransactionsTotalRows()).isEqualTo(2) + assertThat(fullTransactionsTotalRows()).isEqualTo(2) + } +} diff --git a/transaction-exclusion-api/persistence/rejectedtransaction/src/integrationTest/resources/log4j2.xml b/transaction-exclusion-api/persistence/rejectedtransaction/src/integrationTest/resources/log4j2.xml new file mode 100644 index 00000000..c0bd8424 --- /dev/null +++ b/transaction-exclusion-api/persistence/rejectedtransaction/src/integrationTest/resources/log4j2.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/transaction-exclusion-api/persistence/rejectedtransaction/src/main/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RejectedTransactionsDao.kt b/transaction-exclusion-api/persistence/rejectedtransaction/src/main/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RejectedTransactionsDao.kt new file mode 100644 index 00000000..140ab685 --- /dev/null +++ b/transaction-exclusion-api/persistence/rejectedtransaction/src/main/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RejectedTransactionsDao.kt @@ -0,0 +1,20 @@ +package net.consensys.zkevm.persistence.dao.rejectedtransaction + +import kotlinx.datetime.Instant +import net.consensys.linea.transactionexclusion.RejectedTransaction +import tech.pegasys.teku.infrastructure.async.SafeFuture + +interface RejectedTransactionsDao { + fun saveNewRejectedTransaction( + rejectedTransaction: RejectedTransaction + ): SafeFuture + + fun findRejectedTransactionByTxHash( + txHash: ByteArray, + notRejectedBefore: Instant = Instant.DISTANT_PAST + ): SafeFuture + + fun deleteRejectedTransactions( + createdBefore: Instant + ): SafeFuture +} diff --git a/transaction-exclusion-api/persistence/rejectedtransaction/src/main/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RejectedTransactionsPostgresDao.kt b/transaction-exclusion-api/persistence/rejectedtransaction/src/main/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RejectedTransactionsPostgresDao.kt new file mode 100644 index 00000000..5291fe98 --- /dev/null +++ b/transaction-exclusion-api/persistence/rejectedtransaction/src/main/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RejectedTransactionsPostgresDao.kt @@ -0,0 +1,196 @@ +package net.consensys.zkevm.persistence.dao.rejectedtransaction + +import com.fasterxml.jackson.databind.ObjectMapper +import io.vertx.core.Future +import io.vertx.sqlclient.Row +import io.vertx.sqlclient.SqlClient +import io.vertx.sqlclient.Tuple +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant +import net.consensys.encodeHex +import net.consensys.linea.async.toSafeFuture +import net.consensys.linea.transactionexclusion.ModuleOverflow +import net.consensys.linea.transactionexclusion.RejectedTransaction +import net.consensys.linea.transactionexclusion.TransactionInfo +import net.consensys.zkevm.persistence.db.DuplicatedRecordException +import net.consensys.zkevm.persistence.db.SQLQueryLogger +import net.consensys.zkevm.persistence.db.isDuplicateKeyException +import org.apache.logging.log4j.Level +import org.apache.logging.log4j.LogManager +import tech.pegasys.teku.infrastructure.async.SafeFuture + +class RejectedTransactionsPostgresDao( + private val readConnection: SqlClient, + private val writeConnection: SqlClient, + private val clock: Clock = Clock.System +) : RejectedTransactionsDao { + private val log = LogManager.getLogger(this.javaClass.name) + private val queryLog = SQLQueryLogger(log) + + companion object { + // Public instead of internal to allow usage in integrationTest source set + fun rejectedStageToDbValue(txRejectionStage: RejectedTransaction.Stage): String { + return when (txRejectionStage) { + RejectedTransaction.Stage.SEQUENCER -> "SEQ" + RejectedTransaction.Stage.RPC -> "RPC" + RejectedTransaction.Stage.P2P -> "P2P" + } + } + + fun dbValueToRejectedStage(dbStrValue: String): RejectedTransaction.Stage { + return when (dbStrValue) { + "SEQ" -> RejectedTransaction.Stage.SEQUENCER + "RPC" -> RejectedTransaction.Stage.RPC + "P2P" -> RejectedTransaction.Stage.P2P + else -> throw IllegalStateException( + "The db string value: \"$dbStrValue\" cannot be mapped to any RejectedTransaction.Stage enums: " + + RejectedTransaction.Stage.entries.joinToString(",", "[", "]") { it.name } + ) + } + } + + fun parseModuleOverflowListFromJsonString(jsonString: String): List { + return ObjectMapper().readValue( + jsonString, + Array::class.java + ).toList() + } + + fun parseRecord(record: Row): RejectedTransaction { + return RejectedTransaction( + txRejectionStage = record.getString("reject_stage").run(::dbValueToRejectedStage), + timestamp = Instant.fromEpochMilliseconds(record.getLong("reject_timestamp")), + blockNumber = record.getLong("block_number")?.toULong(), + transactionRLP = record.getBuffer("tx_rlp").bytes, + reasonMessage = record.getString("reject_reason"), + overflows = parseModuleOverflowListFromJsonString( + record.getJsonArray("overflows").encode() + ), + transactionInfo = TransactionInfo( + hash = record.getBuffer("tx_hash").bytes, + from = record.getBuffer("tx_from").bytes, + to = record.getBuffer("tx_to").bytes, + nonce = record.getLong("tx_nonce").toULong() + ) + ) + } + + @JvmStatic + val rejectedTransactionsTable = "rejected_transactions" + + @JvmStatic + val fullTransactionsTable = "full_transactions" + } + + private val insertSql = + """ + with x as ( + insert into $rejectedTransactionsTable + (created_epoch_milli, tx_hash, tx_from, tx_to, tx_nonce, + reject_stage, reject_reason, reject_timestamp, block_number, overflows) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, cast($10::text as jsonb)) + returning tx_hash + ) + insert into $fullTransactionsTable + (tx_hash, tx_rlp) + select x.tx_hash, $11 + from x + on conflict on constraint ${fullTransactionsTable}_pkey + do nothing + """ + .trimIndent() + + private val selectSql = + """ + select + $rejectedTransactionsTable.*, + $fullTransactionsTable.tx_rlp as tx_rlp + from $rejectedTransactionsTable + join $fullTransactionsTable on $rejectedTransactionsTable.tx_hash = $fullTransactionsTable.tx_hash + where $fullTransactionsTable.tx_hash = $1 and $rejectedTransactionsTable.reject_timestamp >= $2 + order by $rejectedTransactionsTable.reject_timestamp desc + limit 1 + """ + .trimIndent() + + private val deleteRejectedTransactionsSql = + """ + delete from $rejectedTransactionsTable + where created_epoch_milli < $1 + """ + .trimIndent() + + private val deleteFullTransactionsSql = + """ + delete from $fullTransactionsTable + where tx_hash not in (select x.tx_hash from $rejectedTransactionsTable x) + """ + .trimIndent() + + private val insertSqlQuery = writeConnection.preparedQuery(insertSql) + private val selectSqlQuery = readConnection.preparedQuery(selectSql) + private val deleteRejectedTransactionsSqlQuery = writeConnection.preparedQuery(deleteRejectedTransactionsSql) + private val deleteFullTransactionsSqlQuery = writeConnection.preparedQuery(deleteFullTransactionsSql) + + override fun saveNewRejectedTransaction(rejectedTransaction: RejectedTransaction): SafeFuture { + val params: List = + listOf( + clock.now().toEpochMilliseconds(), + rejectedTransaction.transactionInfo.hash, + rejectedTransaction.transactionInfo.from, + rejectedTransaction.transactionInfo.to, + rejectedTransaction.transactionInfo.nonce.toLong(), + rejectedStageToDbValue(rejectedTransaction.txRejectionStage), + rejectedTransaction.reasonMessage, + rejectedTransaction.timestamp.toEpochMilliseconds(), + rejectedTransaction.blockNumber?.toLong(), + ObjectMapper().writeValueAsString(rejectedTransaction.overflows), + rejectedTransaction.transactionRLP + ) + queryLog.log(Level.TRACE, insertSql, params) + + return insertSqlQuery.execute(Tuple.tuple(params)) + .map { } + .recover { th -> + if (isDuplicateKeyException(th)) { + Future.failedFuture( + DuplicatedRecordException( + "RejectedTransaction ${rejectedTransaction.transactionInfo.hash.encodeHex()} is already persisted!", + th + ) + ) + } else { + Future.failedFuture(th) + } + } + .toSafeFuture() + } + + override fun findRejectedTransactionByTxHash( + txHash: ByteArray, + notRejectedBefore: Instant + ): SafeFuture { + return selectSqlQuery + .execute( + Tuple.of( + txHash, + notRejectedBefore.toEpochMilliseconds() + ) + ) + .toSafeFuture() + .thenApply { rowSet -> rowSet.map(::parseRecord) } + .thenApply { rejectedTxRecords -> rejectedTxRecords.firstOrNull() } + } + + override fun deleteRejectedTransactions( + createdBefore: Instant + ): SafeFuture { + return deleteRejectedTransactionsSqlQuery + .execute(Tuple.of(createdBefore.toEpochMilliseconds())) + .map { rowSet -> rowSet.rowCount() } + .also { + deleteFullTransactionsSqlQuery.execute() + } + .toSafeFuture() + } +} diff --git a/transaction-exclusion-api/persistence/rejectedtransaction/src/main/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RetryingRejectedTransactionsPostgresDao.kt b/transaction-exclusion-api/persistence/rejectedtransaction/src/main/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RetryingRejectedTransactionsPostgresDao.kt new file mode 100644 index 00000000..5a50edd6 --- /dev/null +++ b/transaction-exclusion-api/persistence/rejectedtransaction/src/main/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RetryingRejectedTransactionsPostgresDao.kt @@ -0,0 +1,26 @@ +package net.consensys.zkevm.persistence.dao.rejectedtransaction + +import kotlinx.datetime.Instant +import net.consensys.linea.transactionexclusion.RejectedTransaction +import net.consensys.zkevm.persistence.db.PersistenceRetryer +import tech.pegasys.teku.infrastructure.async.SafeFuture + +class RetryingRejectedTransactionsPostgresDao( + private val delegate: RejectedTransactionsPostgresDao, + private val persistenceRetryer: PersistenceRetryer +) : RejectedTransactionsDao { + override fun saveNewRejectedTransaction(rejectedTransaction: RejectedTransaction): SafeFuture { + return persistenceRetryer.retryQuery({ delegate.saveNewRejectedTransaction(rejectedTransaction) }) + } + + override fun findRejectedTransactionByTxHash( + txHash: ByteArray, + notRejectedBefore: Instant + ): SafeFuture { + return persistenceRetryer.retryQuery({ delegate.findRejectedTransactionByTxHash(txHash, notRejectedBefore) }) + } + + override fun deleteRejectedTransactions(createdBefore: Instant): SafeFuture { + return persistenceRetryer.retryQuery({ delegate.deleteRejectedTransactions(createdBefore) }) + } +} diff --git a/transaction-exclusion-api/persistence/rejectedtransaction/src/main/resources/db/V001__initial_schema.sql b/transaction-exclusion-api/persistence/rejectedtransaction/src/main/resources/db/V001__initial_schema.sql new file mode 100644 index 00000000..9a8e4bc0 --- /dev/null +++ b/transaction-exclusion-api/persistence/rejectedtransaction/src/main/resources/db/V001__initial_schema.sql @@ -0,0 +1,26 @@ +-- ======================================================= +-- full_transactions table +-- ======================================================= +create table if not exists full_transactions ( + tx_hash bytea, + tx_rlp bytea, + primary key (tx_hash) +); + +-- ======================================================= +-- rejected_transactions table +-- ======================================================= +create table if not exists rejected_transactions ( + created_epoch_milli bigint, + tx_hash bytea, + tx_from bytea, + tx_to bytea, + tx_nonce bigint, + reject_stage varchar(5), -- SEQ,RPC,P2P + reject_reason varchar(256), + reject_timestamp bigint, + block_number bigint, + overflows jsonb, + primary key (tx_hash, reject_reason), + foreign key (tx_hash) references full_transactions +); diff --git a/transaction-exclusion-api/persistence/rejectedtransaction/src/test/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RetryingRejectedTransactionsPostgresDaoTest.kt b/transaction-exclusion-api/persistence/rejectedtransaction/src/test/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RetryingRejectedTransactionsPostgresDaoTest.kt new file mode 100644 index 00000000..b21dbf81 --- /dev/null +++ b/transaction-exclusion-api/persistence/rejectedtransaction/src/test/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RetryingRejectedTransactionsPostgresDaoTest.kt @@ -0,0 +1,76 @@ +package net.consensys.zkevm.persistence.dao.rejectedtransaction + +import io.vertx.core.Vertx +import io.vertx.junit5.VertxExtension +import kotlinx.datetime.Clock +import net.consensys.linea.transactionexclusion.test.defaultRejectedTransaction +import net.consensys.zkevm.persistence.db.PersistenceRetryer +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Mockito.mock +import org.mockito.Mockito.times +import org.mockito.kotlin.eq +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import tech.pegasys.teku.infrastructure.async.SafeFuture +import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.hours +import kotlin.time.Duration.Companion.milliseconds + +@ExtendWith(VertxExtension::class) +class RetryingRejectedTransactionsPostgresDaoTest { + private lateinit var retryingRejectedTransactionsPostgresDao: RetryingRejectedTransactionsPostgresDao + private val delegateRejectedTransactionsDao = mock() + private val now = Clock.System.now() + private val notRejectedBefore = now.minus(24.hours) + private val createdBefore = now.minus(7.days) + private val rejectedTransaction = defaultRejectedTransaction + + @BeforeEach + fun beforeEach(vertx: Vertx) { + retryingRejectedTransactionsPostgresDao = RetryingRejectedTransactionsPostgresDao( + delegate = delegateRejectedTransactionsDao, + PersistenceRetryer( + vertx = vertx, + PersistenceRetryer.Config( + backoffDelay = 1.milliseconds + ) + ) + ) + + whenever(delegateRejectedTransactionsDao.saveNewRejectedTransaction(eq(rejectedTransaction))) + .thenReturn(SafeFuture.completedFuture(Unit)) + + whenever( + delegateRejectedTransactionsDao.findRejectedTransactionByTxHash( + eq(rejectedTransaction.transactionInfo.hash), + eq(notRejectedBefore) + ) + ) + .thenReturn(SafeFuture.completedFuture(null)) + + whenever(delegateRejectedTransactionsDao.deleteRejectedTransactions(eq(createdBefore))) + .thenReturn(SafeFuture.completedFuture(0)) + } + + @Test + fun `retrying rejected transactions dao should delegate all queries to standard dao`() { + retryingRejectedTransactionsPostgresDao.saveNewRejectedTransaction(rejectedTransaction) + verify(delegateRejectedTransactionsDao, times(1)).saveNewRejectedTransaction(eq(rejectedTransaction)) + + retryingRejectedTransactionsPostgresDao.findRejectedTransactionByTxHash( + rejectedTransaction.transactionInfo.hash, + notRejectedBefore + ) + verify(delegateRejectedTransactionsDao, times(1)).findRejectedTransactionByTxHash( + eq(rejectedTransaction.transactionInfo.hash), + eq(notRejectedBefore) + ) + + retryingRejectedTransactionsPostgresDao.deleteRejectedTransactions(createdBefore) + verify(delegateRejectedTransactionsDao, times(1)).deleteRejectedTransactions( + eq(createdBefore) + ) + } +}