feat: improve tx-exclusion-api performance and increase size of reaso… (#327)

* feat: improve tx-exclusion-api performance and increase size of reason message column

* feat: set waitForTcpPorts as true for localStackPostgresDbOnly and update prover-v3 image tag in compose

* feat: update prover-v3 image tag to a more recent one

* feat: update coordinator image tag to a more recent one

build.gradle

@@ -162,7 +162,7 @@ dockerCompose {
     ]
     useComposeFiles = ["${project.rootDir.path}/docker/compose.yml"]
     waitForHealthyStateTimeout = Duration.ofMinutes(3)
-    waitForTcpPorts = false
+    waitForTcpPorts = true
     removeOrphans = true
     noRecreate = true
     projectName = "docker"

docker/compose-local-dev-traces-v2.overrides.yml

@@ -68,7 +68,7 @@ services:
   prover-v3: # prover compatible with the traces from zkbesu
     container_name: prover-v3
     hostname: prover-v3
-    image: consensys/linea-prover:${PROVER_TAG:-bba9677}
+    image: consensys/linea-prover:${PROVER_TAG:-c4abdd9}
     platform: linux/amd64
     # to avoid spinning up on CI for now
     profiles: [ "l2" ]

docker/compose.yml

@@ -330,7 +330,7 @@ services:
   coordinator:
     hostname: coordinator
     container_name: coordinator
-    image: consensys/linea-coordinator:${COORDINATOR_TAG:-2ee4363}
+    image: consensys/linea-coordinator:${COORDINATOR_TAG:-c4abdd9}
     platform: linux/amd64
     profiles: [ "l2", "debug" ]
     depends_on:

jvm-libs/generic/persistence/db/src/testFixtures/kotlin/net/consensys/zkevm/persistence/db/test/CleanDbTestSuiteParallel.kt

@@ -26,8 +26,6 @@ abstract class CleanDbTestSuiteParallel {
 
   private fun createDataSource(databaseName: String): DataSource {
     return PGSimpleDataSource().also {
-      it.ssl = false
-      it.gssEncMode = "disable"
       it.serverNames = arrayOf(host)
       it.portNumbers = intArrayOf(port)
       it.databaseName = databaseName

transaction-exclusion-api/app/src/integrationTest/kotlin/net/consensys/linea/transactionexclusion/TransactionExclusionAppTest.kt

@@ -32,7 +32,7 @@ import kotlin.random.Random
 @ExtendWith(VertxExtension::class)
 class TransactionExclusionAppTest : CleanDbTestSuiteParallel() {
   init {
-    target = "2"
+    target = "3"
   }
 
   override var databaseName = DbHelper.generateUniqueDbName("tx-exclusion-api-app-tests")

transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/TransactionExclusionApp.kt

@@ -159,7 +159,7 @@ class TransactionExclusionApp(config: AppConfig) {
     readPipeliningLimit: Int,
     skipMigration: Boolean = false
   ): SqlClient {
-    val dbVersion = "2"
+    val dbVersion = "3"
     if (!skipMigration) {
       Db.applyDbMigrations(
         host = connectionConfig.host,

transaction-exclusion-api/app/src/main/kotlin/net/consensys/linea/transactionexclusion/app/api/ArgumentParser.kt

@@ -13,7 +13,7 @@ import org.hyperledger.besu.ethereum.core.encoding.TransactionDecoder
 import java.time.format.DateTimeFormatter
 import java.time.format.DateTimeParseException
 
-const val MAX_REASON_MESSAGE_STR_LEN = 512
+const val MAX_REASON_MESSAGE_STR_LEN = 1024
 
 object ArgumentParser {
   fun getTransactionRLPInRawBytes(transactionRLP: String): ByteArray {

transaction-exclusion-api/app/src/test/kotlin/net/consensys/linea/transactionexclusion/app/api/ArgumentParserTest.kt

@@ -216,15 +216,15 @@ class ArgumentParserTest {
   }
 
   @Test
-  fun getReasonMessage_should_throw_error_for_string_length_longer_than_512() {
-    // reason message string with more than 512 characters
+  fun getReasonMessage_should_throw_error_for_string_length_longer_than_1024() {
+    // reason message string with more than 1024 characters
     assertThrows<IllegalArgumentException> {
       ArgumentParser.getReasonMessage(
-        Random.Default.nextBytes(256).encodeHex(prefix = false) + "0"
+        Random.Default.nextBytes(512).encodeHex(prefix = false) + "0"
       )
     }.also { error ->
       Assertions.assertTrue(
-        error.message!!.contains("Reason message should not be more than 512 characters")
+        error.message!!.contains("Reason message should not be more than 1024 characters")
       )
     }
   }

transaction-exclusion-api/persistence/rejectedtransaction/src/integrationTest/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RejectedTransactionsPostgresDaoTest.kt

@@ -32,7 +32,7 @@ import kotlin.time.Duration.Companion.seconds
 @ExtendWith(VertxExtension::class)
 class RejectedTransactionsPostgresDaoTest : CleanDbTestSuiteParallel() {
   init {
-    target = "2"
+    target = "3"
   }
 
   override val databaseName = DbHelper.generateUniqueDbName("tx-exclusion-api-rejectedtxns-dao-tests")
@@ -275,26 +275,7 @@ class RejectedTransactionsPostgresDaoTest : CleanDbTestSuiteParallel() {
 
   @Test
   fun `deleteRejectedTransactions returns 2 row deleted as created timestamp exceeds storage window`() {
-    // insert a new rejected transaction
-    performInsertTest(
-      createRejectedTransaction(
-        timestamp = fakeClock.now()
-      )
-    )
-    // advance the fake clock to make its created timestamp exceeds the 10-hours storage window
-    fakeClock.advanceBy(1.hours)
-
-    // insert another rejected transaction with same txHash but different reason
-    performInsertTest(
-      createRejectedTransaction(
-        reasonMessage = "Transaction line count for module EXP=9000 is above the limit 8192",
-        timestamp = fakeClock.now()
-      )
-    )
-    // advance the fake clock to make its created timestamp just within the 10-hours storage window
-    fakeClock.advanceBy(1.hours)
-
-    // insert another rejected transaction with different txHash and reason
+    // insert a new rejected transaction A
     performInsertTest(
       createRejectedTransaction(
         transactionInfo = TransactionInfo(
@@ -307,11 +288,30 @@ class RejectedTransactionsPostgresDaoTest : CleanDbTestSuiteParallel() {
         timestamp = fakeClock.now()
       )
     )
-    // advance the fake clock to make its created timestamp within the 10-hours storage window
-    fakeClock.advanceBy(9.hours)
+    // advance the fake clock to make its created timestamp exceeds the 10-hours storage window
+    fakeClock.advanceBy(1.hours)
 
-    // assert that the total number of rows in the two tables are both three which
-    // implies all the rejected transactions above are present in db
+    // insert another rejected transaction B1
+    performInsertTest(
+      createRejectedTransaction(
+        timestamp = fakeClock.now()
+      )
+    )
+    // advance the fake clock to make its created timestamp exceeds the 10-hours storage window
+    fakeClock.advanceBy(1.hours)
+
+    // insert another rejected transaction B2 with same txHash as B1 but different reason
+    performInsertTest(
+      createRejectedTransaction(
+        reasonMessage = "Transaction line count for module EXP=9000 is above the limit 8192",
+        timestamp = fakeClock.now()
+      )
+    )
+    // advance the fake clock to make its created timestamp stay within the 10-hours storage window
+    fakeClock.advanceBy(10.hours)
+
+    // assert that the total number of rows in the two tables are 3 and 2 respectively
+    // which implies all the rejected transactions above are present in db
     assertThat(rejectedTransactionsTotalRows()).isEqualTo(3)
     assertThat(fullTransactionsTotalRows()).isEqualTo(2)
 
@@ -320,13 +320,13 @@ class RejectedTransactionsPostgresDaoTest : CleanDbTestSuiteParallel() {
       fakeClock.now().minus(10.hours)
     ).get()
 
-    // assert that number of total deleted rows is just one
-    assertThat(deletedRows).isEqualTo(1)
+    // assert that number of total deleted rows in rejected_transactions table is 2
+    assertThat(deletedRows).isEqualTo(2)
 
-    // assert that the total number of rows in the two tables are both two which
-    // implies only the rejected transactions with created timestamp exceeds
-    // the storage window was deleted
-    assertThat(rejectedTransactionsTotalRows()).isEqualTo(2)
-    assertThat(fullTransactionsTotalRows()).isEqualTo(2)
+    // assert that the total number of rows in the two tables are both 1 which
+    // implies only the rejected transaction A and B1 and A's corresponding full transaction
+    // were deleted due to created timestamp exceeds the storage window
+    assertThat(rejectedTransactionsTotalRows()).isEqualTo(1)
+    assertThat(fullTransactionsTotalRows()).isEqualTo(1)
   }
 }

transaction-exclusion-api/persistence/rejectedtransaction/src/main/kotlin/net/consensys/zkevm/persistence/dao/rejectedtransaction/RejectedTransactionsPostgresDao.kt

@@ -123,7 +123,7 @@ class RejectedTransactionsPostgresDao(
   private val deleteFullTransactionsSql =
     """
       delete from $fullTransactionsTable
-      where tx_hash not in (select x.tx_hash from $rejectedTransactionsTable x)
+      f where not exists (select null from $rejectedTransactionsTable x where f.tx_hash = x.tx_hash)
     """
       .trimIndent()
 

transaction-exclusion-api/persistence/rejectedtransaction/src/main/resources/db/V003__increase_reject_reason_size_and_add_index_on_tx_hash_schema.sql

@@ -0,0 +1,6 @@
+-- =======================================================
+-- rejected_transactions table
+-- =======================================================
+alter table if exists rejected_transactions
+  alter column reject_reason type varchar(1024);
+create index if not exists tx_hash_idx on rejected_transactions using btree (tx_hash);