implemented crude version of extract_geth_traces.py

ethereumetl/jobs/extract_geth_traces_job.py

@@ -0,0 +1,59 @@
+# MIT License
+#
+# Copyright (c) 2018 Evgeniy Filatov, evgeniyfilatov@gmail.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from ethereumetl.executors.batch_work_executor import BatchWorkExecutor
+from ethereumetl.jobs.base_job import BaseJob
+from ethereumetl.mappers.trace_mapper import EthTraceMapper
+from ethereumetl.mappers.geth_trace_mapper import EthGethTraceMapper
+
+
+class ExtractGethTracesJob(BaseJob):
+    def __init__(
+            self,
+            traces_iterable,
+            batch_size,
+            max_workers,
+            item_exporter):
+        self.traces_iterable = traces_iterable
+
+        self.batch_work_executor = BatchWorkExecutor(batch_size, max_workers)
+        self.item_exporter = item_exporter
+
+        self.trace_mapper = EthTraceMapper()
+        self.geth_trace_mapper = EthGethTraceMapper()
+
+    def _start(self):
+        self.item_exporter.open()
+
+    def _export(self):
+        self.batch_work_executor.execute(self.traces_iterable, self._extract_geth_traces)
+
+    def _extract_geth_traces(self, geth_traces):
+        for geth_trace_dict in geth_traces:
+            geth_trace = self.geth_trace_mapper.json_dict_to_geth_trace(geth_trace_dict)
+            traces = self.trace_mapper.geth_trace_to_traces(geth_trace)
+            for trace in traces:
+                self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace))
+                
+    def _end(self):
+        self.batch_work_executor.shutdown()
+        self.item_exporter.close()

ethereumetl/mappers/trace_mapper.py

@@ -77,6 +77,61 @@ class EthTraceMapper(object):
 
         return trace
 
+    def geth_trace_to_traces(self, geth_trace):
+        block_number = geth_trace.block_number
+        transaction_traces = geth_trace.traces
+
+        traces = []
+
+        for tx_index, tx_trace in enumerate(transaction_traces):
+            traces.extend(self._iterate_transaction_trace(
+                block_number,
+                tx_index,
+                tx_trace.get('result')
+            ))
+
+        return traces
+
+    def _iterate_transaction_trace(self, block_number, tx_index, tx_trace, trace_address=[]):
+        trace = EthTrace()
+
+        trace.block_number = block_number
+        trace.transaction_hash = tx_index
+
+        trace.from_address = to_normalized_address(tx_trace.get('from', None))
+        trace.to_address = to_normalized_address(tx_trace.get('to', None))
+        trace.value = hex_to_dec(tx_trace.get('value', None))
+        trace.input = tx_trace.get('input', None)
+        trace.output = tx_trace.get('output', None)
+        trace.gas = hex_to_dec(tx_trace.get('gas', None))
+        trace.gas_used = hex_to_dec(tx_trace.get('gasUsed', None))
+        trace.subtraces = len(tx_trace.get('calls', []))
+        trace.trace_type = tx_trace.get('type', None).lower() # TODO: map
+        # TODO: normal error handling
+        trace.error = tx_trace.get('error', None)
+        if trace.trace_type == 'selfdestruct':
+            trace.trace_type = 'suicide'
+        if trace.trace_type == 'create':
+            trace.to_address = to_normalized_address(0)
+            trace.contract_address = tx_trace.get('to', None)
+            # TODO: fix in parity traces
+            trace.output = ''
+        trace.trace_address = trace_address
+
+        result = [trace]
+
+        calls = tx_trace.get('calls', [])
+
+        for call_index, call_trace in enumerate(calls):
+            result.extend(self._iterate_transaction_trace(
+                block_number,
+                tx_index,
+                call_trace,
+                trace_address + [call_index]
+            ))
+
+        return result
+
     def trace_to_dict(self, trace):
         return {
             'type': 'trace',

extract_geth_traces.py

@@ -0,0 +1,55 @@
+# MIT License
+#
+# Copyright (c) 2018 Evgeniy Filatov, evgebiyfilatov@gmail.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+import argparse
+import csv
+import json
+
+from ethereumetl.file_utils import smart_open
+from ethereumetl.jobs.exporters.traces_item_exporter import traces_item_exporter
+from ethereumetl.jobs.extract_geth_traces_job import ExtractGethTracesJob
+from ethereumetl.logging_utils import logging_basic_config
+
+logging_basic_config()
+
+parser = argparse.ArgumentParser(
+    description='Extracts geth traces from file generated by export_geth_traces.py')
+parser.add_argument('-i', '--input', type=str, required=True, help='The JSON file containing geth traces.')
+parser.add_argument('-b', '--batch-size', default=100, type=int, help='The number of blocks to filter at a time.')
+parser.add_argument('-o', '--output', default='-', type=str, help='The output file. If not specified stdout is used.')
+parser.add_argument('-w', '--max-workers', default=5, type=int, help='The maximum number of workers.')
+
+args = parser.parse_args()
+
+with smart_open(args.input, 'r') as geth_traces_file:
+    if args.input.endswith('.json'):
+        traces_iterable = (json.loads(line) for line in geth_traces_file)
+    else:
+        traces_iterable = (trace for trace in csv.DictReader(geth_traces_file))
+    job = ExtractGethTracesJob(
+        traces_iterable=traces_iterable,
+        batch_size=args.batch_size,
+        max_workers=args.max_workers,
+        item_exporter=traces_item_exporter(args.output))
+
+    job.run()

tests/ethereumetl/job/test_extract_geth_traces_job.py

@@ -0,0 +1,60 @@
+# MIT License
+#
+# Copyright (c) 2018 Evgeniy Filatov, evgeniyfilatov@gmail.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import json
+import io
+
+import pytest
+
+import tests.resources
+from ethereumetl.jobs.exporters.traces_item_exporter import traces_item_exporter
+from ethereumetl.jobs.extract_geth_traces_job import ExtractGethTracesJob
+from tests.helpers import compare_lines_ignore_order, read_file
+
+RESOURCE_GROUP = 'test_extract_geth_traces_job'
+
+def read_resource(resource_group, file_name):
+    return tests.resources.read_resource([RESOURCE_GROUP, resource_group], file_name)
+
+@pytest.mark.parametrize('resource_group', [
+    'block_without_transactions',
+    'block_with_create',
+    'block_with_suicide',
+    'block_with_subtraces',
+    'block_with_error',
+])
+def test_extract_traces_job(tmpdir, resource_group):
+    output_file = tmpdir.join('actual_traces.csv')
+
+    geth_traces_content = read_resource(resource_group, 'geth_traces.json')
+    traces_iterable = (json.loads(line) for line in geth_traces_content.splitlines())
+    job = ExtractGethTracesJob(
+        traces_iterable=traces_iterable,
+        batch_size=2,
+        item_exporter=traces_item_exporter(output_file),
+        max_workers=5
+    )
+    job.run()
+
+    compare_lines_ignore_order(
+        read_resource(resource_group, 'expected_traces.csv'), read_file(output_file)
+    )

tests/resources/test_extract_geth_traces_job/block_with_create/expected_traces.csv

@@ -0,0 +1,3 @@
+block_number,transaction_hash,from_address,to_address,value,contract_address,input,output,trace_type,gas,gas_used,subtraces,trace_address,error
+1000690,0,0xaf21e07e5a929d16026a7b4d88f3906a8d2e4942,0x5b3c526b152b1f3d8eabe2ec27f49b904ad51cad,64655529900000002048,,0x,0x,call,0,0,0,,
+1000690,1,0xacdee28d8ca76187883831a37f551a5904cdf191,0,0,0xa7e3cf952ea8d9438a26ee346c295f1ada328ae1,0x606060405260026101086000505560405161015638038061015683398101604052805160805160a051919092019190808383815160019081018155600090600160a060020a0332169060029060038390559183525061010260205260408220555b82518110156100eb57828181518110156100025790602001906020020151600160a060020a03166002600050826002016101008110156100025790900160005081905550806002016101026000506000858481518110156100025790602001906020020151600160a060020a0316815260200190815260200160002060005081905550600101610060565b81600060005081905550505050806101056000508190555061010f62015180420490565b61010755505050506031806101256000396000f3003660008037602060003660003473273930d21e01ee25e4c219b63259d214872220a261235a5a03f21560015760206000f30000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000052b7d2dcc80cd2e40000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000acdee28d8ca76187883831a37f551a5904cdf191,,create,954720,160631,0,,

tests/resources/test_extract_geth_traces_job/block_with_create/geth_traces.json

@@ -0,0 +1 @@
+{"block_number": 1000690, "traces": [{"result": {"type": "CALL", "from": "0xaf21e07e5a929d16026a7b4d88f3906a8d2e4942", "to": "0x5b3c526b152b1f3d8eabe2ec27f49b904ad51cad", "value": "0x3814695e26625c000", "gas": "0x0", "gasUsed": "0x0", "input": "0x", "output": "0x", "time": "5.168\u00b5s"}}, {"result": {"type": "CREATE", "from": "0xacdee28d8ca76187883831a37f551a5904cdf191", "to": "0xa7e3cf952ea8d9438a26ee346c295f1ada328ae1", "value": "0x0", "gas": "0xe9160", "gasUsed": "0x27377", "input": "0x606060405260026101086000505560405161015638038061015683398101604052805160805160a051919092019190808383815160019081018155600090600160a060020a0332169060029060038390559183525061010260205260408220555b82518110156100eb57828181518110156100025790602001906020020151600160a060020a03166002600050826002016101008110156100025790900160005081905550806002016101026000506000858481518110156100025790602001906020020151600160a060020a0316815260200190815260200160002060005081905550600101610060565b81600060005081905550505050806101056000508190555061010f62015180420490565b61010755505050506031806101256000396000f3003660008037602060003660003473273930d21e01ee25e4c219b63259d214872220a261235a5a03f21560015760206000f30000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000052b7d2dcc80cd2e40000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000acdee28d8ca76187883831a37f551a5904cdf191", "output": "0x3660008037602060003660003473273930d21e01ee25e4c219b63259d214872220a261235a5a03f21560015760206000f3", "time": "1.88608ms"}}]}

tests/resources/test_extract_geth_traces_job/block_with_error/expected_traces.csv

@@ -0,0 +1,3 @@
+block_number,transaction_hash,from_address,to_address,value,contract_address,input,output,trace_type,gas,gas_used,subtraces,trace_address,error
+1000895,0,0xad9253df75b066c67aff5cdd9d6d2b9245444726,0x627da06356442122f08e2203c749978151e55800,1000000000000000000,,0x,,call,0,0,0,,out of gas
+1000895,1,0x9288fe5be3be048b5c7a68bfd4b9a0746b7e4a00,0xe05ff93a9978bbb48356accc74088f3841fc5d72,1100000000000000000,,0x,0x,call,100000,0,0,,

tests/resources/test_extract_geth_traces_job/block_with_error/geth_traces.json

@@ -0,0 +1 @@
+{"block_number": 1000895, "traces": [{"result": {"type": "CALL", "from": "0xad9253df75b066c67aff5cdd9d6d2b9245444726", "to": "0x627da06356442122f08e2203c749978151e55800", "value": "0xde0b6b3a7640000", "gas": "0x0", "gasUsed": "0x0", "input": "0x", "error": "out of gas", "time": "46.051\u00b5s"}}, {"result": {"type": "CALL", "from": "0x9288fe5be3be048b5c7a68bfd4b9a0746b7e4a00", "to": "0xe05ff93a9978bbb48356accc74088f3841fc5d72", "value": "0xf43fc2c04ee0000", "gas": "0x186a0", "gasUsed": "0x0", "input": "0x", "output": "0x", "time": "4.057\u00b5s"}}]}

tests/resources/test_extract_geth_traces_job/block_with_subtraces/expected_traces.csv

@@ -0,0 +1,4 @@
+block_number,transaction_hash,from_address,to_address,value,contract_address,input,output,trace_type,gas,gas_used,subtraces,trace_address,error
+1000000,0,0x39fa8c5f2793459d6622857e7d9fbb4bd91766d3,0xc083e9947cf02b8ffc7d3090ae9aea72df98fd47,100000000000000000000,,0x,0x0000000000000000000000000000000000000000000000000000000000000000,call,108244,8244,1,,
+1000000,0,0xc083e9947cf02b8ffc7d3090ae9aea72df98fd47,0x273930d21e01ee25e4c219b63259d214872220a2,100000000000000000000,,0x,0x0000000000000000000000000000000000000000000000000000000000000000,callcode,101462,1444,0,0,
+1000000,1,0x32be343b94f860124dc4fee278fdcbd38c102d88,0xdf190dc7190dfba737d7777a163445b7fff16133,437194980000000000,,0x,0x,call,29000,0,0,,

tests/resources/test_extract_geth_traces_job/block_with_subtraces/geth_traces.json

@@ -0,0 +1 @@
+{"block_number": 1000000, "traces": [{"result": {"type": "CALL", "from": "0x39fa8c5f2793459d6622857e7d9fbb4bd91766d3", "to": "0xc083e9947cf02b8ffc7d3090ae9aea72df98fd47", "value": "0x56bc75e2d63100000", "gas": "0x1a6d4", "gasUsed": "0x2034", "input": "0x", "output": "0x0000000000000000000000000000000000000000000000000000000000000000", "time": "1.568749ms", "calls": [{"type": "CALLCODE", "from": "0xc083e9947cf02b8ffc7d3090ae9aea72df98fd47", "to": "0x273930d21e01ee25e4c219b63259d214872220a2", "value": "0x56bc75e2d63100000", "gas": "0x18c56", "gasUsed": "0x5a4", "input": "0x", "output": "0x0000000000000000000000000000000000000000000000000000000000000000"}]}}, {"result": {"type": "CALL", "from": "0x32be343b94f860124dc4fee278fdcbd38c102d88", "to": "0xdf190dc7190dfba737d7777a163445b7fff16133", "value": "0x6113a84987be800", "gas": "0x7148", "gasUsed": "0x0", "input": "0x", "output": "0x", "time": "6.567\u00b5s"}}]}

tests/resources/test_extract_geth_traces_job/block_with_suicide/expected_traces.csv

@@ -0,0 +1,3 @@
+block_number,transaction_hash,from_address,to_address,value,contract_address,input,output,trace_type,gas,gas_used,subtraces,trace_address,error
+1011973,0,0x83973747eec131bf9a08ac64fb1a518e891bdf4b,0x474faa5018639791952fae334e2911700ac7fe9b,0,,0x41c0e1b5,0x,call,68728,232,1,,
+1011973,0,,,,,,,suicide,,,0,0,

tests/resources/test_extract_geth_traces_job/block_with_suicide/geth_traces.json

@@ -0,0 +1 @@
+{"block_number": 1011973, "traces": [{"result": {"type": "CALL", "from": "0x83973747eec131bf9a08ac64fb1a518e891bdf4b", "to": "0x474faa5018639791952fae334e2911700ac7fe9b", "value": "0x0", "gas": "0x10c78", "gasUsed": "0xe8", "input": "0x41c0e1b5", "output": "0x", "time": "1.088838ms", "calls": [{"type": "SELFDESTRUCT"}]}}]}

tests/resources/test_extract_geth_traces_job/block_without_transactions/geth_traces.json

@@ -0,0 +1 @@
+{"block_number": 1, "traces": []}