Merge pull request #128 from blockchain-etl/develop

Traces

.dockerignore

@@ -0,0 +1,3 @@
+.*
+last_synced_block.txt
+output

.travis.yml

@@ -2,6 +2,6 @@ language: python
 python:
   - "3.6"
 install:
-  - travis_retry pip install -r requirements.txt
+  - travis_retry pip install -e . -r requirements-dev.txt
 script:
   - pytest -vv

Dockerfile

@@ -4,9 +4,8 @@ ENV PROJECT_DIR=ethereum-etl
 
 RUN mkdir /$PROJECT_DIR
 WORKDIR /$PROJECT_DIR
-COPY requirements.txt .
-RUN apk add --no-cache gcc musl-dev  #for C libraries: <limits.h> <stdio.h>
-RUN pip install --upgrade pip && pip install -r /$PROJECT_DIR/requirements.txt
 COPY . .
+RUN apk add --no-cache gcc musl-dev  #for C libraries: <limits.h> <stdio.h>
+RUN pip install --upgrade pip && pip install -e /$PROJECT_DIR/
 
-ENTRYPOINT ["python", "export_all.py"]
+ENTRYPOINT ["python", "ethereumetl"]

README.md

@@ -292,8 +292,8 @@ Read this article for details https://medium.com/@medvedev1088/how-to-export-the
 
 1. Run a container out of the image
     ```bash
-    > docker run -v $HOME/output:/ethereum-etl/output ethereum-etl:latest -s 0 -e 5499999 -b 100000 -p https://mainnet.infura.io
-    > docker run -v $HOME/output:/ethereum-etl/output ethereum-etl:latest -s 2018-01-01 -e 2018-01-01 -b 100000 -p https://mainnet.infura.io
+    > docker run -v $HOME/output:/ethereum-etl/output ethereum-etl:latest export_all -s 0 -e 5499999 -b 100000 -p https://mainnet.infura.io
+    > docker run -v $HOME/output:/ethereum-etl/output ethereum-etl:latest export_all -s 2018-01-01 -e 2018-01-01 -p https://mainnet.infura.io
     ```
 
 ### Command Reference
@@ -491,7 +491,7 @@ You can tune `--batch-size`, `--max-workers` for performance.
 #### Running Tests
 
 ```bash
-> pip install -e . -r requirements.txt
+> pip install -e . -r requirements-dev.txt
 > export ETHEREUM_ETL_RUN_SLOW_TESTS=True
 > pytest -vv
 ```

ethereumetl/cli/export_all.py

@@ -108,6 +108,6 @@ def get_partitions(start, end, partition_batch_size, provider_uri):
 @click.option('-w', '--max-workers', default=5, type=int, help='The maximum number of workers.')
 @click.option('-B', '--export-batch-size', default=100, type=int, help='The number of requests in JSON RPC batches.')
 def export_all(start, end, partition_batch_size, provider_uri, output_dir, max_workers, export_batch_size):
-    """Exports all for a range of blocks."""
+    """Exports all data for a range of blocks."""
     export_all_common(get_partitions(start, end, partition_batch_size, provider_uri),
                       output_dir, provider_uri, max_workers, export_batch_size)

ethereumetl/cli/export_traces.py

@@ -43,7 +43,10 @@ logging_basic_config()
 @click.option('-p', '--provider-uri', required=True, type=str,
               help='The URI of the web3 provider e.g. '
                    'file://$HOME/.local/share/io.parity.ethereum/jsonrpc.ipc or http://localhost:8545/')
-def export_traces(start_block, end_block, batch_size, output, max_workers, provider_uri):
+@click.option('--genesis-traces/--no-genesis-traces', default=False, help='Whether to include genesis traces')
+@click.option('--daofork-traces/--no-daofork-traces', default=False, help='Whether to include daofork traces')
+def export_traces(start_block, end_block, batch_size, output, max_workers, provider_uri,
+                  genesis_traces, daofork_traces):
     """Exports traces from parity node."""
     job = ExportTracesJob(
         start_block=start_block,
@@ -51,6 +54,8 @@ def export_traces(start_block, end_block, batch_size, output, max_workers, provi
         batch_size=batch_size,
         web3=ThreadLocalProxy(lambda: Web3(get_provider_from_uri(provider_uri))),
         item_exporter=traces_item_exporter(output),
-        max_workers=max_workers)
+        max_workers=max_workers,
+        include_genesis_traces=genesis_traces,
+        include_daofork_traces=daofork_traces)
 
     job.run()

ethereumetl/cli/get_block_range_for_date.py

@@ -21,14 +21,15 @@
 # SOFTWARE.
 
 
-from datetime import datetime
-
 import click
+
+from datetime import datetime
 from web3 import Web3
 
+from ethereumetl.file_utils import smart_open
 from ethereumetl.logging_utils import logging_basic_config
-from ethereumetl.providers.auto import get_provider_from_uri
 from ethereumetl.service.eth_service import EthService
+from ethereumetl.providers.auto import get_provider_from_uri
 
 logging_basic_config()
 
@@ -48,5 +49,5 @@ def get_block_range_for_date(provider_uri, date, output):
 
     start_block, end_block = eth_service.get_block_range_for_date(date)
 
-    with click.open_file(output, 'w') as output_file:
+    with smart_open(output, 'w') as output_file:
         output_file.write('{},{}\n'.format(start_block, end_block))

ethereumetl/domain/trace.py

@@ -36,6 +36,6 @@ class EthTrace(object):
         self.reward_type = None
         self.gas = None
         self.gas_used = None
-        self.subtraces = None
+        self.subtraces = 0
         self.trace_address = None
         self.error = None

ethereumetl/jobs/export_traces_job.py

@@ -22,8 +22,10 @@
 
 from ethereumetl.executors.batch_work_executor import BatchWorkExecutor
 from ethereumetl.jobs.base_job import BaseJob
-from ethereumetl.utils import validate_range
+from ethereumetl.mainnet_daofork_state_changes import DAOFORK_BLOCK_NUMBER
 from ethereumetl.mappers.trace_mapper import EthTraceMapper
+from ethereumetl.service.eth_special_trace_service import EthSpecialTraceService
+from ethereumetl.utils import validate_range
 
 
 class ExportTracesJob(BaseJob):
@@ -34,7 +36,9 @@ class ExportTracesJob(BaseJob):
             batch_size,
             web3,
             item_exporter,
-            max_workers):
+            max_workers,
+            include_genesis_traces=False,
+            include_daofork_traces=False):
         validate_range(start_block, end_block)
         self.start_block = start_block
         self.end_block = end_block
@@ -47,6 +51,10 @@ class ExportTracesJob(BaseJob):
 
         self.trace_mapper = EthTraceMapper()
 
+        self.special_trace_service = EthSpecialTraceService()
+        self.include_genesis_traces = include_genesis_traces
+        self.include_daofork_traces = include_daofork_traces
+
     def _start(self):
         self.item_exporter.open()
 
@@ -63,10 +71,23 @@ class ExportTracesJob(BaseJob):
         assert len(block_number_batch) == 1
         block_number = block_number_batch[0]
 
+        if self.include_genesis_traces and 0 in block_number_batch:
+            genesis_traces = self.special_trace_service.get_genesis_traces()
+            for trace in genesis_traces:
+                self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace))
+
+        if self.include_daofork_traces and DAOFORK_BLOCK_NUMBER in block_number_batch:
+            daofork_traces = self.special_trace_service.get_daofork_traces()
+            for trace in daofork_traces:
+                self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace))
+
         # TODO: Change to traceFilter when this issue is fixed
         # https://github.com/paritytech/parity-ethereum/issues/9822
         json_traces = self.web3.parity.traceBlock(block_number)
 
+        if json_traces is None:
+            raise ValueError('Response from the node is None. Is the node fully synced?')
+
         for json_trace in json_traces:
             trace = self.trace_mapper.json_dict_to_trace(json_trace)
             self.item_exporter.export_item(self.trace_mapper.trace_to_dict(trace))

ethereumetl/mainnet_daofork_state_changes.py

@@ -0,0 +1,149 @@
+# MIT License
+#
+# Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# https://github.com/ethereum/EIPs/blob/master/EIPS/eip-779.md
+# https://blog.ethereum.org/2016/07/20/hard-fork-completed/
+# https://gist.github.com/gavofyork/af747a034fbee2920f862ed352d32347
+# https://blog.ethereum.org/2016/07/15/to-fork-or-not-to-fork/
+
+DAOFORK_BLOCK_NUMBER = 1920000
+WITHDRAW_DAO_ADDRESS = '0xbf4ed7b27f1d666546e30d74d50d173d20bca754'
+
+MAINNET_DAOFORK_STATE_CHANGES = [
+    # from_address, to_address, value
+    ("0x005f5cee7a43331d5a3d3eec71305925a62f34b6", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x0101f3be8ebb4bbd39a2e3b9a3639d4259832fd9", WITHDRAW_DAO_ADDRESS, 559384955979606013894),
+    ("0x057b56736d32b86616a10f619859c6cd6f59092a", WITHDRAW_DAO_ADDRESS, 9900012824972102),
+    ("0x06706dd3f2c9abf0a21ddcc6941d9b86f0596936", WITHDRAW_DAO_ADDRESS, 1428573279216753537),
+    ("0x0737a6b837f97f46ebade41b9bc3e1c509c85c53", WITHDRAW_DAO_ADDRESS, 7144077587762826223),
+    ("0x07f5c1e1bc2c93e0402f23341973a0e043f7bf8a", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x0e0da70933f4c7849fc0d203f5d1d43b9ae4532d", WITHDRAW_DAO_ADDRESS, 19173240336954131945545),
+    ("0x0ff30d6de14a8224aa97b78aea5388d1c51c1f00", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x12e626b0eebfe86a56d633b9864e389b45dcb260", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x1591fc0f688c81fbeb17f5426a162a7024d430c2", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x17802f43a0137c506ba92291391a8a8f207f487d", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x1975bd06d486162d5dc297798dfc41edd5d160a7", WITHDRAW_DAO_ADDRESS, 989001281201758473335),
+    ("0x1ca6abd14d30affe533b24d7a21bff4c2d5e1f3b", WITHDRAW_DAO_ADDRESS, 76761842290232377901),
+    ("0x1cba23d343a983e9b5cfd19496b9a9701ada385f", WITHDRAW_DAO_ADDRESS, 68587370259945226),
+    ("0x200450f06520bdd6c527622a273333384d870efb", WITHDRAW_DAO_ADDRESS, 1250001619314659344457),
+    ("0x21c7fdb9ed8d291d79ffd82eb2c4356ec0d81241", WITHDRAW_DAO_ADDRESS, 27428797178668633),
+    ("0x23b75c2f6791eef49c69684db4c6c1f93bf49a50", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x24c4d950dfd4dd1902bbed3508144a54542bba94", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x253488078a4edf4d6f42f113d1e62836a942cf1a", WITHDRAW_DAO_ADDRESS, 3486036451558542464),
+    ("0x27b137a85656544b1ccb5a0f2e561a5703c6a68f", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x2a5ed960395e2a49b1c758cef4aa15213cfd874c", WITHDRAW_DAO_ADDRESS, 18693039890011849),
+    ("0x2b3455ec7fedf16e646268bf88846bd7a2319bb2", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x2c19c7f9ae8b751e37aeb2d93a699722395ae18f", WITHDRAW_DAO_ADDRESS, 8519214441755701),
+    ("0x304a554a310c7e546dfe434669c62820b7d83490", WITHDRAW_DAO_ADDRESS, 3642408527612792706899331),
+    ("0x319f70bab6845585f412ec7724b744fec6095c85", WITHDRAW_DAO_ADDRESS, 90658),
+    ("0x35a051a0010aba705c9008d7a7eff6fb88f6ea7b", WITHDRAW_DAO_ADDRESS, 15276059789372406985),
+    ("0x3ba4d81db016dc2890c81f3acec2454bff5aada5", WITHDRAW_DAO_ADDRESS, 1),
+    ("0x3c02a7bc0391e86d91b7d144e61c2c01a25a79c5", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x40b803a9abce16f50f36a77ba41180eb90023925", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x440c59b325d2997a134c2c7c60a8c61611212bad", WITHDRAW_DAO_ADDRESS, 266854104538362875475),
+    ("0x4486a3d68fac6967006d7a517b889fd3f98c102b", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x4613f3bca5c44ea06337a9e439fbc6d42e501d0a", WITHDRAW_DAO_ADDRESS, 28927603152430302650042),
+    ("0x47e7aa56d6bdf3f36be34619660de61275420af8", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x4863226780fe7c0356454236d3b1c8792785748d", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x492ea3bb0f3315521c31f273e565b868fc090f17", WITHDRAW_DAO_ADDRESS, 367380383063135344585),
+    ("0x4cb31628079fb14e4bc3cd5e30c2f7489b00960c", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x4deb0033bb26bc534b197e61d19e0733e5679784", WITHDRAW_DAO_ADDRESS, 1256101627216914882057),
+    ("0x4fa802324e929786dbda3b8820dc7834e9134a2a", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x4fd6ace747f06ece9c49699c7cabc62d02211f75", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x51e0ddd9998364a2eb38588679f0d2c42653e4a6", WITHDRAW_DAO_ADDRESS, 10000012954517274755),
+    ("0x52c5317c848ba20c7504cb2c8052abd1fde29d03", WITHDRAW_DAO_ADDRESS, 1996002585721648041229),
+    ("0x542a9515200d14b68e934e9830d91645a980dd7a", WITHDRAW_DAO_ADDRESS, 12548793143344641481996),
+    ("0x5524c55fb03cf21f549444ccbecb664d0acad706", WITHDRAW_DAO_ADDRESS, 6773243673260677597543),
+    ("0x579a80d909f346fbfb1189493f521d7f48d52238", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x58b95c9a9d5d26825e70a82b6adb139d3fd829eb", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x5c6e67ccd5849c0d29219c4f95f1a7a93b3f5dc5", WITHDRAW_DAO_ADDRESS, 1),
+    ("0x5c8536898fbb74fc7445814902fd08422eac56d0", WITHDRAW_DAO_ADDRESS, 205100000000392887672),
+    ("0x5d2b2e6fcbe3b11d26b525e085ff818dae332479", WITHDRAW_DAO_ADDRESS, 5000006477258637377),
+    ("0x5dc28b15dffed94048d73806ce4b7a4612a1d48f", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x5f9f3392e9f62f63b8eac0beb55541fc8627f42c", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x6131c42fa982e56929107413a9d526fd99405560", WITHDRAW_DAO_ADDRESS, 2121837249362469256186),
+    ("0x6231b6d0d5e77fe001c2a460bd9584fee60d409b", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x627a0a960c079c21c34f7612d5d230e01b4ad4c7", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x63ed5a272de2f6d968408b4acb9024f4cc208ebf", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x6966ab0d485353095148a2155858910e0965b6f9", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x6b0c4d41ba9ab8d8cfb5d379c69a612f2ced8ecb", WITHDRAW_DAO_ADDRESS, 854763543),
+    ("0x6d87578288b6cb5549d5076a207456a1f6a63dc0", WITHDRAW_DAO_ADDRESS, 1944767821345229848),
+    ("0x6f6704e5a10332af6672e50b3d9754dc460dfa4d", WITHDRAW_DAO_ADDRESS, 41173345768012804300),
+    ("0x7602b46df5390e432ef1c307d4f2c9ff6d65cc97", WITHDRAW_DAO_ADDRESS, 369231179004682274248),
+    ("0x779543a0491a837ca36ce8c635d6154e3c4911a6", WITHDRAW_DAO_ADDRESS, 100000000000000000),
+    ("0x77ca7b50b6cd7e2f3fa008e24ab793fd56cb15f6", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x782495b7b3355efb2833d56ecb34dc22ad7dfcc4", WITHDRAW_DAO_ADDRESS, 250000323862931868891),
+    ("0x807640a13483f8ac783c557fcdf27be11ea4ac7a", WITHDRAW_DAO_ADDRESS, 89472700),
+    ("0x8163e7fb499e90f8544ea62bbf80d21cd26d9efd", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x84ef4b2357079cd7a7c69fd7a37cd0609a679106", WITHDRAW_DAO_ADDRESS, 598974326560793095813484),
+    ("0x86af3e9626fce1957c82e88cbf04ddf3a2ed7915", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x8d9edb3054ce5c5774a420ac37ebae0ac02343c6", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x914d1b8b43e92723e64fd0a06f5bdb8dd9b10c79", WITHDRAW_DAO_ADDRESS, 285714295714285714286),
+    ("0x97f43a37f595ab5dd318fb46e7a155eae057317a", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x9aa008f65de0b923a2a4f02012ad034a5e2e2192", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x9c15b54878ba618f494b38f0ae7443db6af648ba", WITHDRAW_DAO_ADDRESS, 2236999142516500888),
+    ("0x9c50426be05db97f5d64fc54bf89eff947f0a321", WITHDRAW_DAO_ADDRESS, 0),
+    ("0x9da397b9e80755301a3b32173283a91c0ef6c87e", WITHDRAW_DAO_ADDRESS, 934889382511061152962),
+    ("0x9ea779f907f0b315b364b0cfc39a0fde5b02a416", WITHDRAW_DAO_ADDRESS, 15841461690131427090010),
+    ("0x9f27daea7aca0aa0446220b98d028715e3bc803d", WITHDRAW_DAO_ADDRESS, 99998647723253121277),
+    ("0x9fcd2deaff372a39cc679d5c5e4de7bafb0b1339", WITHDRAW_DAO_ADDRESS, 1409336722195117395464),
+    ("0xa2f1ccba9395d7fcb155bba8bc92db9bafaeade7", WITHDRAW_DAO_ADDRESS, 5000006477258637377),
+    ("0xa3acf3a1e16b1d7c315e23510fdd7847b48234f6", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xa5dc5acd6a7968a4554d89d65e59b7fd3bff0f90", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xa82f360a8d3455c5c41366975bde739c37bfeb8a", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xac1ecab32727358dba8962a0f3b261731aad9723", WITHDRAW_DAO_ADDRESS, 1),
+    ("0xaccc230e8a6e5be9160b8cdf2864dd2a001c28b6", WITHDRAW_DAO_ADDRESS, 23997787866533545896),
+    ("0xacd87e28b0c9d1254e868b81cba4cc20d9a32225", WITHDRAW_DAO_ADDRESS, 207153967008322399135),
+    ("0xadf80daec7ba8dcf15392f1ac611fff65d94f880", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xaeeb8ff27288bdabc0fa5ebb731b6f409507516c", WITHDRAW_DAO_ADDRESS, 859189750496835322093),
+    ("0xb136707642a4ea12fb4bae820f03d2562ebff487", WITHDRAW_DAO_ADDRESS, 7277385711515429122911683),
+    ("0xb2c6f0dfbb716ac562e2d85d6cb2f8d5ee87603e", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xb3fb0e5aba0e20e5c49d252dfd30e102b171a425", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xb52042c8ca3f8aa246fa79c3feaa3d959347c0ab", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xb9637156d330c0d605a791f1c31ba5890582fe1c", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xbb9bc244d798123fde783fcc1c72d3bb8c189413", WITHDRAW_DAO_ADDRESS, 1200000000000000001),
+    ("0xbc07118b9ac290e4622f5e77a0853539789effbe", WITHDRAW_DAO_ADDRESS, 5634097608979247392143),
+    ("0xbcf899e6c7d9d5a215ab1e3444c86806fa854c76", WITHDRAW_DAO_ADDRESS, 30696803822257124360133),
+    ("0xbe8539bfe837b67d1282b2b1d61c3f723966f049", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xc4bbd073882dd2add2424cf47d35213405b01324", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xca544e5c4687d109611d0f8f928b53a25af72448", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xcbb9d3703e651b0d496cdefb8b92c25aeb2171f7", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xcc34673c6c40e791051898567a1222daf90be287", WITHDRAW_DAO_ADDRESS, 60000077727103648),
+    ("0xceaeb481747ca6c540a000c1f3641f8cef161fa7", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xd131637d5275fd1a68a3200f4ad25c71a2a9522e", WITHDRAW_DAO_ADDRESS, 118886510785155274580),
+    ("0xd164b088bd9108b60d0ca3751da4bceb207b0782", WITHDRAW_DAO_ADDRESS, 1000001295451727475566),
+    ("0xd1ac8b1ef1b69ff51d1d401a476e7e612414f091", WITHDRAW_DAO_ADDRESS, 18387737083543350),
+    ("0xd343b217de44030afaa275f54d31a9317c7f441e", WITHDRAW_DAO_ADDRESS, 5192307692307692307692),
+    ("0xd4fe7bc31cedb7bfb8a345f31e668033056b2728", WITHDRAW_DAO_ADDRESS, 110000142499690430),
+    ("0xd9aef3a1e38a39c16b31d1ace71bca8ef58d315b", WITHDRAW_DAO_ADDRESS, 100000129545172747556),
+    ("0xda2fef9e4a3230988ff17df2165440f37e8b1708", WITHDRAW_DAO_ADDRESS, 73722042576599901129491),
+    ("0xdbe9b615a3ae8709af8b93336ce9b477e4ac0940", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xe308bd1ac5fda103967359b2712dd89deffb7973", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xe4ae1efdfc53b73893af49113d8694a057b9c0d1", WITHDRAW_DAO_ADDRESS, 5000006477258637377),
+    ("0xec8e57756626fdc07c63ad2eafbd28d08e7b0ca5", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xecd135fa4f61a655311e86238c92adcd779555d2", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xf0b1aa0eb660754448a7937c022e30aa692fe0c5", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xf1385fb24aad0cd7432824085e42aff90886fef5", WITHDRAW_DAO_ADDRESS, 0),
+    ("0xf14c14075d6c4ed84b86798af0956deef67365b5", WITHDRAW_DAO_ADDRESS, 2123311222366559138),
+    ("0xf4c64518ea10f995918a454158c6b61407ea345c", WITHDRAW_DAO_ADDRESS, 269565591797974102411594),
+    ("0xfe24cdd8648121a43a7c86d289be4dd2951ed49f", WITHDRAW_DAO_ADDRESS, 269833661813680507459)
+]

ethereumetl/mappers/trace_mapper.py

@@ -22,6 +22,7 @@
 
 
 from ethereumetl.domain.trace import EthTrace
+from ethereumetl.mainnet_daofork_state_changes import DAOFORK_BLOCK_NUMBER
 from ethereumetl.utils import hex_to_dec, to_normalized_address
 
 
@@ -93,6 +94,34 @@ class EthTraceMapper(object):
 
         return traces
 
+    def genesis_alloc_to_trace(self, allocation):
+        address = allocation[0]
+        value = allocation[1]
+
+        trace = EthTrace()
+
+        trace.block_number = 0
+        trace.to_address = address
+        trace.value = value
+        trace.trace_type = 'genesis'
+
+        return trace
+
+    def daofork_state_change_to_trace(self, state_change):
+        from_address = state_change[0]
+        to_address = state_change[1]
+        value = state_change[2]
+
+        trace = EthTrace()
+
+        trace.block_number = DAOFORK_BLOCK_NUMBER
+        trace.from_address = from_address
+        trace.to_address = to_address
+        trace.value = value
+        trace.trace_type = 'daofork'
+
+        return trace
+
     def _iterate_transaction_trace(self, block_number, tx_index, tx_trace, trace_address=[]):
         trace = EthTrace()
 

ethereumetl/service/eth_special_trace_service.py

@@ -0,0 +1,42 @@
+# MIT License
+#
+# Copyright (c) 2018 Evgeny Medvedev, evge.medvedev@gmail.com
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+from ethereumetl.mappers.trace_mapper import EthTraceMapper
+
+
+class EthSpecialTraceService(object):
+
+    def __init__(self):
+        self.trace_mapper = EthTraceMapper()
+
+    def get_genesis_traces(self):
+        from ethereumetl.mainnet_genesis_alloc import MAINNET_GENESIS_ALLOC
+        genesis_traces = [self.trace_mapper.genesis_alloc_to_trace(alloc)
+                          for alloc in MAINNET_GENESIS_ALLOC]
+        return genesis_traces
+
+    def get_daofork_traces(self):
+        from ethereumetl.mainnet_daofork_state_changes import MAINNET_DAOFORK_STATE_CHANGES
+        daofork_traces = [self.trace_mapper.daofork_state_change_to_trace(change)
+                          for change in MAINNET_DAOFORK_STATE_CHANGES]
+        return daofork_traces

export_all.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.export_all import export_all
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 export_all()

export_blocks_and_transactions.py

@@ -19,11 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+import sys
 
 from ethereumetl.cli.export_blocks_and_transactions import export_blocks_and_transactions
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 export_blocks_and_transactions()

export_contracts.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.export_contracts import export_contracts
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 export_contracts()

export_geth_traces.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.export_geth_traces import export_geth_traces
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 export_geth_traces()

export_receipts_and_logs.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.export_receipts_and_logs import export_receipts_and_logs
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 export_receipts_and_logs()

export_token_transfers.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.export_token_transfers import export_token_transfers
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 export_token_transfers()

export_tokens.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.export_tokens import export_tokens
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 export_tokens()

export_traces.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.export_traces import export_traces
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 export_traces()

extract_csv_column.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.extract_csv_column import extract_csv_column
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 extract_csv_column()

extract_field.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.extract_field import extract_field
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 extract_field()

extract_geth_traces.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.extract_geth_traces import extract_geth_traces
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 extract_geth_traces()

extract_token_transfers.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.extract_token_transfers import extract_token_transfers
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 extract_token_transfers()

filter_items.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.filter_items import filter_items
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 filter_items()

get_block_range_for_date.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.get_block_range_for_date import get_block_range_for_date
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 get_block_range_for_date()

get_block_range_for_timestamps.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.get_block_range_for_timestamps import get_block_range_for_timestamps
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 get_block_range_for_timestamps()

get_keccak_hash.py

@@ -19,12 +19,12 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-
+import sys
 
 from ethereumetl.cli.get_keccak_hash import get_keccak_hash
 
-print('========================================================================================')
-print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.')
-print('========================================================================================')
+print('========================================================================================', file=sys.stderr)
+print('THIS SCRIPT IS DEPRECATED AND WILL BE REMOVED ON 2019-01-01. Use ethereumetl.py instead.', file=sys.stderr)
+print('========================================================================================', file=sys.stderr)
 
 get_keccak_hash()

tests/ethereumetl/service/test_eth_service.py

@@ -22,11 +22,9 @@
 
 
 import pytest
-from click.testing import CliRunner
 from dateutil.parser import parse
 from web3 import HTTPProvider, Web3
 
-from ethereumetl.cli import get_block_range_for_date
 from ethereumetl.service.eth_service import EthService
 from ethereumetl.service.graph_operations import OutOfBoundsError
 from tests.helpers import skip_if_slow_tests_disabled
@@ -41,10 +39,10 @@ from tests.helpers import skip_if_slow_tests_disabled
     ('2018-06-10', 5761663, 5767303)
 ])
 def test_get_block_range_for_date(date, expected_start_block, expected_end_block):
-    runner = CliRunner()
-    result = runner.invoke(get_block_range_for_date, ['--date', date])
-    assert 0 == result.exit_code
-    assert '{},{}\n'.format(expected_start_block, expected_end_block) == result.output
+    eth_service = get_new_eth_service()
+    parsed_date = parse(date)
+    blocks = eth_service.get_block_range_for_date(parsed_date)
+    assert blocks == (expected_start_block, expected_end_block)
 
 
 @skip_if_slow_tests_disabled