github/ethereum-etl
1
1
Fork 0
mirror of https://github.com/blockchain-etl/ethereum-etl.git synced 2026-01-10 14:27:54 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add date partitionized version of aws schemas

Browse Source
This commit is contained in:
tokusyu
2018-08-14 05:49:11 +09:00
parent 01a77ab127
commit 190384fe50
20 changed files with 464 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
schemas/aws_partition_by_date/blocks.sql Normal file
View File

@@ -0,0 +1,34 @@
CREATE EXTERNAL TABLE IF NOT EXISTS blocks (
number BIGINT,
hash STRING,
parent_hash STRING,
nonce STRING,
sha3_uncles STRING,
logs_bloom STRING,
transactions_root STRING,
state_root STRING,
receipts_root STRING,
miner STRING,
difficulty DECIMAL(38,0),
total_difficulty DECIMAL(38,0),
size BIGINT,
extra_data STRING,
gas_limit BIGINT,
gas_used BIGINT,
timestamp BIGINT,
transaction_count BIGINT
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/blocks'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE blocks;

21
schemas/aws_partition_by_date/contracts.sql Normal file
View File

@@ -0,0 +1,21 @@
CREATE EXTERNAL TABLE IF NOT EXISTS contracts (
address STRING,
bytecode STRING,
function_sighashes STRING,
is_erc20 BOOLEAN,
is_erc721 BOOLEAN
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/contracts'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE contracts;

24
schemas/aws_partition_by_date/logs.sql Normal file
View File

@@ -0,0 +1,24 @@
CREATE EXTERNAL TABLE IF NOT EXISTS logs (
log_index BIGINT,
transaction_hash STRING,
transaction_index BIGINT,
block_hash STRING,
block_number BIGINT,
address STRING,
data STRING,
topics STRING
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/logs'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE logs;

25
schemas/aws_partition_by_date/parquet/parquet_blocks.sql Normal file
View File

@@ -0,0 +1,25 @@
CREATE EXTERNAL TABLE IF NOT EXISTS parquet_blocks (
number BIGINT,
hash STRING,
parent_hash STRING,
nonce STRING,
sha3_uncles STRING,
logs_bloom STRING,
transactions_root STRING,
state_root STRING,
receipts_root STRING,
miner STRING,
difficulty DECIMAL(38,0),
total_difficulty DECIMAL(38,0),
size BIGINT,
extra_data STRING,
gas_limit BIGINT,
gas_used BIGINT,
timestamp BIGINT,
transaction_count BIGINT
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
STORED AS PARQUET
LOCATION 's3://<your_bucket>/ethereumetl/parquet/blocks';
MSCK REPAIR TABLE parquet_blocks;

14
schemas/aws_partition_by_date/parquet/parquet_token_transfers.sql Normal file
View File

@@ -0,0 +1,14 @@
CREATE EXTERNAL TABLE IF NOT EXISTS parquet_token_transfers (
token_address STRING,
from_address STRING,
to_address STRING,
value DECIMAL(38,0),
transaction_hash STRING,
log_index BIGINT,
block_number BIGINT
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
STORED AS PARQUET
LOCATION 's3://<your_bucket>/ethereumetl/parquet/token_transfers';
MSCK REPAIR TABLE parquet_token_transfers;

18
schemas/aws_partition_by_date/parquet/parquet_transactions.sql Normal file
View File

@@ -0,0 +1,18 @@
CREATE EXTERNAL TABLE IF NOT EXISTS parquet_transactions (
hash STRING,
nonce BIGINT,
block_hash STRING,
block_number BIGINT,
transaction_index BIGINT,
from_address STRING,
to_address STRING,
value DECIMAL(38,0),
gas BIGINT,
gas_price BIGINT,
input STRING
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
STORED AS PARQUET
LOCATION 's3://<your_bucket>/ethereumetl/parquet/transactions';
MSCK REPAIR TABLE parquet_transactions;

25
schemas/aws_partition_by_date/receipts.sql Normal file
View File

@@ -0,0 +1,25 @@
CREATE EXTERNAL TABLE IF NOT EXISTS receipts (
transaction_hash STRING,
transaction_index BIGINT,
block_hash STRING,
block_number BIGINT,
cumulative_gas_used BIGINT,
gas_used BIGINT,
contract_address STRING,
root STRING,
status BIGINT
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/receipts'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE receipts;

23
schemas/aws_partition_by_date/token_transfers.sql Normal file
View File

@@ -0,0 +1,23 @@
CREATE EXTERNAL TABLE IF NOT EXISTS token_transfers (
token_address STRING,
from_address STRING,
to_address STRING,
value DECIMAL(38,0),
transaction_hash STRING,
log_index BIGINT,
block_number BIGINT
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/token_transfers'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE token_transfers;

21
schemas/aws_partition_by_date/tokens.sql Normal file
View File

@@ -0,0 +1,21 @@
CREATE EXTERNAL TABLE IF NOT EXISTS tokens (
address STRING,
symbol STRING,
name STRING,
decimals BIGINT,
total_supply DECIMAL(38,0)
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/tokens'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE tokens;

27
schemas/aws_partition_by_date/transactions.sql Normal file
View File

@@ -0,0 +1,27 @@
CREATE EXTERNAL TABLE IF NOT EXISTS transactions (
hash STRING,
nonce BIGINT,
block_hash STRING,
block_number BIGINT,
transaction_index BIGINT,
from_address STRING,
to_address STRING,
value DECIMAL(38,0),
gas BIGINT,
gas_price BIGINT,
input STRING
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/transactions'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE transactions;

34
schemas/aws_partitoy_by_date/blocks.sql Normal file
View File

@@ -0,0 +1,34 @@
CREATE EXTERNAL TABLE IF NOT EXISTS blocks (
number BIGINT,
hash STRING,
parent_hash STRING,
nonce STRING,
sha3_uncles STRING,
logs_bloom STRING,
transactions_root STRING,
state_root STRING,
receipts_root STRING,
miner STRING,
difficulty DECIMAL(38,0),
total_difficulty DECIMAL(38,0),
size BIGINT,
extra_data STRING,
gas_limit BIGINT,
gas_used BIGINT,
timestamp BIGINT,
transaction_count BIGINT
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/blocks'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE blocks;

21
schemas/aws_partitoy_by_date/contracts.sql Normal file
View File

@@ -0,0 +1,21 @@
CREATE EXTERNAL TABLE IF NOT EXISTS contracts (
address STRING,
bytecode STRING,
function_sighashes STRING,
is_erc20 BOOLEAN,
is_erc721 BOOLEAN
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/contracts'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE contracts;

24
schemas/aws_partitoy_by_date/logs.sql Normal file
View File

@@ -0,0 +1,24 @@
CREATE EXTERNAL TABLE IF NOT EXISTS logs (
log_index BIGINT,
transaction_hash STRING,
transaction_index BIGINT,
block_hash STRING,
block_number BIGINT,
address STRING,
data STRING,
topics STRING
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/logs'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE logs;

25
schemas/aws_partitoy_by_date/parquet/parquet_blocks.sql Normal file
View File

@@ -0,0 +1,25 @@
CREATE EXTERNAL TABLE IF NOT EXISTS parquet_blocks (
number BIGINT,
hash STRING,
parent_hash STRING,
nonce STRING,
sha3_uncles STRING,
logs_bloom STRING,
transactions_root STRING,
state_root STRING,
receipts_root STRING,
miner STRING,
difficulty DECIMAL(38,0),
total_difficulty DECIMAL(38,0),
size BIGINT,
extra_data STRING,
gas_limit BIGINT,
gas_used BIGINT,
timestamp BIGINT,
transaction_count BIGINT
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
STORED AS PARQUET
LOCATION 's3://<your_bucket>/ethereumetl/parquet/blocks';
MSCK REPAIR TABLE parquet_blocks;

14
schemas/aws_partitoy_by_date/parquet/parquet_token_transfers.sql Normal file
View File

@@ -0,0 +1,14 @@
CREATE EXTERNAL TABLE IF NOT EXISTS parquet_token_transfers (
token_address STRING,
from_address STRING,
to_address STRING,
value DECIMAL(38,0),
transaction_hash STRING,
log_index BIGINT,
block_number BIGINT
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
STORED AS PARQUET
LOCATION 's3://<your_bucket>/ethereumetl/parquet/token_transfers';
MSCK REPAIR TABLE parquet_token_transfers;

18
schemas/aws_partitoy_by_date/parquet/parquet_transactions.sql Normal file
View File

@@ -0,0 +1,18 @@
CREATE EXTERNAL TABLE IF NOT EXISTS parquet_transactions (
hash STRING,
nonce BIGINT,
block_hash STRING,
block_number BIGINT,
transaction_index BIGINT,
from_address STRING,
to_address STRING,
value DECIMAL(38,0),
gas BIGINT,
gas_price BIGINT,
input STRING
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
STORED AS PARQUET
LOCATION 's3://<your_bucket>/ethereumetl/parquet/transactions';
MSCK REPAIR TABLE parquet_transactions;

25
schemas/aws_partitoy_by_date/receipts.sql Normal file
View File

@@ -0,0 +1,25 @@
CREATE EXTERNAL TABLE IF NOT EXISTS receipts (
transaction_hash STRING,
transaction_index BIGINT,
block_hash STRING,
block_number BIGINT,
cumulative_gas_used BIGINT,
gas_used BIGINT,
contract_address STRING,
root STRING,
status BIGINT
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/receipts'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE receipts;

23
schemas/aws_partitoy_by_date/token_transfers.sql Normal file
View File

@@ -0,0 +1,23 @@
CREATE EXTERNAL TABLE IF NOT EXISTS token_transfers (
token_address STRING,
from_address STRING,
to_address STRING,
value DECIMAL(38,0),
transaction_hash STRING,
log_index BIGINT,
block_number BIGINT
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/token_transfers'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE token_transfers;

21
schemas/aws_partitoy_by_date/tokens.sql Normal file
View File

@@ -0,0 +1,21 @@
CREATE EXTERNAL TABLE IF NOT EXISTS tokens (
address STRING,
symbol STRING,
name STRING,
decimals BIGINT,
total_supply DECIMAL(38,0)
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/tokens'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE tokens;

27
schemas/aws_partitoy_by_date/transactions.sql Normal file
View File

@@ -0,0 +1,27 @@
CREATE EXTERNAL TABLE IF NOT EXISTS transactions (
hash STRING,
nonce BIGINT,
block_hash STRING,
block_number BIGINT,
transaction_index BIGINT,
from_address STRING,
to_address STRING,
value DECIMAL(38,0),
gas BIGINT,
gas_price BIGINT,
input STRING
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/transactions'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE transactions;