github/ethereum-etl
1
1
Fork 0
mirror of https://github.com/blockchain-etl/ethereum-etl.git synced 2026-01-09 05:48:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

schemas: aws_partition_by_date -> aws

Browse Source 
Sponsored by CoinStats.app
This commit is contained in:
Anton Bryzgalov @ CoinStats
2022-06-10 13:56:43 +04:00
parent ac812a0f36
commit 7a337e724a
17 changed files with 52 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
schemas/aws/blocks.sql
View File

@@ -16,19 +16,16 @@ CREATE EXTERNAL TABLE IF NOT EXISTS blocks (
gas_limit BIGINT,
gas_used BIGINT,
timestamp BIGINT,
transaction_count BIGINT
transaction_count BIGINT,
base_fee_per_gas BIGINT
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/blocks'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
PARTITIONED BY (block_date STRING)
ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
LOCATION 's3://<your_bucket>/export/blocks/';
MSCK REPAIR TABLE blocks;
MSCK REPAIR TABLE blocks;
select *
from blocks
where block_date = '2015-10-01'
;

2
schemas/aws/contracts.sql
View File

@@ -5,7 +5,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS contracts (
is_erc20 BOOLEAN,
is_erc721 BOOLEAN
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',

23
schemas/aws/logs.sql
View File

@@ -6,19 +6,16 @@ CREATE EXTERNAL TABLE IF NOT EXISTS logs (
block_number BIGINT,
address STRING,
data STRING,
topics STRING
topics ARRAY<STRING>
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/logs'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
PARTITIONED BY (block_date STRING)
ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
LOCATION 's3://<your_bucket>/export/logs/';
MSCK REPAIR TABLE logs;
show partitions logs;
select *
from logs
where block_date = '2015-11-08'
;

23
schemas/aws/receipts.sql
View File

@@ -7,19 +7,16 @@ CREATE EXTERNAL TABLE IF NOT EXISTS receipts (
gas_used BIGINT,
contract_address STRING,
root STRING,
status BIGINT
status BIGINT,
effective_gas_price BIGINT
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/receipts'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
PARTITIONED BY (block_date STRING)
ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
LOCATION 's3://<your_bucket>/export/receipts/';
MSCK REPAIR TABLE receipts;
select *
from receipts
where block_date = '2015-11-08'
;

23
schemas/aws/token_transfers.sql
View File

@@ -2,22 +2,17 @@ CREATE EXTERNAL TABLE IF NOT EXISTS token_transfers (
token_address STRING,
from_address STRING,
to_address STRING,
value DECIMAL(38,0),
value STRING,
transaction_hash STRING,
log_index BIGINT,
block_number BIGINT
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/token_transfers'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
PARTITIONED BY (block_date STRING)
ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
LOCATION 's3://<your_bucket>/export/token_transfers/';
MSCK REPAIR TABLE token_transfers;
MSCK REPAIR TABLE token_transfers;
select *
from token_transfers
where block_date = '2015-11-13'
;

2
schemas/aws/tokens.sql
View File

@@ -5,7 +5,7 @@ CREATE EXTERNAL TABLE IF NOT EXISTS tokens (
decimals BIGINT,
total_supply DECIMAL(38,0)
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',

24
schemas/aws/transactions.sql
View File

@@ -9,19 +9,15 @@ CREATE EXTERNAL TABLE IF NOT EXISTS transactions (
value DECIMAL(38,0),
gas BIGINT,
gas_price BIGINT,
input STRING
input STRING,
max_fee_per_gas BIGINT,
max_priority_fee_per_gas BIGINT,
transaction_type BIGINT
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/transactions'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
PARTITIONED BY (block_date STRING)
ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
LOCATION 's3://<your_bucket>/export/transactions/';
MSCK REPAIR TABLE transactions;
MSCK REPAIR TABLE transactions;
SHOW PARTITIONS anton_test.transactions;
SELECT * FROM anton_test.transactions;

31
schemas/aws_partition_by_date/blocks.sql
View File

@@ -1,31 +0,0 @@
CREATE EXTERNAL TABLE IF NOT EXISTS blocks (
number BIGINT,
hash STRING,
parent_hash STRING,
nonce STRING,
sha3_uncles STRING,
logs_bloom STRING,
transactions_root STRING,
state_root STRING,
receipts_root STRING,
miner STRING,
difficulty DECIMAL(38,0),
total_difficulty DECIMAL(38,0),
size BIGINT,
extra_data STRING,
gas_limit BIGINT,
gas_used BIGINT,
timestamp BIGINT,
transaction_count BIGINT,
base_fee_per_gas BIGINT
)
PARTITIONED BY (block_date STRING)
ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
LOCATION 's3://<your_bucket>/export/blocks/';
MSCK REPAIR TABLE blocks;
select *
from blocks
where block_date = '2015-10-01'
;

21
schemas/aws_partition_by_date/contracts.sql
View File

@@ -1,21 +0,0 @@
CREATE EXTERNAL TABLE IF NOT EXISTS contracts (
address STRING,
bytecode STRING,
function_sighashes STRING,
is_erc20 BOOLEAN,
is_erc721 BOOLEAN
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/contracts'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE contracts;

21
schemas/aws_partition_by_date/logs.sql
View File

@@ -1,21 +0,0 @@
CREATE EXTERNAL TABLE IF NOT EXISTS logs (
log_index BIGINT,
transaction_hash STRING,
transaction_index BIGINT,
block_hash STRING,
block_number BIGINT,
address STRING,
data STRING,
topics ARRAY<STRING>
)
PARTITIONED BY (block_date STRING)
ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
LOCATION 's3://<your_bucket>/export/logs/';
MSCK REPAIR TABLE logs;
show partitions logs;
select *
from logs
where block_date = '2015-11-08'
;

25
schemas/aws_partition_by_date/parquet/parquet_blocks.sql
View File

@@ -1,25 +0,0 @@
CREATE EXTERNAL TABLE IF NOT EXISTS parquet_blocks (
number BIGINT,
hash STRING,
parent_hash STRING,
nonce STRING,
sha3_uncles STRING,
logs_bloom STRING,
transactions_root STRING,
state_root STRING,
receipts_root STRING,
miner STRING,
difficulty DECIMAL(38,0),
total_difficulty DECIMAL(38,0),
size BIGINT,
extra_data STRING,
gas_limit BIGINT,
gas_used BIGINT,
timestamp BIGINT,
transaction_count BIGINT
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
STORED AS PARQUET
LOCATION 's3://<your_bucket>/ethereumetl/parquet/blocks';
MSCK REPAIR TABLE parquet_blocks;

14
schemas/aws_partition_by_date/parquet/parquet_token_transfers.sql
View File

@@ -1,14 +0,0 @@
CREATE EXTERNAL TABLE IF NOT EXISTS parquet_token_transfers (
token_address STRING,
from_address STRING,
to_address STRING,
value DECIMAL(38,0),
transaction_hash STRING,
log_index BIGINT,
block_number BIGINT
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
STORED AS PARQUET
LOCATION 's3://<your_bucket>/ethereumetl/parquet/token_transfers';
MSCK REPAIR TABLE parquet_token_transfers;

18
schemas/aws_partition_by_date/parquet/parquet_transactions.sql
View File

@@ -1,18 +0,0 @@
CREATE EXTERNAL TABLE IF NOT EXISTS parquet_transactions (
hash STRING,
nonce BIGINT,
block_hash STRING,
block_number BIGINT,
transaction_index BIGINT,
from_address STRING,
to_address STRING,
value DECIMAL(38,0),
gas BIGINT,
gas_price BIGINT,
input STRING
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
STORED AS PARQUET
LOCATION 's3://<your_bucket>/ethereumetl/parquet/transactions';
MSCK REPAIR TABLE parquet_transactions;

22
schemas/aws_partition_by_date/receipts.sql
View File

@@ -1,22 +0,0 @@
CREATE EXTERNAL TABLE IF NOT EXISTS receipts (
transaction_hash STRING,
transaction_index BIGINT,
block_hash STRING,
block_number BIGINT,
cumulative_gas_used BIGINT,
gas_used BIGINT,
contract_address STRING,
root STRING,
status BIGINT,
effective_gas_price BIGINT
)
PARTITIONED BY (block_date STRING)
ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
LOCATION 's3://<your_bucket>/export/receipts/';
MSCK REPAIR TABLE receipts;
select *
from receipts
where block_date = '2015-11-08'
;

18
schemas/aws_partition_by_date/token_transfers.sql
View File

@@ -1,18 +0,0 @@
CREATE EXTERNAL TABLE IF NOT EXISTS token_transfers (
token_address STRING,
from_address STRING,
to_address STRING,
value STRING,
transaction_hash STRING,
log_index BIGINT,
block_number BIGINT
)
PARTITIONED BY (block_date STRING)
ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
LOCATION 's3://<your_bucket>/export/token_transfers/';
MSCK REPAIR TABLE token_transfers;
select *
from token_transfers
where block_date = '2015-11-13'
;

21
schemas/aws_partition_by_date/tokens.sql
View File

@@ -1,21 +0,0 @@
CREATE EXTERNAL TABLE IF NOT EXISTS tokens (
address STRING,
symbol STRING,
name STRING,
decimals BIGINT,
total_supply DECIMAL(38,0)
)
PARTITIONED BY (date STRING)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
WITH SERDEPROPERTIES (
'serialization.format' = ',',
'field.delim' = ',',
'escape.delim' = '\\'
)
STORED AS TEXTFILE
LOCATION 's3://<your_bucket>/ethereumetl/export/tokens'
TBLPROPERTIES (
'skip.header.line.count' = '1'
);
MSCK REPAIR TABLE tokens;

23
schemas/aws_partition_by_date/transactions.sql
View File

@@ -1,23 +0,0 @@
CREATE EXTERNAL TABLE IF NOT EXISTS transactions (
hash STRING,
nonce BIGINT,
block_hash STRING,
block_number BIGINT,
transaction_index BIGINT,
from_address STRING,
to_address STRING,
value DECIMAL(38,0),
gas BIGINT,
gas_price BIGINT,
input STRING,
max_fee_per_gas BIGINT,
max_priority_fee_per_gas BIGINT,
transaction_type BIGINT
)
PARTITIONED BY (block_date STRING)
ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
LOCATION 's3://<your_bucket>/export/transactions/';
MSCK REPAIR TABLE transactions;
SHOW PARTITIONS anton_test.transactions;
SELECT * FROM anton_test.transactions;