Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Optimism / OP Stack Transaction Receipt Fields #446

Merged
merged 21 commits into from
Aug 2, 2023
Merged

Add Optimism / OP Stack Transaction Receipt Fields #446

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
6f44daf
add l1 fields
MSilb7 May 19, 2023
aae968c
Update README.md
MSilb7 May 19, 2023
84518f7
Update README.md
MSilb7 May 19, 2023
3affbad
comma
MSilb7 May 19, 2023
7c80c09
Merge branch 'develop' of /~https://github.com/MSilb7/optimism-etl into…
MSilb7 May 19, 2023
161aa6e
add error check for l1_gas_used_paid
MSilb7 May 19, 2023
af2ef17
Update README.md
MSilb7 May 19, 2023
e8b0447
Update README.md
MSilb7 May 19, 2023
f967d73
Revert "Update README.md"
MSilb7 May 25, 2023
993ebe6
Revert "Update README.md"
MSilb7 May 25, 2023
f8878ff
Revert "Update README.md"
MSilb7 May 25, 2023
a58fe45
Revert "Update README.md"
MSilb7 May 25, 2023
de96e39
rm l1 gas paid
MSilb7 May 25, 2023
f8a5f25
rm comma
MSilb7 May 25, 2023
6599a43
add to sql
MSilb7 May 26, 2023
eaf4bf0
add tests
MSilb7 Jun 1, 2023
ce906f0
Merge branch 'blockchain-etl:develop' into develop
MSilb7 Jun 1, 2023
7529c43
update
MSilb7 Jun 1, 2023
c0fd158
add to streaming
MSilb7 Jun 1, 2023
94ebd3f
mod scalar to be a float
MSilb7 Jun 2, 2023
7cbfd0e
Update receipts.sql
MSilb7 Jul 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ethereumetl/domain/receipt.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,8 @@ def __init__(self):
self.root = None
self.status = None
self.effective_gas_price = None
self.l1_fee = None
self.l1_gas_used = None
self.l1_gas_price = None
self.l1_fee_scalar = None

7 changes: 6 additions & 1 deletion ethereumetl/jobs/exporters/receipts_and_logs_item_exporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,12 @@
'contract_address',
'root',
'status',
'effective_gas_price'
'effective_gas_price',
'l1_fee',
'l1_gas_used',
'l1_gas_price',
'l1_fee_scalar'

]

LOG_FIELDS_TO_EXPORT = [
Expand Down
13 changes: 12 additions & 1 deletion ethereumetl/mappers/receipt_mapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@ def json_dict_to_receipt(self, json_dict):

receipt.effective_gas_price = hex_to_dec(json_dict.get('effectiveGasPrice'))

receipt.l1_fee = hex_to_dec(json_dict.get('l1Fee'))
receipt.l1_gas_used = hex_to_dec(json_dict.get('l1GasUsed'))
receipt.l1_gas_price = hex_to_dec(json_dict.get('l1GasPrice'))
receipt.l1_fee_scalar = hex_to_dec(json_dict.get('l1FeeScalar'))


if 'logs' in json_dict:
receipt.logs = [
self.receipt_log_mapper.json_dict_to_receipt_log(log) for log in json_dict['logs']
Expand All @@ -69,5 +75,10 @@ def receipt_to_dict(self, receipt):
'contract_address': receipt.contract_address,
'root': receipt.root,
'status': receipt.status,
'effective_gas_price': receipt.effective_gas_price
'effective_gas_price': receipt.effective_gas_price,
'l1_fee': receipt.l1_fee,
'l1_gas_used': receipt.l1_gas_used,
'l1_gas_price': receipt.l1_gas_price,
'l1_fee_scalar': receipt.l1_fee_scalar

}
7 changes: 6 additions & 1 deletion ethereumetl/streaming/enrich.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,12 @@ def enrich_transactions(transactions, receipts):
('contract_address', 'receipt_contract_address'),
('root', 'receipt_root'),
('status', 'receipt_status'),
('effective_gas_price', 'receipt_effective_gas_price')
('effective_gas_price', 'receipt_effective_gas_price'),
('l1_fee', 'receipt_l1_fee'),
('l1_gas_used', 'receipt_l1_gas_used'),
('l1_gas_price', 'receipt_l1_gas_price'),
('l1_fee_scalar', 'receipt_l1_fee_scalar')

]))

if len(result) != len(transactions):
Expand Down
6 changes: 5 additions & 1 deletion ethereumetl/streaming/postgres_tables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
# SOFTWARE.

from sqlalchemy import Table, Column, Integer, BigInteger, Boolean, String, Numeric, \
MetaData, PrimaryKeyConstraint, VARCHAR, TIMESTAMP
MetaData, PrimaryKeyConstraint, VARCHAR, TIMESTAMP, Float
from sqlalchemy.dialects.postgresql import ARRAY

metadata = MetaData()
Expand Down Expand Up @@ -74,6 +74,10 @@
Column('max_priority_fee_per_gas', BigInteger),
Column('transaction_type', BigInteger),
Column('receipt_effective_gas_price', BigInteger),
Column('receipt_l1_fee', BigInteger),
Column('receipt_l1_gas_used', BigInteger),
Column('receipt_l1_gas_price', BigInteger),
Column('receipt_l1_fee_scalar', Float),
)

LOGS = Table(
Expand Down
6 changes: 5 additions & 1 deletion schemas/aws/receipts.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@ CREATE EXTERNAL TABLE IF NOT EXISTS receipts (
contract_address STRING,
root STRING,
status BIGINT,
effective_gas_price BIGINT
effective_gas_price BIGINT,
l1_fee BIGINT,
l1_gas_used BIGINT,
l1_gas_price BIGINT,
l1_fee_scalar DECIMAL
)
PARTITIONED BY (block_date STRING)
ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
Expand Down
Binary file added tests/resources/test_export_receipts_job/.DS_Store
View file
Open in desktop
Binary file not shown.
10 changes: 5 additions & 5 deletions tests/resources/test_export_receipts_job/receipts_with_logs/expected_receipts.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
transaction_hash,transaction_index,block_hash,block_number,cumulative_gas_used,gas_used,contract_address,root,status,effective_gas_price
0x463d53f0ad57677a3b430a007c1c31d15d62c37fab5eee598551697c297c235c,2,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,122706,21000,,0x2f98549737594bf832213696d954cc1ee5ccbb1349f63e3983ea3d1b494180eb,,50000000000
0x05287a561f218418892ab053adfb3d919860988b19458c570c5c30f51c146f02,3,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,143706,21000,,0x4ab93bd0e8d40aaa3668404162449a76fa671a1cde7da668cccab99359924d2f,,50000000000
0x04cbcb236043d8fb7839e07bbc7f5eed692fb2ca55d897f1101eac3e3ad4fab8,0,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,50853,50853,,0x2ec017656e20275e92cbd1cdee9aeb43c1a090a5e217797da7c58dbf5be50e5b,,50000000000
0xcea6f89720cc1d2f46cc7a935463ae0b99dd5fad9c91bb7357de5421511cee49,1,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,101706,50853,,0xf7c67a3c8bc02b2c581b66f2bdf589a2a7ae9fccb2bf2ca3345b15cdcec6aefa,,50000000000
transaction_hash,transaction_index,block_hash,block_number,cumulative_gas_used,gas_used,contract_address,root,status,effective_gas_price,l1_fee,l1_gas_used,l1_gas_price,l1_fee_scalar
0x463d53f0ad57677a3b430a007c1c31d15d62c37fab5eee598551697c297c235c,2,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,122706,21000,,0x2f98549737594bf832213696d954cc1ee5ccbb1349f63e3983ea3d1b494180eb,,50000000000,,,,
0x05287a561f218418892ab053adfb3d919860988b19458c570c5c30f51c146f02,3,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,143706,21000,,0x4ab93bd0e8d40aaa3668404162449a76fa671a1cde7da668cccab99359924d2f,,50000000000,,,,
0x04cbcb236043d8fb7839e07bbc7f5eed692fb2ca55d897f1101eac3e3ad4fab8,0,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,50853,50853,,0x2ec017656e20275e92cbd1cdee9aeb43c1a090a5e217797da7c58dbf5be50e5b,,50000000000,,,,
0xcea6f89720cc1d2f46cc7a935463ae0b99dd5fad9c91bb7357de5421511cee49,1,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,101706,50853,,0xf7c67a3c8bc02b2c581b66f2bdf589a2a7ae9fccb2bf2ca3345b15cdcec6aefa,,50000000000,,,,
8 changes: 4 additions & 4 deletions tests/resources/test_export_receipts_job/receipts_with_logs/expected_receipts.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{"transaction_hash": "0x05287a561f218418892ab053adfb3d919860988b19458c570c5c30f51c146f02", "transaction_index": 3, "block_hash": "0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae", "block_number": 483920, "cumulative_gas_used": 143706, "gas_used": 21000, "contract_address": null, "root": "0x4ab93bd0e8d40aaa3668404162449a76fa671a1cde7da668cccab99359924d2f", "status": null, "effective_gas_price": 50000000000}
{"transaction_hash": "0xcea6f89720cc1d2f46cc7a935463ae0b99dd5fad9c91bb7357de5421511cee49", "transaction_index": 1, "block_hash": "0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae", "block_number": 483920, "cumulative_gas_used": 101706, "gas_used": 50853, "contract_address": null, "root": "0xf7c67a3c8bc02b2c581b66f2bdf589a2a7ae9fccb2bf2ca3345b15cdcec6aefa", "status": null, "effective_gas_price": 50000000000}
{"transaction_hash": "0x04cbcb236043d8fb7839e07bbc7f5eed692fb2ca55d897f1101eac3e3ad4fab8", "transaction_index": 0, "block_hash": "0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae", "block_number": 483920, "cumulative_gas_used": 50853, "gas_used": 50853, "contract_address": null, "root": "0x2ec017656e20275e92cbd1cdee9aeb43c1a090a5e217797da7c58dbf5be50e5b", "status": null, "effective_gas_price": 50000000000}
{"transaction_hash": "0x463d53f0ad57677a3b430a007c1c31d15d62c37fab5eee598551697c297c235c", "transaction_index": 2, "block_hash": "0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae", "block_number": 483920, "cumulative_gas_used": 122706, "gas_used": 21000, "contract_address": null, "root": "0x2f98549737594bf832213696d954cc1ee5ccbb1349f63e3983ea3d1b494180eb", "status": null, "effective_gas_price": 50000000000}
{"transaction_hash": "0x05287a561f218418892ab053adfb3d919860988b19458c570c5c30f51c146f02", "transaction_index": 3, "block_hash": "0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae", "block_number": 483920, "cumulative_gas_used": 143706, "gas_used": 21000, "contract_address": null, "root": "0x4ab93bd0e8d40aaa3668404162449a76fa671a1cde7da668cccab99359924d2f", "status": null, "effective_gas_price": 50000000000, "l1_fee": null, "l1_gas_used": null, "l1_gas_price": null, "l1_fee_scalar": null}
{"transaction_hash": "0xcea6f89720cc1d2f46cc7a935463ae0b99dd5fad9c91bb7357de5421511cee49", "transaction_index": 1, "block_hash": "0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae", "block_number": 483920, "cumulative_gas_used": 101706, "gas_used": 50853, "contract_address": null, "root": "0xf7c67a3c8bc02b2c581b66f2bdf589a2a7ae9fccb2bf2ca3345b15cdcec6aefa", "status": null, "effective_gas_price": 50000000000, "l1_fee": null, "l1_gas_used": null, "l1_gas_price": null, "l1_fee_scalar": null}
{"transaction_hash": "0x04cbcb236043d8fb7839e07bbc7f5eed692fb2ca55d897f1101eac3e3ad4fab8", "transaction_index": 0, "block_hash": "0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae", "block_number": 483920, "cumulative_gas_used": 50853, "gas_used": 50853, "contract_address": null, "root": "0x2ec017656e20275e92cbd1cdee9aeb43c1a090a5e217797da7c58dbf5be50e5b", "status": null, "effective_gas_price": 50000000000, "l1_fee": null, "l1_gas_used": null, "l1_gas_price": null, "l1_fee_scalar": null}
{"transaction_hash": "0x463d53f0ad57677a3b430a007c1c31d15d62c37fab5eee598551697c297c235c", "transaction_index": 2, "block_hash": "0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae", "block_number": 483920, "cumulative_gas_used": 122706, "gas_used": 21000, "contract_address": null, "root": "0x2f98549737594bf832213696d954cc1ee5ccbb1349f63e3983ea3d1b494180eb", "status": null, "effective_gas_price": 50000000000, "l1_fee": null, "l1_gas_used": null, "l1_gas_price": null, "l1_fee_scalar": null}
Loading