Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Delete gas used from state #630

Merged
merged 3 commits into from
May 3, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions evm/chains/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ def get_block_by_hash(self, block_hash):

def get_block_by_header(self, block_header):
vm = self.get_vm(block_header)
return vm.get_block_by_header(block_header, self.chaindb)
return vm.get_block_class().from_header(block_header, self.chaindb)

@to_tuple
def get_ancestors(self, limit):
Expand Down Expand Up @@ -422,10 +422,6 @@ def get_block_by_hash(self, block_hash):
block_header = self.get_block_header_by_hash(block_hash)
return self.get_block_by_header(block_header)

def get_block_by_header(self, block_header):
vm = self.get_vm(block_header)
return vm.get_block_by_header(block_header, self.chaindb)

#
# Chain Initialization
#
Expand Down Expand Up @@ -488,7 +484,7 @@ def apply_transaction(self, transaction):
vm = self.get_vm()
base_block = vm.block

new_header, receipt, computation = vm.apply_transaction(transaction)
new_header, receipt, computation = vm.apply_transaction(base_block.header, transaction)

transactions = base_block.transactions + (transaction, )
receipts = base_block.get_receipts(self.chaindb) + (receipt, )
Expand Down
81 changes: 51 additions & 30 deletions evm/vm/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ def __init__(self, header, chaindb):
db=self.chaindb.db,
execution_context=self.block.header.create_execution_context(self.previous_hashes),
state_root=self.block.header.state_root,
gas_used=self.block.header.gas_used,
)

#
Expand Down Expand Up @@ -131,30 +130,64 @@ def execute_bytecode(self,
)

@abstractmethod
def make_receipt(self, transaction, computation, state):
def make_receipt(self, base_header, transaction, computation, state):
"""
Make receipt.
Generate the receipt resulting from applying the transaction.

:param base_header: the header of the block before the transaction was applied.
:param transaction: the transaction used to generate the receipt
:param computation: the result of running the transaction computation
:param state: the resulting state, after executing the computation
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great to see you're taking the chance to directly improve the documentation ❤️


:return: receipt
"""
raise NotImplementedError("Must be implemented by subclasses")

def apply_transaction(self, transaction):
@abstractmethod
def validate_transaction_against_header(self, base_header, transaction):
"""
Validate that the given transaction is valid to apply to the given header.

:param base_header: header before applying the transaction
:param transaction: the transaction to validate

:raises: ValidationError if the transaction is not valid to apply
"""
raise NotImplementedError("Must be implemented by subclasses")

def apply_transaction(self, header, transaction):
"""
Apply the transaction to the current block. This is a wrapper around
:func:`~evm.vm.state.State.apply_transaction` with some extra orchestration logic.

:param header: header of the block before application
:param transaction: to apply
"""
self.validate_transaction_against_header(header, transaction)
state_root, computation = self.state.apply_transaction(transaction)
receipt = self.make_receipt(transaction, computation, self.state)
# TODO: remove this mutation.
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was the inspiration for most of the PR.

self.state.gas_used = receipt.gas_used
receipt = self.make_receipt(header, transaction, computation, self.state)

new_header = self.block.header.copy(
bloom=int(BloomFilter(self.block.header.bloom) | receipt.bloom),
new_header = header.copy(
bloom=int(BloomFilter(header.bloom) | receipt.bloom),
gas_used=receipt.gas_used,
state_root=state_root,
)

return new_header, receipt, computation

def _apply_all_transactions(self, transactions, base_header):
Copy link
Contributor

@cburgdorf cburgdorf May 3, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How do you feel about sneaking in type definitions for new / changed functions? I do realize it (temporary) reduces conformity for that file in question when only few defs are typed while others are not but on the other hand it helps to work towards our goal to have full coverage and eventually enable stricter type checks to disallow untyped defs altogether.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚖️ I'm a little on the fence. These APIs will continue to change a lot. I guess I'm slightly 👎 at the moment, but when I feel like things start to settle, I'll be a lot more keen.

receipts = []
previous_header = base_header
result_header = base_header

for transaction in transactions:
result_header, receipt, _ = self.apply_transaction(previous_header, transaction)

previous_header = result_header
receipts.append(receipt)

return result_header, receipts
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was nice to pull this chunk of logic out, but it was awkward to both:

  • chain the result of one application into the next, and
  • accumulate the results

A small version involved a lot of variable clobbering, like:

    def _apply_all_transactions(self, transactions, header):
        receipts = []

        for transaction in transactions:
            header, receipt, _ = self.apply_transaction(header, transaction)
            receipts.append(receipt)

        return header, receipts

I'm happy to hear suggestions about a way to do chaining & accumulation at the same time, in a more elegant way.

Copy link
Member

@pipermerriam pipermerriam May 3, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hammered out this accumulation based zero mutation way to do this.

import functools
from cytoolz import accumulate

transactions = ('a', 'b', 'c', 'd')


def apply_transaction(header, txn):
    return header + 1, txn * 3

def test_it():
    txn_fns = [
        functools.partial(apply_transaction, txn=transaction)
        for transaction in transactions
    ]
    return accumulate(lambda prev, txn_fn: txn_fn(prev[0]), txn_fns, (0, None))


if __name__ == "__main__":
    print(tuple(test_it()))

# ((0, None), (1, 'aaa'), (2, 'bbb'), (3, 'ccc'), (4, 'ddd'))

Requires using the toolz.accumulate because it allows you to supply an initial value. Result. Probably want to do away with the lambda for readability.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks! It got me thinking of some other ways to look at it.

Right now, to my eyes, that particular implementation is a drop in readability, compared to having some variable reuse in a short function.

I have an idea for an improvement to get rid of it, but I don't want to throw this PR off track, so I'll put it in the next one.

Copy link
Member

@pipermerriam pipermerriam May 3, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think in implementation it will look like this.

# top level helper function.
def _accumulate_apply_txn_fn(prev_result, apply_transaction_fn):
    return apply_transaction_fn(prev_result[0])

# in VM class
def import_block(self, block):
    ...
    if block.transactions:
        apply_transaction_fns = tuple(
            functools.partial(self.apply_transaction, transaction=transaction)
            for transaction in block.transactions
        )
        headers, receipts = zip(*tuple(accumulate(
            _accumulate_apply_txn_fn, 
            apply_transaction_fns,
            (self.block.header, None),
        ))[1:])
        last_header = headers[-1]
    else:
        last_header = self.block.header
        receipts = tuple()
     ...

I think I generally agree about the readability, but I'm also inclined to cut the extra API from the VM class. We could still use the logic above to encapulate the act of applying all of the transactions and getting back the resulting header and list of receipts but still keep it encapsulated in a utility function to keep the import_block method easier to read.

Dunno, your call. Just wanted to see it all written out mostly.


#
# Mining
#
Expand All @@ -179,25 +212,14 @@ def import_block(self, block):
db=self.chaindb.db,
execution_context=self.block.header.create_execution_context(self.previous_hashes),
state_root=self.block.header.state_root,
gas_used=self.block.header.gas_used,
)

# run all of the transactions.
execution_data = [
self.apply_transaction(transaction)
for transaction
in block.transactions
]
if execution_data:
headers, receipts, _ = zip(*execution_data)
header_with_txns = headers[-1]
else:
receipts = tuple()
header_with_txns = self.block.header
last_header, receipts = self._apply_all_transactions(block.transactions, self.block.header)

self.block = self.set_block_transactions(
self.block,
header_with_txns,
last_header,
block.transactions,
receipts,
)
Expand Down Expand Up @@ -283,7 +305,6 @@ def state_in_temp_block(self):
db=self.chaindb.db,
execution_context=temp_block.header.create_execution_context(prev_hashes),
state_root=temp_block.header.state_root,
gas_used=0,
)

snapshot = state.snapshot()
Expand Down Expand Up @@ -315,6 +336,13 @@ def validate_block(self, block):
"""
Validate the the given block.
"""
if not isinstance(block, self.get_block_class()):
raise ValidationError(
"This vm ({0!r}) is not equipped to validate a block of type {1!r}".format(
self,
block,
)
)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess this check will be moot, when this method moves up to Chain. But we are a couple days from moving this one, I think.

if not block.is_genesis:
parent_header = get_parent_header(block.header, self.chaindb)

Expand Down Expand Up @@ -435,13 +463,6 @@ def get_block_class(cls):
"""
return cls.get_state_class().get_block_class()

@classmethod
def get_block_by_header(cls, block_header, db):
"""
Lookup and return the block for the given header.
"""
return cls.get_block_class().from_header(block_header, db)

@classmethod
@functools.lru_cache(maxsize=32)
@to_tuple
Expand Down
18 changes: 5 additions & 13 deletions evm/vm/forks/byzantium/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
from evm.rlp.receipts import (
Receipt,
)
from evm.vm.forks.spurious_dragon import SpuriousDragonVM
from evm.vm.forks.frontier import make_frontier_receipt

Expand All @@ -16,20 +13,15 @@
from .state import ByzantiumState


def make_byzantium_receipt(transaction, computation, state):
old_receipt = make_frontier_receipt(transaction, computation, state)
def make_byzantium_receipt(base_header, transaction, computation, state):
frontier_receipt = make_frontier_receipt(base_header, transaction, computation, state)

if computation.is_error:
state_root = EIP658_TRANSACTION_STATUS_CODE_FAILURE
status_code = EIP658_TRANSACTION_STATUS_CODE_FAILURE
else:
state_root = EIP658_TRANSACTION_STATUS_CODE_SUCCESS
status_code = EIP658_TRANSACTION_STATUS_CODE_SUCCESS

receipt = Receipt(
state_root=state_root,
gas_used=old_receipt.gas_used,
logs=old_receipt.logs,
)
return receipt
return frontier_receipt.copy(state_root=status_code)


ByzantiumVM = SpuriousDragonVM.configure(
Expand Down
8 changes: 5 additions & 3 deletions evm/vm/forks/frontier/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,10 @@
compute_frontier_difficulty,
configure_frontier_header,
)
from .validation import validate_frontier_transaction_against_header


def make_frontier_receipt(transaction, computation, state):
def make_frontier_receipt(base_header, transaction, computation, state):
# Reusable for other forks

logs = [
Expand All @@ -33,7 +34,7 @@ def make_frontier_receipt(transaction, computation, state):
gas_refund,
(transaction.gas - gas_remaining) // 2,
)
gas_used = state.gas_used + tx_gas_used
gas_used = base_header.gas_used + tx_gas_used

receipt = Receipt(
state_root=state.state_root,
Expand All @@ -55,5 +56,6 @@ def make_frontier_receipt(transaction, computation, state):
create_header_from_parent=staticmethod(create_frontier_header_from_parent),
compute_difficulty=staticmethod(compute_frontier_difficulty),
configure_header=configure_frontier_header,
make_receipt=staticmethod(make_frontier_receipt)
make_receipt=staticmethod(make_frontier_receipt),
validate_transaction_against_header=validate_frontier_transaction_against_header,
)
2 changes: 1 addition & 1 deletion evm/vm/forks/frontier/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ class FrontierState(BaseState, FrontierTransactionExecutor):
account_db_class = AccountDB # Type[BaseAccountDB]

def validate_transaction(self, transaction):
validate_frontier_transaction(self, transaction)
validate_frontier_transaction(self.account_db, transaction)

@staticmethod
def get_block_reward():
Expand Down
20 changes: 14 additions & 6 deletions evm/vm/forks/frontier/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
)


def validate_frontier_transaction(state, transaction):
def validate_frontier_transaction(account_db, transaction):
gas_cost = transaction.gas * transaction.gas_price
sender_balance = state.account_db.get_balance(transaction.sender)
sender_balance = account_db.get_balance(transaction.sender)

if sender_balance < gas_cost:
raise ValidationError(
Expand All @@ -17,8 +17,16 @@ def validate_frontier_transaction(state, transaction):
if sender_balance < total_cost:
raise ValidationError("Sender account balance cannot afford txn")

if state.gas_used + transaction.gas > state.gas_limit:
raise ValidationError("Transaction exceeds gas limit")

if state.account_db.get_nonce(transaction.sender) != transaction.nonce:
if account_db.get_nonce(transaction.sender) != transaction.nonce:
raise ValidationError("Invalid transaction nonce")


def validate_frontier_transaction_against_header(_vm, base_header, transaction):
if base_header.gas_used + transaction.gas > base_header.gas_limit:
raise ValidationError(
"Transaction exceeds gas limit: using {}, bringing total to {}, but limit is {}".format(
transaction.gas,
base_header.gas_used + transaction.gas,
base_header.gas_limit,
)
)
2 changes: 1 addition & 1 deletion evm/vm/forks/homestead/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ class HomesteadState(FrontierState):
computation_class = HomesteadComputation

def validate_transaction(self, transaction):
validate_homestead_transaction(self, transaction)
validate_homestead_transaction(self.account_db, transaction)
4 changes: 2 additions & 2 deletions evm/vm/forks/homestead/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
)


def validate_homestead_transaction(evm, transaction):
def validate_homestead_transaction(account_db, transaction):
if transaction.s > SECPK1_N // 2 or transaction.s == 0:
raise ValidationError("Invalid signature S value")

validate_frontier_transaction(evm, transaction)
validate_frontier_transaction(account_db, transaction)
4 changes: 1 addition & 3 deletions evm/vm/state.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,18 +63,16 @@ class for vm execution.
_chaindb = None
execution_context = None
state_root = None
gas_used = None

block_class = None # type: Type[BaseBlock]
computation_class = None # type: Type[BaseComputation]
transaction_context_class = None # type: Type[BaseTransactionContext]
account_db_class = None # type: Type[BaseAccountDB]

def __init__(self, db, execution_context, state_root, gas_used):
def __init__(self, db, execution_context, state_root):
self._db = db
self.execution_context = execution_context
self.account_db = self.get_account_db_class()(self._db, state_root)
self.gas_used = gas_used

#
# Logging
Expand Down
6 changes: 3 additions & 3 deletions tests/core/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,16 @@ def fill_block(chain, from_, key, gas, data):
amount = 100

vm = chain.get_vm()
assert vm.state.gas_used == 0
assert vm.block.header.gas_used == 0

while True:
tx = new_transaction(chain.get_vm(), from_, recipient, amount, key, gas=gas, data=data)
try:
chain.apply_transaction(tx)
except ValidationError as exc:
if "Transaction exceeds gas limit" == str(exc):
if str(exc).startswith("Transaction exceeds gas limit"):
break
else:
raise exc

assert chain.get_vm().state.gas_used > 0
assert chain.get_vm().block.header.gas_used > 0
2 changes: 1 addition & 1 deletion tests/core/vm/test_vm.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def test_apply_transaction(
amount = 100
from_ = funded_address
tx = new_transaction(vm, from_, recipient, amount, funded_address_private_key)
new_header, _, computation = vm.apply_transaction(tx)
new_header, _, computation = vm.apply_transaction(vm.block.header, tx)

assert not computation.is_error
tx_gas = tx.gas_price * constants.GAS_TX
Expand Down
2 changes: 1 addition & 1 deletion tests/json-fixtures/test_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ def test_state_fixtures(fixture, fixture_vm_class):
)

try:
header, receipt, computation = vm.apply_transaction(transaction)
header, receipt, computation = vm.apply_transaction(vm.block.header, transaction)
transactions = vm.block.transactions + (transaction, )
receipts = vm.block.get_receipts(chaindb) + (receipt, )
block = vm.set_block_transactions(vm.block, header, transactions, receipts)
Expand Down