diff --git a/tests/core/pyspec/eth2spec/test/altair/random/__init__.py b/tests/core/pyspec/eth2spec/test/altair/random/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/altair/random/test_random.py b/tests/core/pyspec/eth2spec/test/altair/random/test_random.py new file mode 100644 index 000000000..2250101bd --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/altair/random/test_random.py @@ -0,0 +1,438 @@ +""" +This module is generated from the ``random`` test generator. +Please do not edit this file manually. +See the README for that generator for more information. +""" + +from eth2spec.test.helpers.constants import ALTAIR +from eth2spec.test.context import ( + misc_balances_in_default_range_with_many_validators, + with_phases, + zero_activation_threshold, + only_generator, +) +from eth2spec.test.context import ( + always_bls, + spec_test, + with_custom_state, + single_phase, +) +from eth2spec.test.utils.randomized_block_tests import ( + run_generated_randomized_test, +) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_0(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_1(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_2(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_3(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_4(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_5(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_6(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_7(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_8(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_9(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_10(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_11(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_12(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_13(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_14(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([ALTAIR]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_15(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block_altair_with_cycling_sync_committee_participation + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block_altair_with_cycling_sync_committee_participation', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state_altair'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 7fddc0762..346cdc8f1 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -152,6 +152,22 @@ def misc_balances(spec): return balances +def misc_balances_in_default_range_with_many_validators(spec): + """ + Helper method to create a series of balances that includes some misc. balances but + none that are below the ``EJECTION_BALANCE``. + """ + # Double validators to facilitate randomized testing + num_validators = spec.SLOTS_PER_EPOCH * 8 * 2 + floor = spec.config.EJECTION_BALANCE + spec.EFFECTIVE_BALANCE_INCREMENT + balances = [ + max(spec.MAX_EFFECTIVE_BALANCE * 2 * i // num_validators, floor) for i in range(num_validators) + ] + rng = Random(1234) + rng.shuffle(balances) + return balances + + def low_single_balance(spec): """ Helper method to create a single of balance of 1 Gwei. @@ -440,6 +456,17 @@ with_altair_and_later = with_phases([ALTAIR, MERGE]) with_merge_and_later = with_phases([MERGE]) # TODO: include sharding when spec stabilizes. +def only_generator(reason): + def _decorator(inner): + def _wrapper(*args, **kwargs): + if is_pytest: + dump_skipping_message(reason) + return None + return inner(*args, **kwargs) + return _wrapper + return _decorator + + def fork_transition_test(pre_fork_name, post_fork_name, fork_epoch=None): """ A decorator to construct a "transition" test from one fork of the eth2 spec diff --git a/tests/core/pyspec/eth2spec/test/helpers/multi_operations.py b/tests/core/pyspec/eth2spec/test/helpers/multi_operations.py index 4b6c9b25d..14b281a95 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/multi_operations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/multi_operations.py @@ -7,6 +7,10 @@ from eth2spec.test.helpers.state import ( from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, ) +from eth2spec.test.helpers.sync_committee import ( + compute_committee_indices, + compute_aggregate_sync_committee_signature, +) from eth2spec.test.helpers.proposer_slashings import get_valid_proposer_slashing from eth2spec.test.helpers.attester_slashings import get_valid_attester_slashing_by_indices from eth2spec.test.helpers.attestations import get_valid_attestation @@ -44,8 +48,12 @@ def run_slash_and_exit(spec, state, slash_index, exit_index, valid=True): def get_random_proposer_slashings(spec, state, rng): - num_slashings = rng.randrange(spec.MAX_PROPOSER_SLASHINGS) - indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).copy() + num_slashings = rng.randrange(1, spec.MAX_PROPOSER_SLASHINGS) + active_indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).copy() + indices = [ + index for index in active_indices + if not state.validators[index].slashed + ] slashings = [ get_valid_proposer_slashing( spec, state, @@ -56,14 +64,33 @@ def get_random_proposer_slashings(spec, state, rng): return slashings -def get_random_attester_slashings(spec, state, rng): - num_slashings = rng.randrange(spec.MAX_ATTESTER_SLASHINGS) - indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).copy() +def get_random_attester_slashings(spec, state, rng, slashed_indices=[]): + """ + Caller can supply ``slashed_indices`` if they are aware of other indices + that will be slashed by other operations in the same block as the one that + contains the output of this function. + """ + # ensure at least one attester slashing, the max count + # is small so not much room for random inclusion + num_slashings = rng.randrange(1, spec.MAX_ATTESTER_SLASHINGS) + active_indices = spec.get_active_validator_indices(state, spec.get_current_epoch(state)).copy() + indices = [ + index for index in active_indices + if ( + not state.validators[index].slashed + and index not in slashed_indices + ) + ] + sample_upper_bound = 4 + max_slashed_count = num_slashings * sample_upper_bound - 1 + if len(indices) < max_slashed_count: + return [] + slot_range = list(range(state.slot - spec.SLOTS_PER_HISTORICAL_ROOT + 1, state.slot)) slashings = [ get_valid_attester_slashing_by_indices( spec, state, - sorted([indices.pop(rng.randrange(len(indices))) for _ in range(rng.randrange(1, 4))]), + sorted([indices.pop(rng.randrange(len(indices))) for _ in range(rng.randrange(1, sample_upper_bound))]), slot=slot_range.pop(rng.randrange(len(slot_range))), signed_1=True, signed_2=True, ) @@ -73,7 +100,7 @@ def get_random_attester_slashings(spec, state, rng): def get_random_attestations(spec, state, rng): - num_attestations = rng.randrange(spec.MAX_ATTESTATIONS) + num_attestations = rng.randrange(1, spec.MAX_ATTESTATIONS) attestations = [ get_valid_attestation( @@ -87,7 +114,7 @@ def get_random_attestations(spec, state, rng): def prepare_state_and_get_random_deposits(spec, state, rng): - num_deposits = rng.randrange(spec.MAX_DEPOSITS) + num_deposits = rng.randrange(1, spec.MAX_DEPOSITS) deposit_data_leaves = [spec.DepositData() for _ in range(len(state.validators))] deposits = [] @@ -117,24 +144,65 @@ def prepare_state_and_get_random_deposits(spec, state, rng): return deposits +def _eligible_for_exit(spec, state, index): + validator = state.validators[index] + + not_slashed = not validator.slashed + + current_epoch = spec.get_current_epoch(state) + activation_epoch = validator.activation_epoch + active_for_long_enough = current_epoch >= activation_epoch + spec.config.SHARD_COMMITTEE_PERIOD + + not_exited = validator.exit_epoch == spec.FAR_FUTURE_EPOCH + + return not_slashed and active_for_long_enough and not_exited + + def get_random_voluntary_exits(spec, state, to_be_slashed_indices, rng): - num_exits = rng.randrange(spec.MAX_VOLUNTARY_EXITS) - indices = set(spec.get_active_validator_indices(state, spec.get_current_epoch(state)).copy()) + num_exits = rng.randrange(1, spec.MAX_VOLUNTARY_EXITS) + active_indices = set(spec.get_active_validator_indices(state, spec.get_current_epoch(state)).copy()) + indices = set( + index for index in active_indices + if _eligible_for_exit(spec, state, index) + ) eligible_indices = indices - to_be_slashed_indices - exit_indices = [eligible_indices.pop() for _ in range(num_exits)] + indices_count = min(num_exits, len(eligible_indices)) + exit_indices = [eligible_indices.pop() for _ in range(indices_count)] return prepare_signed_exits(spec, state, exit_indices) -def run_test_full_random_operations(spec, state, rng=Random(2080)): - # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit - state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH +def get_random_sync_aggregate(spec, state, slot, fraction_participated=1.0, rng=Random(2099)): + committee_indices = compute_committee_indices(spec, state, state.current_sync_committee) + participant_count = int(len(committee_indices) * fraction_participated) + participant_indices = rng.sample(range(len(committee_indices)), participant_count) + participants = [ + committee_indices[index] + for index in participant_indices + ] + signature = compute_aggregate_sync_committee_signature( + spec, + state, + slot, + participants, + ) + return spec.SyncAggregate( + sync_committee_bits=[index in participant_indices for index in range(len(committee_indices))], + sync_committee_signature=signature, + ) + +def build_random_block_from_state_for_next_slot(spec, state, rng=Random(2188)): # prepare state for deposits before building block deposits = prepare_state_and_get_random_deposits(spec, state, rng) block = build_empty_block_for_next_slot(spec, state) - block.body.proposer_slashings = get_random_proposer_slashings(spec, state, rng) - block.body.attester_slashings = get_random_attester_slashings(spec, state, rng) + proposer_slashings = get_random_proposer_slashings(spec, state, rng) + block.body.proposer_slashings = proposer_slashings + slashed_indices = [ + slashing.signed_header_1.message.proposer_index + for slashing in proposer_slashings + ] + block.body.attester_slashings = get_random_attester_slashings(spec, state, rng, slashed_indices) block.body.attestations = get_random_attestations(spec, state, rng) block.body.deposits = deposits @@ -148,6 +216,15 @@ def run_test_full_random_operations(spec, state, rng=Random(2080)): slashed_indices = slashed_indices.union(attester_slashing.attestation_2.attesting_indices) block.body.voluntary_exits = get_random_voluntary_exits(spec, state, slashed_indices, rng) + return block + + +def run_test_full_random_operations(spec, state, rng=Random(2080)): + # move state forward SHARD_COMMITTEE_PERIOD epochs to allow for exit + state.slot += spec.config.SHARD_COMMITTEE_PERIOD * spec.SLOTS_PER_EPOCH + + block = build_random_block_from_state_for_next_slot(spec, state, rng) + yield 'pre', state signed_block = state_transition_and_sign_block(spec, state, block) diff --git a/tests/core/pyspec/eth2spec/test/helpers/random.py b/tests/core/pyspec/eth2spec/test/helpers/random.py index 70c871a34..8f095aebb 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/random.py +++ b/tests/core/pyspec/eth2spec/test/helpers/random.py @@ -20,16 +20,20 @@ def set_some_new_deposits(spec, state, rng): state.validators[index].activation_eligibility_epoch = spec.get_current_epoch(state) -def exit_random_validators(spec, state, rng): +def exit_random_validators(spec, state, rng, fraction=None): + if fraction is None: + # Exit ~1/2 + fraction = 0.5 + if spec.get_current_epoch(state) < 5: # Move epochs forward to allow for some validators already exited/withdrawable for _ in range(5): next_epoch(spec, state) current_epoch = spec.get_current_epoch(state) - # Exit ~1/2 of validators for index in spec.get_active_validator_indices(state, current_epoch): - if rng.choice([True, False]): + sampled = rng.random() < fraction + if not sampled: continue validator = state.validators[index] @@ -41,11 +45,15 @@ def exit_random_validators(spec, state, rng): validator.withdrawable_epoch = current_epoch + 1 -def slash_random_validators(spec, state, rng): - # Slash ~1/2 of validators +def slash_random_validators(spec, state, rng, fraction=None): + if fraction is None: + # Slash ~1/2 of validators + fraction = 0.5 + for index in range(len(state.validators)): # slash at least one validator - if index == 0 or rng.choice([True, False]): + sampled = rng.random() < fraction + if index == 0 or sampled: spec.slash_validator(state, index) @@ -115,8 +123,8 @@ def randomize_attestation_participation(spec, state, rng=Random(8020)): randomize_epoch_participation(spec, state, spec.get_current_epoch(state), rng) -def randomize_state(spec, state, rng=Random(8020)): +def randomize_state(spec, state, rng=Random(8020), exit_fraction=None, slash_fraction=None): set_some_new_deposits(spec, state, rng) - exit_random_validators(spec, state, rng) - slash_random_validators(spec, state, rng) + exit_random_validators(spec, state, rng, fraction=exit_fraction) + slash_random_validators(spec, state, rng, fraction=slash_fraction) randomize_attestation_participation(spec, state, rng) diff --git a/tests/core/pyspec/eth2spec/test/helpers/state.py b/tests/core/pyspec/eth2spec/test/helpers/state.py index 666023fec..b4c9e1d67 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/state.py +++ b/tests/core/pyspec/eth2spec/test/helpers/state.py @@ -1,5 +1,6 @@ from eth2spec.test.context import expect_assertion_error, is_post_altair from eth2spec.test.helpers.block import apply_empty_block, sign_block, transition_unsigned_block +from eth2spec.test.helpers.voluntary_exits import get_exited_validators def get_balance(state, index): @@ -133,3 +134,24 @@ def _set_empty_participation(spec, state, current=True, previous=True): def set_empty_participation(spec, state, rng=None): _set_empty_participation(spec, state) + + +def ensure_state_has_validators_across_lifecycle(spec, state): + """ + Scan the validator registry to ensure there is at least 1 validator + for each of the following lifecycle states: + 1. Pending / deposited + 2. Active + 3. Exited + 4. Slashed + """ + has_pending = any(filter(spec.is_eligible_for_activation_queue, state.validators)) + + current_epoch = spec.get_current_epoch(state) + has_active = any(filter(lambda v: spec.is_active_validator(v, current_epoch), state.validators)) + + has_exited = any(get_exited_validators(spec, state)) + + has_slashed = any(filter(lambda v: v.slashed, state.validators)) + + return has_pending and has_active and has_exited and has_slashed diff --git a/tests/core/pyspec/eth2spec/test/phase0/random/__init__.py b/tests/core/pyspec/eth2spec/test/phase0/random/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/pyspec/eth2spec/test/phase0/random/test_random.py b/tests/core/pyspec/eth2spec/test/phase0/random/test_random.py new file mode 100644 index 000000000..89a457bed --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/phase0/random/test_random.py @@ -0,0 +1,438 @@ +""" +This module is generated from the ``random`` test generator. +Please do not edit this file manually. +See the README for that generator for more information. +""" + +from eth2spec.test.helpers.constants import PHASE0 +from eth2spec.test.context import ( + misc_balances_in_default_range_with_many_validators, + with_phases, + zero_activation_threshold, + only_generator, +) +from eth2spec.test.context import ( + always_bls, + spec_test, + with_custom_state, + single_phase, +) +from eth2spec.test.utils.randomized_block_tests import ( + run_generated_randomized_test, +) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_0(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_1(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_2(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_3(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_4(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_5(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_6(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_7(spec, state): + # scenario as high-level, informal text: + # epochs:0,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'validation': 'validate_is_not_leaking', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_8(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_9(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_10(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_11(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_12(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:last_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'last_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_13(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:random_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'random_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_14(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:penultimate_slot_in_epoch,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 0, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 'penultimate_slot_in_epoch', 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) + + +@only_generator("randomized test for broad coverage, not point-to-point CI") +@with_phases([PHASE0]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_15(spec, state): + # scenario as high-level, informal text: + # epochs:epochs_until_leak,slots:0,with-block:no_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block + # epochs:1,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:no_block + # epochs:0,slots:0,with-block:random_block + scenario = {'transitions': [{'epochs_to_skip': 'epochs_until_leak', 'validation': 'validate_is_leaking', 'slots_to_skip': 0, 'block_producer': 'no_block'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}, {'epochs_to_skip': 1, 'slots_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'slots_to_skip': 0, 'epochs_to_skip': 0, 'block_producer': 'no_block', 'validation': 'no_op_validation'}, {'block_producer': 'random_block', 'epochs_to_skip': 0, 'slots_to_skip': 0, 'validation': 'no_op_validation'}], 'state_randomizer': 'randomize_state'} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + ) diff --git a/tests/core/pyspec/eth2spec/test/utils/__init__.py b/tests/core/pyspec/eth2spec/test/utils/__init__.py new file mode 100644 index 000000000..f6b2a8a44 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/utils/__init__.py @@ -0,0 +1,12 @@ +from .utils import ( + vector_test, + with_meta_tags, + build_transition_test, +) + + +__all__ = [ # avoid "unused import" lint error + "vector_test", + "with_meta_tags", + "build_transition_test", +] diff --git a/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py new file mode 100644 index 000000000..44dab0e0e --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/utils/randomized_block_tests.py @@ -0,0 +1,319 @@ +""" +Utility code to generate randomized block tests +""" + +import sys +import warnings +from random import Random +from typing import Callable + +from eth2spec.test.helpers.multi_operations import ( + build_random_block_from_state_for_next_slot, + get_random_sync_aggregate, +) +from eth2spec.test.helpers.inactivity_scores import ( + randomize_inactivity_scores, +) +from eth2spec.test.helpers.random import ( + randomize_state as randomize_state_helper, +) +from eth2spec.test.helpers.state import ( + next_slot, + next_epoch, + ensure_state_has_validators_across_lifecycle, + state_transition_and_sign_block, +) + +# primitives: +# state + + +def randomize_state(spec, state, exit_fraction=0.1, slash_fraction=0.1): + randomize_state_helper(spec, state, exit_fraction=exit_fraction, slash_fraction=slash_fraction) + + +def randomize_state_altair(spec, state): + randomize_state(spec, state, exit_fraction=0.1, slash_fraction=0.1) + randomize_inactivity_scores(spec, state) + + +# epochs + +def epochs_until_leak(spec): + """ + State is "leaking" if the current epoch is at least + this value after the last finalized epoch. + """ + return spec.MIN_EPOCHS_TO_INACTIVITY_PENALTY + 1 + + +def epochs_for_shard_committee_period(spec): + return spec.config.SHARD_COMMITTEE_PERIOD + + +# slots + +def last_slot_in_epoch(spec): + return spec.SLOTS_PER_EPOCH - 1 + + +def random_slot_in_epoch(spec, rng=Random(1336)): + return rng.randrange(1, spec.SLOTS_PER_EPOCH - 2) + + +def penultimate_slot_in_epoch(spec): + return spec.SLOTS_PER_EPOCH - 2 + + +# blocks + +def no_block(_spec, _pre_state, _signed_blocks): + return None + + +# May need to make several attempts to find a block that does not correspond to a slashed +# proposer with the randomization helpers... +BLOCK_ATTEMPTS = 32 + + +def _warn_if_empty_operations(block): + if len(block.body.deposits) == 0: + warnings.warn(f"deposits missing in block at slot {block.slot}") + + if len(block.body.proposer_slashings) == 0: + warnings.warn(f"proposer slashings missing in block at slot {block.slot}") + + if len(block.body.attester_slashings) == 0: + warnings.warn(f"attester slashings missing in block at slot {block.slot}") + + if len(block.body.attestations) == 0: + warnings.warn(f"attestations missing in block at slot {block.slot}") + + if len(block.body.voluntary_exits) == 0: + warnings.warn(f"voluntary exits missing in block at slot {block.slot}") + + +def random_block(spec, state, _signed_blocks): + """ + Produce a random block. + NOTE: this helper may mutate state, as it will attempt + to produce a block over ``BLOCK_ATTEMPTS`` slots in order + to find a valid block in the event that the proposer has already been slashed. + """ + # NOTE: ``state`` has been "randomized" at this point and so will likely + # contain a large number of slashed validators. This function needs to return + # a valid block so it needs to check that the proposer of the next slot is not + # slashed. + # To do this, generate a ``temp_state`` to use for checking the propser in the next slot. + # This ensures no accidental mutations happen to the ``state`` the caller expects to get back + # after this function returns. + # Using a copy of the state for proposer sampling is also sound as any inputs used for the + # shuffling are fixed a few epochs prior to ``spec.get_current_epoch(state)``. + temp_state = state.copy() + next_slot(spec, temp_state) + for _ in range(BLOCK_ATTEMPTS): + proposer_index = spec.get_beacon_proposer_index(temp_state) + proposer = state.validators[proposer_index] + if proposer.slashed: + next_slot(spec, state) + next_slot(spec, temp_state) + else: + block = build_random_block_from_state_for_next_slot(spec, state) + _warn_if_empty_operations(block) + return block + else: + raise AssertionError("could not find a block with an unslashed proposer, check ``state`` input") + + +SYNC_AGGREGATE_PARTICIPATION_BUCKETS = 4 + + +def random_block_altair_with_cycling_sync_committee_participation(spec, + state, + signed_blocks): + block = random_block(spec, state, signed_blocks) + block_index = len(signed_blocks) % SYNC_AGGREGATE_PARTICIPATION_BUCKETS + fraction_missed = block_index * (1 / SYNC_AGGREGATE_PARTICIPATION_BUCKETS) + fraction_participated = 1.0 - fraction_missed + block.body.sync_aggregate = get_random_sync_aggregate( + spec, + state, + block.slot - 1, + fraction_participated=fraction_participated, + ) + return block + + +# validations + +def no_op_validation(spec, state): + return True + + +def validate_is_leaking(spec, state): + return spec.is_in_inactivity_leak(state) + + +def validate_is_not_leaking(spec, state): + return not validate_is_leaking(spec, state) + + +# transitions + +def with_validation(transition, validation): + if isinstance(transition, Callable): + transition = transition() + transition["validation"] = validation + return transition + + +def no_op_transition(): + return {} + + +def epoch_transition(n=0): + return { + "epochs_to_skip": n, + } + + +def slot_transition(n=0): + return { + "slots_to_skip": n, + } + + +def transition_to_leaking(): + return { + "epochs_to_skip": epochs_until_leak, + "validation": validate_is_leaking, + } + + +transition_without_leak = with_validation(no_op_transition, validate_is_not_leaking) + +# block transitions + + +def transition_with_random_block(block_randomizer): + """ + Build a block transition with randomized data. + Provide optional sub-transitions to advance some + number of epochs or slots before applying the random block. + """ + return { + "block_producer": block_randomizer, + } + + +# setup and test gen + + +def _randomized_scenario_setup(state_randomizer): + """ + Return a sequence of pairs of ("mutation", "validation"), + a function that accepts (spec, state) arguments and performs some change + and a function that accepts (spec, state) arguments and validates some change was made. + """ + def _skip_epochs(epoch_producer): + def f(spec, state): + """ + The unoptimized spec implementation is too slow to advance via ``next_epoch``. + Instead, just overwrite the ``state.slot`` and continue... + """ + epochs_to_skip = epoch_producer(spec) + slots_to_skip = epochs_to_skip * spec.SLOTS_PER_EPOCH + state.slot += slots_to_skip + return f + + def _simulate_honest_execution(spec, state): + """ + Want to start tests not in a leak state; the finality data + may not reflect this condition with prior (arbitrary) mutations, + so this mutator addresses that fact. + """ + state.justification_bits = (True, True, True, True) + previous_epoch = spec.get_previous_epoch(state) + previous_root = spec.get_block_root(state, previous_epoch) + previous_previous_epoch = max(spec.GENESIS_EPOCH, spec.Epoch(previous_epoch - 1)) + previous_previous_root = spec.get_block_root(state, previous_previous_epoch) + state.previous_justified_checkpoint = spec.Checkpoint( + epoch=previous_previous_epoch, + root=previous_previous_root, + ) + state.current_justified_checkpoint = spec.Checkpoint( + epoch=previous_epoch, + root=previous_root, + ) + state.finalized_checkpoint = spec.Checkpoint( + epoch=previous_previous_epoch, + root=previous_previous_root, + ) + + return ( + # NOTE: the block randomization function assumes at least 1 shard committee period + # so advance the state before doing anything else. + (_skip_epochs(epochs_for_shard_committee_period), no_op_validation), + (_simulate_honest_execution, no_op_validation), + (state_randomizer, ensure_state_has_validators_across_lifecycle), + ) + +# Run the generated tests: + + +# while the test implementation works via code-gen, +# references to helper code in this module are serialized as str names. +# to resolve this references at runtime, we need a reference to this module: +_this_module = sys.modules[__name__] + + +def _resolve_ref(ref): + if isinstance(ref, str): + return getattr(_this_module, ref) + return ref + + +def _iter_temporal(spec, description): + """ + Intended to advance some number of {epochs, slots}. + Caller can provide a constant integer or a callable deriving a number from + the ``spec`` under consideration. + """ + numeric = _resolve_ref(description) + if isinstance(numeric, Callable): + numeric = numeric(spec) + for i in range(numeric): + yield i + + +def run_generated_randomized_test(spec, state, scenario): + if "setup" not in scenario: + state_randomizer = _resolve_ref(scenario.get("state_randomizer", randomize_state)) + scenario["setup"] = _randomized_scenario_setup(state_randomizer) + + for mutation, validation in scenario["setup"]: + mutation(spec, state) + validation(spec, state) + + yield "pre", state + + blocks = [] + for transition in scenario["transitions"]: + epochs_to_skip = _iter_temporal(spec, transition["epochs_to_skip"]) + for _ in epochs_to_skip: + next_epoch(spec, state) + slots_to_skip = _iter_temporal(spec, transition["slots_to_skip"]) + for _ in slots_to_skip: + next_slot(spec, state) + + block_producer = _resolve_ref(transition["block_producer"]) + block = block_producer(spec, state, blocks) + if block: + signed_block = state_transition_and_sign_block(spec, state, block) + blocks.append(signed_block) + + validation = _resolve_ref(transition["validation"]) + assert validation(spec, state) + + yield "blocks", blocks + yield "post", state diff --git a/tests/core/pyspec/eth2spec/test/utils.py b/tests/core/pyspec/eth2spec/test/utils/utils.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/utils.py rename to tests/core/pyspec/eth2spec/test/utils/utils.py diff --git a/tests/generators/random/Makefile b/tests/generators/random/Makefile new file mode 100644 index 000000000..1b518bfde --- /dev/null +++ b/tests/generators/random/Makefile @@ -0,0 +1,8 @@ +all: + if ! test -d venv; then python3 -m venv venv; fi; + . ./venv/bin/activate + pip3 install -r requirements.txt + rm -f ../../core/pyspec/eth2spec/test/phase0/random/test_random.py + rm -f ../../core/pyspec/eth2spec/test/altair/random/test_random.py + python3 generate.py phase0 > ../../core/pyspec/eth2spec/test/phase0/random/test_random.py + python3 generate.py altair > ../../core/pyspec/eth2spec/test/altair/random/test_random.py diff --git a/tests/generators/random/README.md b/tests/generators/random/README.md new file mode 100644 index 000000000..fd1728441 --- /dev/null +++ b/tests/generators/random/README.md @@ -0,0 +1,31 @@ +# Randomized tests + +Randomized tests in the format of `sanity` blocks tests, with randomized operations. + +Information on the format of the tests can be found in the [sanity test formats documentation](../../formats/sanity/README.md). + +# To generate test sources + +```bash +$ make +``` + +The necessary commands are in the `Makefile`, as the only target. + +The generated files are committed to the repo so you should not need to do this. + +# To run tests + +Each of the generated test does produce a `pytest` test instance but by default is +currently skipped. Running the test via the generator (see next) will trigger any errors +that would arise during the running of `pytest`. + +# To generate spec tests (from the generated files) + +Run the test generator in the usual way. + +E.g. from the root of this repo, you can run: + +```bash +$ make gen_random +``` diff --git a/tests/generators/random/generate.py b/tests/generators/random/generate.py new file mode 100644 index 000000000..b880cb12b --- /dev/null +++ b/tests/generators/random/generate.py @@ -0,0 +1,258 @@ +""" +This test format currently uses code generation to assemble the tests +as the current test infra does not have a facility to dynamically +generate tests that can be seen by ``pytest``. + +This will likley change in future releases of the testing infra. + +NOTE: To add additional scenarios, add test cases below in ``_generate_randomized_scenarios``. +""" + +import sys +import random +import warnings +from typing import Callable +import itertools + +from eth2spec.test.utils.randomized_block_tests import ( + no_block, + no_op_validation, + randomize_state, + randomize_state_altair, + random_block, + random_block_altair_with_cycling_sync_committee_participation, + last_slot_in_epoch, + random_slot_in_epoch, + penultimate_slot_in_epoch, + epoch_transition, + slot_transition, + transition_with_random_block, + transition_to_leaking, + transition_without_leak, +) +from eth2spec.test.helpers.constants import PHASE0, ALTAIR + + +# Ensure this many blocks are present in *each* randomized scenario +BLOCK_TRANSITIONS_COUNT = 2 + + +def _normalize_transition(transition): + """ + Provide "empty" or "no op" sub-transitions + to a given transition. + """ + if isinstance(transition, Callable): + transition = transition() + if "epochs_to_skip" not in transition: + transition["epochs_to_skip"] = 0 + if "slots_to_skip" not in transition: + transition["slots_to_skip"] = 0 + if "block_producer" not in transition: + transition["block_producer"] = no_block + if "validation" not in transition: + transition["validation"] = no_op_validation + return transition + + +def _normalize_scenarios(scenarios): + """ + "Normalize" a "scenario" so that a producer of a test case + does not need to provide every expected key/value. + """ + for scenario in scenarios: + transitions = scenario["transitions"] + for i, transition in enumerate(transitions): + transitions[i] = _normalize_transition(transition) + + +def _flatten(t): + leak_transition = t[0] + result = [leak_transition] + for transition_batch in t[1]: + for transition in transition_batch: + if isinstance(transition, tuple): + for subtransition in transition: + result.append(subtransition) + else: + result.append(transition) + return result + + +def _generate_randomized_scenarios(block_randomizer): + """ + Generates a set of randomized testing scenarios. + Return a sequence of "scenarios" where each scenario: + 1. Provides some setup + 2. Provides a sequence of transitions that mutate the state in some way, + possibly yielding blocks along the way + NOTE: scenarios are "normalized" with empty/no-op elements before returning + to the test generation to facilitate brevity when writing scenarios by hand. + NOTE: the main block driver builds a block for the **next** slot, so + the slot transitions are offset by -1 to target certain boundaries. + """ + # go forward 0 or 1 epochs + epochs_set = ( + epoch_transition(n=0), + epoch_transition(n=1), + ) + # within those epochs, go forward to: + slots_set = ( + # the first slot in an epoch (see note in docstring about offsets...) + slot_transition(last_slot_in_epoch), + # the second slot in an epoch + slot_transition(n=0), + # some random number of slots, but not at epoch boundaries + slot_transition(random_slot_in_epoch), + # the last slot in an epoch (see note in docstring about offsets...) + slot_transition(penultimate_slot_in_epoch), + ) + # and produce a block... + blocks_set = ( + transition_with_random_block(block_randomizer), + ) + + rng = random.Random(1447) + all_skips = list(itertools.product(epochs_set, slots_set)) + randomized_skips = ( + rng.sample(all_skips, len(all_skips)) + for _ in range(BLOCK_TRANSITIONS_COUNT) + ) + + # build a set of block transitions from combinations of sub-transitions + transitions_generator = ( + itertools.product(prefix, blocks_set) + for prefix in randomized_skips + ) + block_transitions = zip(*transitions_generator) + + # and preface each set of block transitions with the possible leak transitions + leak_transitions = ( + transition_without_leak, + transition_to_leaking, + ) + scenarios = [ + {"transitions": _flatten(t)} + for t in itertools.product(leak_transitions, block_transitions) + ] + _normalize_scenarios(scenarios) + return scenarios + + +def _id_from_scenario(test_description): + """ + Construct a name for the scenario based its data. + """ + def _to_id_part(prefix, x): + suffix = str(x) + if isinstance(x, Callable): + suffix = x.__name__ + return f"{prefix}{suffix}" + + def _id_from_transition(transition): + return ",".join(( + _to_id_part("epochs:", transition["epochs_to_skip"]), + _to_id_part("slots:", transition["slots_to_skip"]), + _to_id_part("with-block:", transition["block_producer"]) + )) + + return "|".join(map(_id_from_transition, test_description["transitions"])) + + +test_imports_template = """\"\"\" +This module is generated from the ``random`` test generator. +Please do not edit this file manually. +See the README for that generator for more information. +\"\"\" + +from eth2spec.test.helpers.constants import {phase} +from eth2spec.test.context import ( + misc_balances_in_default_range_with_many_validators, + with_phases, + zero_activation_threshold, + only_generator, +) +from eth2spec.test.context import ( + always_bls, + spec_test, + with_custom_state, + single_phase, +) +from eth2spec.test.utils.randomized_block_tests import ( + run_generated_randomized_test, +)""" + +test_template = """ +@only_generator(\"randomized test for broad coverage, not point-to-point CI\") +@with_phases([{phase}]) +@with_custom_state( + balances_fn=misc_balances_in_default_range_with_many_validators, + threshold_fn=zero_activation_threshold +) +@spec_test +@single_phase +@always_bls +def test_randomized_{index}(spec, state): + # scenario as high-level, informal text: +{name_as_comment} + scenario = {scenario} # noqa: E501 + yield from run_generated_randomized_test( + spec, + state, + scenario, + )""" + + +def _to_comment(name, indent_level): + parts = name.split("|") + indentation = " " * indent_level + parts = [ + indentation + "# " + part for part in parts + ] + return "\n".join(parts) + + +def run_generate_tests_to_std_out(phase, state_randomizer, block_randomizer): + scenarios = _generate_randomized_scenarios(block_randomizer) + test_content = {"phase": phase.upper()} + test_imports = test_imports_template.format(**test_content) + test_file = [test_imports] + for index, scenario in enumerate(scenarios): + # required for setup phase + scenario["state_randomizer"] = state_randomizer.__name__ + + # need to pass name, rather than function reference... + transitions = scenario["transitions"] + for transition in transitions: + for name, value in transition.items(): + if isinstance(value, Callable): + transition[name] = value.__name__ + + test_content = test_content.copy() + name = _id_from_scenario(scenario) + test_content["name_as_comment"] = _to_comment(name, 1) + test_content["index"] = index + test_content["scenario"] = scenario + test_instance = test_template.format(**test_content) + test_file.append(test_instance) + print("\n\n".join(test_file)) + + +if __name__ == "__main__": + did_generate = False + if PHASE0 in sys.argv: + did_generate = True + run_generate_tests_to_std_out( + PHASE0, + state_randomizer=randomize_state, + block_randomizer=random_block, + ) + if ALTAIR in sys.argv: + did_generate = True + run_generate_tests_to_std_out( + ALTAIR, + state_randomizer=randomize_state_altair, + block_randomizer=random_block_altair_with_cycling_sync_committee_participation, + ) + if not did_generate: + warnings.warn("no phase given for test generation") diff --git a/tests/generators/random/main.py b/tests/generators/random/main.py new file mode 100644 index 000000000..f6f1b1847 --- /dev/null +++ b/tests/generators/random/main.py @@ -0,0 +1,18 @@ +from eth2spec.test.helpers.constants import PHASE0, ALTAIR +from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators + + +if __name__ == "__main__": + phase_0_mods = {key: 'eth2spec.test.phase0.random.test_' + key for key in [ + 'random', + ]} + altair_mods = {key: 'eth2spec.test.altair.random.test_' + key for key in [ + 'random', + ]} + + all_mods = { + PHASE0: phase_0_mods, + ALTAIR: altair_mods, + } + + run_state_test_generators(runner_name="random", all_mods=all_mods) diff --git a/tests/generators/random/requirements.txt b/tests/generators/random/requirements.txt new file mode 100644 index 000000000..182248686 --- /dev/null +++ b/tests/generators/random/requirements.txt @@ -0,0 +1,2 @@ +pytest>=4.4 +../../../[generator]