From 0a86ab4b59a71530cfb7f7188495b089f02acac2 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 7 Nov 2023 12:49:20 +1100 Subject: [PATCH 01/12] Introduce a shuffling prefix for subnet determination --- specs/phase0/p2p-interface.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index a374443b8c..00e6ee86da 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -203,7 +203,8 @@ This section outlines configurations that are used in this spec. | `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | | `ATTESTATION_SUBNET_COUNT` | `2**6` (= 64) | The number of attestation subnets used in the gossipsub protocol. | | `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | -| `ATTESTATION_SUBNET_PREFIX_BITS` | `int(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | +| `ATTESTATION_SUBNET_PREFIX_BITS` | `int(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | +| `ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS` | `3` | The number of bits used to shuffle nodes to a new subnet within `EPOCHS_PER_SUBNET_SUBSCRIPTION` | | ### MetaData @@ -1013,7 +1014,8 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th ```python def compute_subscribed_subnet(node_id: NodeID, epoch: Epoch, index: int) -> SubnetID: node_id_prefix = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS) - node_offset = node_id % EPOCHS_PER_SUBNET_SUBSCRIPTION + shuffling_prefix = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS - ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS) + node_offset = (shuffling_prefix + (shuffling_prefix * (EPOCHS_PER_SUBNET_SUBSCRIPTION // 1 << ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS))) % EPOCHS_PER_SUBNET_SUBSCRIPTION permutation_seed = hash(uint_to_bytes(uint64((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) permutated_prefix = compute_shuffled_index( node_id_prefix, From ec9fbf24a266f2d98131204d487d0ac704cfe23b Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 8 Nov 2023 12:00:10 +1100 Subject: [PATCH 02/12] Split the bits in the calculation to clarify the logic --- specs/phase0/p2p-interface.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 00e6ee86da..35df0f3589 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -1013,12 +1013,12 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th ```python def compute_subscribed_subnet(node_id: NodeID, epoch: Epoch, index: int) -> SubnetID: - node_id_prefix = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS) - shuffling_prefix = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS - ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS) - node_offset = (shuffling_prefix + (shuffling_prefix * (EPOCHS_PER_SUBNET_SUBSCRIPTION // 1 << ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS))) % EPOCHS_PER_SUBNET_SUBSCRIPTION - permutation_seed = hash(uint_to_bytes(uint64((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) + subnet_prefix = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS) + shuffling_bits = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS - ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS) % (1 << ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS) + epoch_transition = (subnet_prefix + (shuffling_bits * (EPOCHS_PER_SUBNET_SUBSCRIPTION >> ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS))) % EPOCHS_PER_SUBNET_SUBSCRIPTION + permutation_seed = hash(uint_to_bytes(uint64((epoch + epoch_transition) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) permutated_prefix = compute_shuffled_index( - node_id_prefix, + subnet_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed, ) From 22def22367781c0a950f3f5a7f20567642efeacc Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 30 Nov 2023 12:44:45 +1100 Subject: [PATCH 03/12] Update specs/phase0/p2p-interface.md Co-authored-by: Divma <26765164+divagant-martian@users.noreply.github.com> --- specs/phase0/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 35df0f3589..85ee8f0ac9 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -1014,7 +1014,7 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th ```python def compute_subscribed_subnet(node_id: NodeID, epoch: Epoch, index: int) -> SubnetID: subnet_prefix = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS) - shuffling_bits = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS - ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS) % (1 << ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS) + shuffling_bits = (node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS - ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS)) % (1 << ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS) epoch_transition = (subnet_prefix + (shuffling_bits * (EPOCHS_PER_SUBNET_SUBSCRIPTION >> ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS))) % EPOCHS_PER_SUBNET_SUBSCRIPTION permutation_seed = hash(uint_to_bytes(uint64((epoch + epoch_transition) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) permutated_prefix = compute_shuffled_index( From 48e685c6fd188a327d14a43edd02110456822582 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 Jan 2024 23:14:21 +0800 Subject: [PATCH 04/12] Update config files --- configs/mainnet.yaml | 1 + configs/minimal.yaml | 1 + specs/phase0/p2p-interface.md | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index e0f9128b70..9da292e378 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -138,6 +138,7 @@ ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS ATTESTATION_SUBNET_PREFIX_BITS: 6 +ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3 # Deneb # `2**7` (=128) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index cdfbca3a2c..98ef69aac3 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -139,6 +139,7 @@ ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS ATTESTATION_SUBNET_PREFIX_BITS: 6 +ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3 # Deneb # `2**7` (=128) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 85ee8f0ac9..e93a838996 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -202,7 +202,7 @@ This section outlines configurations that are used in this spec. | `MESSAGE_DOMAIN_VALID_SNAPPY` | `DomainType('0x01000000')` | 4-byte domain for gossip message-id isolation of *valid* snappy messages | | `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | | `ATTESTATION_SUBNET_COUNT` | `2**6` (= 64) | The number of attestation subnets used in the gossipsub protocol. | -| `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | +| `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | | | `ATTESTATION_SUBNET_PREFIX_BITS` | `int(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | `ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS` | `3` | The number of bits used to shuffle nodes to a new subnet within `EPOCHS_PER_SUBNET_SUBSCRIPTION` | | From 2fe834694d47f0ff540ace7d1080365d7644eba0 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 15 Jan 2024 19:40:03 +1100 Subject: [PATCH 05/12] Add temporary values for lint --- specs/phase0/p2p-interface.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index e93a838996..d81f127a90 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -1014,8 +1014,10 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th ```python def compute_subscribed_subnet(node_id: NodeID, epoch: Epoch, index: int) -> SubnetID: subnet_prefix = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS) - shuffling_bits = (node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS - ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS)) % (1 << ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS) - epoch_transition = (subnet_prefix + (shuffling_bits * (EPOCHS_PER_SUBNET_SUBSCRIPTION >> ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS))) % EPOCHS_PER_SUBNET_SUBSCRIPTION + shuffling_bit_size = NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS - ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS + shuffling_bits = (node_id >> shuffling_bit_size) % (1 << ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS) + shuffling_multiplier = EPOCHS_PER_SUBNET_SUBSCRIPTION >> ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS + epoch_transition = (subnet_prefix + (shuffling_bits * shuffling_multiplier)) % EPOCHS_PER_SUBNET_SUBSCRIPTION permutation_seed = hash(uint_to_bytes(uint64((epoch + epoch_transition) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) permutated_prefix = compute_shuffled_index( subnet_prefix, From 1ec248a19d579b54c22aedf578905ce337ab5d78 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 Jan 2024 23:27:00 +0800 Subject: [PATCH 06/12] Fix lint --- specs/phase0/p2p-interface.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index d81f127a90..bfe51c70eb 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -1014,10 +1014,16 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th ```python def compute_subscribed_subnet(node_id: NodeID, epoch: Epoch, index: int) -> SubnetID: subnet_prefix = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS) - shuffling_bit_size = NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS - ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS + shuffling_bit_size = ( + NODE_ID_BITS + - ATTESTATION_SUBNET_PREFIX_BITS + - ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS + ) shuffling_bits = (node_id >> shuffling_bit_size) % (1 << ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS) shuffling_multiplier = EPOCHS_PER_SUBNET_SUBSCRIPTION >> ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS - epoch_transition = (subnet_prefix + (shuffling_bits * shuffling_multiplier)) % EPOCHS_PER_SUBNET_SUBSCRIPTION + epoch_transition = ( + (subnet_prefix + (shuffling_bits * shuffling_multiplier)) % EPOCHS_PER_SUBNET_SUBSCRIPTION + ) permutation_seed = hash(uint_to_bytes(uint64((epoch + epoch_transition) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) permutated_prefix = compute_shuffled_index( subnet_prefix, From 63ca0bfa9cf475bc011fc0cfc19381d160ce868f Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 14 Mar 2024 17:00:41 +1100 Subject: [PATCH 07/12] Network shards --- configs/mainnet.yaml | 10 ++-- configs/minimal.yaml | 9 ++-- specs/phase0/p2p-interface.md | 94 +++++++++++++++++++++++++++-------- 3 files changed, 84 insertions(+), 29 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 3be48914df..e565626d2f 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -135,10 +135,12 @@ MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 SUBNETS_PER_NODE: 2 # 2**8 (= 64) ATTESTATION_SUBNET_COUNT: 64 -ATTESTATION_SUBNET_EXTRA_BITS: 0 -# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS -ATTESTATION_SUBNET_PREFIX_BITS: 6 -ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3 +# The granularity of the network. +NETWORK_SHARD_COUNT: 64 +NETWORK_SHARD_EXTRA_BITS: 0 +# ceillog2(NETWORK_SHARD_COUNT) + NETWORK_SHARD_EXTRA_BITS +NETWORK_SHARD_PREFIX_BITS: 6 +NETWORK_SHARD_SHUFFLING_PREFIX_BITS: 3 # Deneb # `2**7` (=128) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 0522136c30..9542f579b3 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -136,10 +136,11 @@ MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 SUBNETS_PER_NODE: 2 # 2**8 (= 64) ATTESTATION_SUBNET_COUNT: 64 -ATTESTATION_SUBNET_EXTRA_BITS: 0 -# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS -ATTESTATION_SUBNET_PREFIX_BITS: 6 -ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3 +NETWORK_SHARD_COUNT: 64 +NETWORK_SHARD_EXTRA_BITS: 0 +# ceillog2(NETWORK_SHARD_COUNT) + NETWORK_SHARD_EXTRA_BITS +NETWORK_SHARD_PREFIX_BITS: 6 +NETWORK_SHARD_SHUFFLING_PREFIX_BITS: 3 # Deneb # `2**7` (=128) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index f06c59c678..60ed87b1ea 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -191,7 +191,7 @@ This section outlines configurations that are used in this spec. |---|---|---| | `GOSSIP_MAX_SIZE` | `10 * 2**20` (= 10485760, 10 MiB) | The maximum allowed size of uncompressed gossip messages. | | `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | -| `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on a subnet subscription (~27 hours) | +| `EPOCHS_PER_SHARD_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on a subnet subscription (~27 hours) | | `MIN_EPOCHS_FOR_BLOCK_REQUESTS` | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) | The minimum epoch range over which a node must serve blocks | | `MAX_CHUNK_SIZE` | `10 * 2**20` (=10485760, 10 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | | `TTFB_TIMEOUT` | `5` | The maximum duration in **seconds** to wait for first byte of request response (time-to-first-byte). | @@ -202,9 +202,10 @@ This section outlines configurations that are used in this spec. | `MESSAGE_DOMAIN_VALID_SNAPPY` | `DomainType('0x01000000')` | 4-byte domain for gossip message-id isolation of *valid* snappy messages | | `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | | `ATTESTATION_SUBNET_COUNT` | `2**6` (= 64) | The number of attestation subnets used in the gossipsub protocol. | -| `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | | -| `ATTESTATION_SUBNET_PREFIX_BITS` | `int(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | -| `ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS` | `3` | The number of bits used to shuffle nodes to a new subnet within `EPOCHS_PER_SUBNET_SUBSCRIPTION` | | +| `NETWORK_SHARD_COUNT` | `2**6` (= 64) | The number of network shards. | +| `NETWORK_SHARD_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a network shard | +| `NETWORK_SHARD_PREFIX_BITS` | `int(ceillog2(NETWORK_SHARD_COUNT) + NETWORK_SHARD_EXTRA_BITS)` | +| `NETWORK_SHARD_SHUFFLING_PREFIX_BITS` | `3` | The number of bits used to shuffle nodes to a new shard within `EPOCHS_PER_SHARD_SUBSCRIPTION` | ### MetaData @@ -1011,39 +1012,90 @@ Clients MAY connect to peers with the same `fork_digest` but a different `next_f Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`. -### Attestation subnet subscription +### Network Shards -Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each beacon node should: +In order for gossipsub to function, there must be a stable set of peers for +each topic that remain subscribed for a long period of time (order 1 +day). This allows nodes to search for and maintain a selection of these +long-lived peers in order to publish/receive messages on the topic. As some +topics are transient for most nodes (i.e attestation subnets, DAS-related +columns) it is necessary that we enforce each node on the network to facilitate the +support of these topics by long-lived subscribing to them (and thereby validating and forwarding messages). -* Remain subscribed to `SUBNETS_PER_NODE` for `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs. -* Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. -* Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id, epoch)` function. +To this end we define the abstract concept of a "network shard". Each network +shard is mapped to one or many transient gossipsub topics that require a stable +set of subscribed peers. The primary advantage of this concept is that a node +need only to optimise their peer set to obtain a uniform set of peers on all +network shards, which will then guarantee a uniform set of peers on all transient +gossipsub topics (rather than trying to optimise for each individual set of +topics (i.e attestation_subnets, DAS-related columns). + +The mapping that links a node-id to a network shard is: ```python -def compute_subscribed_subnet(node_id: NodeID, epoch: Epoch, index: int) -> SubnetID: - subnet_prefix = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS) - shuffling_bit_size = ( +def compute_network_shard(node_id: NodeID, epoch: Epoch) -> SubnetID: + # The main prefix bits to determine a network shard + shard_prefix = node_id >> (NODE_ID_BITS - NETWORK_SHARD_PREFIX_BITS) + # Used to extract the total prefix bytes (prefix + shuffling_bits) + shuffling_bits = ( NODE_ID_BITS - - ATTESTATION_SUBNET_PREFIX_BITS - - ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS + - NETWORK_SHARD_PREFIX_BITS + - NETWORK_SHARD_SHUFFLING_PREFIX_BITS ) - shuffling_bits = (node_id >> shuffling_bit_size) % (1 << ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS) - shuffling_multiplier = EPOCHS_PER_SUBNET_SUBSCRIPTION >> ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS + # The NETWORK_SHARD_SHUFFLING_PREFIX_BITS that trail shard_prefix. + # These are used to stagger the rotation of network shards so that all + # nodes do not rotate from shards all at once. + # The larger the NETWORK_SHARD_SHUFFLING_PREFIX_BITS the more granular the + # nodes will transition from one shard to another throughout a period. + shuffling_bits = (node_id >> shuffling_bit_size) % (1 << NETWORK_SHARD_SHUFFLING_PREFIX_BITS) + # Calculates a multiplier that scales the shuffling prefix (assumed smaller + # than the rotation period) to be uniform throughout the entire rotation + # period. Can also be calculated as: + # EPOCHS_PER_SHARD_SUBSCRIPTION // 1 >> NETWORK_SHARD_SHUFFLING_PREFIX_BITS + shuffling_multiplier = EPOCHS_PER_SHARD_SUBSCRIPTION >> NETWORK_SHARD_SHUFFLING_PREFIX_BITS + # The epoch at which this node will rotate to a new shard + # This is distributed uniformly throughout the rotation period with a + # granularity based on the size of NETWORK_SHARD_SHUFFLING_PREFIX_BITS epoch_transition = ( - (subnet_prefix + (shuffling_bits * shuffling_multiplier)) % EPOCHS_PER_SUBNET_SUBSCRIPTION + (shard_prefix + (shuffling_bits * shuffling_multiplier)) % EPOCHS_PER_SHARD_SUBSCRIPTION ) + # A seed which changes every rotation period (EPOCHS_PER_SHARD_SUBSCRIPTION) + # This enforces the rotation period and is staggered for each prefix so + # that nodes do not rotate from shards all at once permutation_seed = hash(uint_to_bytes(uint64((epoch + epoch_transition) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) + # The resulting value that ultimately defines the network shard. permutated_prefix = compute_shuffled_index( - subnet_prefix, - 1 << ATTESTATION_SUBNET_PREFIX_BITS, + shard_prefix, + 1 << NETWORK_SHARD_PREFIX_BITS, permutation_seed, ) - return SubnetID((permutated_prefix + index) % ATTESTATION_SUBNET_COUNT) + return ShardID(permutated_prefix % NETWORK_SHARD_COUNT) ``` +The `compute_network_shard` function is designed with the following +desirable properties: +* It uses only the first set of bytes (defined by prefix_bytes_size) of the + node-id. This allows for efficient discovery searches, by allowing nodes to + search for specific nodes of network shards based on the kademilia XOR + metric. +* Nodes will maintain a shard for EPOCHS_PER_SHARD_SUBSCRIPTION before + rotating to a new shard. +* The rotation is staggered uniformly throughout the rotation period per shard. +* No individual shard will suddenly rotate to another, rather a subset of nodes + per shard transition gradually throughout the rotation period. +* The function is feasibly reversible. Prefixes can be calculated for desired + network shards for any given epoch, allowing nodes to search for these + prefixes. + +### Attestation Subnets + +The backbone structure for attestation subnets can be calculated from their +network shard via the following: + ```python def compute_subscribed_subnets(node_id: NodeID, epoch: Epoch) -> Sequence[SubnetID]: - return [compute_subscribed_subnet(node_id, epoch, index) for index in range(SUBNETS_PER_NODE)] + network_shard = compute_network_shard(node_id, epoch) + return [ShardId(network_shard + index) % ATTESTATION_SUBNET_COUNT for index in range(SUBNETS_PER_NODE)] ``` *Note*: When preparing for a hard fork, a node must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. From 51ffb74e88dfe8e12e3d6afc16ded5e78455c763 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 14 Mar 2024 17:25:15 +1100 Subject: [PATCH 08/12] Update epochs per shard --- configs/mainnet.yaml | 2 +- configs/minimal.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index e565626d2f..2347eab88e 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -117,7 +117,7 @@ GOSSIP_MAX_SIZE: 10485760 # `2**10` (= 1024) MAX_REQUEST_BLOCKS: 1024 # `2**8` (= 256) -EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 +EPOCHS_PER_SHARD_SUBSCRIPTION: 256 # `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 # `10 * 2**20` (=10485760, 10 MiB) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 9542f579b3..c61844788e 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -118,7 +118,7 @@ GOSSIP_MAX_SIZE: 10485760 # `2**10` (= 1024) MAX_REQUEST_BLOCKS: 1024 # `2**8` (= 256) -EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 +EPOCHS_PER_SHARD_SUBSCRIPTION: 256 # [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272 # `10 * 2**20` (=10485760, 10 MiB) From a700594fb92b5511bb97b4b2588cbc588d6e3631 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 18 Mar 2024 14:50:16 +1100 Subject: [PATCH 09/12] Include metadata and reviewers comments --- specs/phase0/p2p-interface.md | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 60ed87b1ea..1567db532a 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -191,7 +191,7 @@ This section outlines configurations that are used in this spec. |---|---|---| | `GOSSIP_MAX_SIZE` | `10 * 2**20` (= 10485760, 10 MiB) | The maximum allowed size of uncompressed gossip messages. | | `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | -| `EPOCHS_PER_SHARD_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on a subnet subscription (~27 hours) | +| `EPOCHS_PER_SHARD_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on a shard subscription (~27 hours) | | `MIN_EPOCHS_FOR_BLOCK_REQUESTS` | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) | The minimum epoch range over which a node must serve blocks | | `MAX_CHUNK_SIZE` | `10 * 2**20` (=10485760, 10 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | | `TTFB_TIMEOUT` | `5` | The maximum duration in **seconds** to wait for first byte of request response (time-to-first-byte). | @@ -214,7 +214,7 @@ Clients MUST locally store the following `MetaData`: ``` ( seq_number: uint64 - attnets: Bitvector[ATTESTATION_SUBNET_COUNT] + shards: Bitvector[NETWORK_SHARD_COUNT] ) ``` @@ -222,7 +222,7 @@ Where - `seq_number` is a `uint64` starting at `0` used to version the node's metadata. If any other field in the local `MetaData` changes, the node MUST increment `seq_number` by 1. -- `attnets` is a `Bitvector` representing the node's persistent attestation subnet subscriptions. +- `shards` is a `Bitvector` representing the node's persistent attestation subnet subscriptions. *Note*: `MetaData.seq_number` is used for versioning of the node's metadata, is entirely independent of the ENR sequence number, @@ -958,16 +958,16 @@ Specifications of these parameters can be found in the [ENR Specification](http: ##### Attestation subnet bitfield -The ENR `attnets` entry signifies the attestation subnet bitfield with the following form +The ENR `shards` entry signifies the attestation subnet bitfield with the following form to more easily discover peers participating in particular attestation gossip subnets. | Key | Value | |:-------------|:-------------------------------------------------| -| `attnets` | SSZ `Bitvector[ATTESTATION_SUBNET_COUNT]` | +| `shards` | SSZ `Bitvector[NETWORK_SHARD_COUNT]` | -If a node's `MetaData.attnets` has any non-zero bit, the ENR MUST include the `attnets` entry with the same value as `MetaData.attnets`. +If a node's `MetaData.shards` has any non-zero bit, the ENR MUST include the `shards` entry with the same value as `MetaData.shards`. -If a node's `MetaData.attnets` is composed of all zeros, the ENR MAY optionally include the `attnets` entry or leave it out entirely. +If a node's `MetaData.shards` is composed of all zeros, the ENR MAY optionally include the `shards` entry or leave it out entirely. ##### `eth2` field @@ -1033,11 +1033,11 @@ topics (i.e attestation_subnets, DAS-related columns). The mapping that links a node-id to a network shard is: ```python -def compute_network_shard(node_id: NodeID, epoch: Epoch) -> SubnetID: +def compute_network_shard(node_id: NodeID, epoch: Epoch) -> ShardID: # The main prefix bits to determine a network shard shard_prefix = node_id >> (NODE_ID_BITS - NETWORK_SHARD_PREFIX_BITS) # Used to extract the total prefix bytes (prefix + shuffling_bits) - shuffling_bits = ( + shuffling_bit_size = ( NODE_ID_BITS - NETWORK_SHARD_PREFIX_BITS - NETWORK_SHARD_SHUFFLING_PREFIX_BITS @@ -1095,11 +1095,16 @@ network shard via the following: ```python def compute_subscribed_subnets(node_id: NodeID, epoch: Epoch) -> Sequence[SubnetID]: network_shard = compute_network_shard(node_id, epoch) - return [ShardId(network_shard + index) % ATTESTATION_SUBNET_COUNT for index in range(SUBNETS_PER_NODE)] + return [SubnetId(network_shard + index) % ATTESTATION_SUBNET_COUNT for index in range(SUBNETS_PER_NODE)] ``` *Note*: When preparing for a hard fork, a node must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. +*Note*: A node MUST subscribe to the defined `SUBNETS_PER_NODE` attestation +subnets for the required period of time, but may optionally subscribe to more. +If a node chooses to subscribe to extra subnets, they SHOULD update their +metadata and ENR fields accordingly. + ## Design decision rationale ### Transport @@ -1410,7 +1415,7 @@ due to not being fully synced to ensure that such (amplified) DOS attacks are no #### How are we going to discover peers in a gossipsub topic? -In Phase 0, peers for attestation subnets will be found using the `attnets` entry in the ENR. +In Phase 0, peers for attestation subnets will be found using the `shards` entry in the ENR. Although this method will be sufficient for early upgrade of the beacon chain, we aim to use the more appropriate discv5 topics for this and other similar tasks in the future. ENRs should ultimately not be used for this purpose. From 2bc33190df43f2b9f12d109517dfdf18306be73f Mon Sep 17 00:00:00 2001 From: Age Manning Date: Mon, 18 Mar 2024 15:06:43 +1100 Subject: [PATCH 10/12] Decouple the shard from subnet count --- specs/phase0/p2p-interface.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 1567db532a..2f7b4f524b 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -200,12 +200,13 @@ This section outlines configurations that are used in this spec. | `MAXIMUM_GOSSIP_CLOCK_DISPARITY` | `500` | The maximum **milliseconds** of clock disparity assumed between honest nodes. | | `MESSAGE_DOMAIN_INVALID_SNAPPY` | `DomainType('0x00000000')` | 4-byte domain for gossip message-id isolation of *invalid* snappy messages | | `MESSAGE_DOMAIN_VALID_SNAPPY` | `DomainType('0x01000000')` | 4-byte domain for gossip message-id isolation of *valid* snappy messages | -| `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | | `ATTESTATION_SUBNET_COUNT` | `2**6` (= 64) | The number of attestation subnets used in the gossipsub protocol. | | `NETWORK_SHARD_COUNT` | `2**6` (= 64) | The number of network shards. | | `NETWORK_SHARD_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a network shard | | `NETWORK_SHARD_PREFIX_BITS` | `int(ceillog2(NETWORK_SHARD_COUNT) + NETWORK_SHARD_EXTRA_BITS)` | | `NETWORK_SHARD_SHUFFLING_PREFIX_BITS` | `3` | The number of bits used to shuffle nodes to a new shard within `EPOCHS_PER_SHARD_SUBSCRIPTION` | +| `SHARDS_PER_NODE` | `1` | The number of network shards assigned to each node-id | +| `SUBNETS_PER_SHARD` | `2` | The number of long-lived subnets a beacon node should be subscribed to per assigned shard. | ### MetaData @@ -1095,15 +1096,12 @@ network shard via the following: ```python def compute_subscribed_subnets(node_id: NodeID, epoch: Epoch) -> Sequence[SubnetID]: network_shard = compute_network_shard(node_id, epoch) - return [SubnetId(network_shard + index) % ATTESTATION_SUBNET_COUNT for index in range(SUBNETS_PER_NODE)] + return [SubnetId(network_shard + index) % ATTESTATION_SUBNET_COUNT for index in range(SUBNETS_PER_SHARD)] ``` *Note*: When preparing for a hard fork, a node must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. -*Note*: A node MUST subscribe to the defined `SUBNETS_PER_NODE` attestation -subnets for the required period of time, but may optionally subscribe to more. -If a node chooses to subscribe to extra subnets, they SHOULD update their -metadata and ENR fields accordingly. +*Note*: A node MUST subscribe to the subnets defined via `compute_subscribed_subnets` function, but MAY subscribe to more. If a node subscribes to extra subnets they SHOULD update their metadata and ENR bitfields to reflect the extra long-lived subscriptions. ## Design decision rationale From 5571d7c389ba69b5e4754fa61988a0e103a4766c Mon Sep 17 00:00:00 2001 From: Age Manning Date: Tue, 19 Mar 2024 09:24:47 +1100 Subject: [PATCH 11/12] Fix some typos --- specs/phase0/p2p-interface.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 2f7b4f524b..6761c2dab8 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -1037,7 +1037,7 @@ The mapping that links a node-id to a network shard is: def compute_network_shard(node_id: NodeID, epoch: Epoch) -> ShardID: # The main prefix bits to determine a network shard shard_prefix = node_id >> (NODE_ID_BITS - NETWORK_SHARD_PREFIX_BITS) - # Used to extract the total prefix bytes (prefix + shuffling_bits) + # Used to extract the total prefix bits (prefix + shuffling_bits) shuffling_bit_size = ( NODE_ID_BITS - NETWORK_SHARD_PREFIX_BITS @@ -1075,7 +1075,8 @@ def compute_network_shard(node_id: NodeID, epoch: Epoch) -> ShardID: The `compute_network_shard` function is designed with the following desirable properties: -* It uses only the first set of bytes (defined by prefix_bytes_size) of the +* It uses only the first set of bits (defined by NETWORK_SHARD_PREFIX_BITS and + NETWORK_SHARD_SHUFFLING_PREFIX_BITS) of the node-id. This allows for efficient discovery searches, by allowing nodes to search for specific nodes of network shards based on the kademilia XOR metric. From 6ea416293254cec0659669891b395d7b52c46bef Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 10 Apr 2024 15:09:17 +1000 Subject: [PATCH 12/12] Decouple rotation, remove the shard concept --- configs/mainnet.yaml | 12 ++-- configs/minimal.yaml | 12 ++-- specs/phase0/p2p-interface.md | 126 +++++++++++++++++----------------- 3 files changed, 76 insertions(+), 74 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 2347eab88e..f851c0c2a9 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -117,7 +117,7 @@ GOSSIP_MAX_SIZE: 10485760 # `2**10` (= 1024) MAX_REQUEST_BLOCKS: 1024 # `2**8` (= 256) -EPOCHS_PER_SHARD_SUBSCRIPTION: 256 +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 # `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 # `10 * 2**20` (=10485760, 10 MiB) @@ -136,11 +136,11 @@ SUBNETS_PER_NODE: 2 # 2**8 (= 64) ATTESTATION_SUBNET_COUNT: 64 # The granularity of the network. -NETWORK_SHARD_COUNT: 64 -NETWORK_SHARD_EXTRA_BITS: 0 -# ceillog2(NETWORK_SHARD_COUNT) + NETWORK_SHARD_EXTRA_BITS -NETWORK_SHARD_PREFIX_BITS: 6 -NETWORK_SHARD_SHUFFLING_PREFIX_BITS: 3 +NETWORK_TOPIC_COUNT: 64 +NETWORK_TOPIC_EXTRA_BITS: 0 +# ceillog2(NETWORK_TOPIC_COUNT) + NETWORK_TOPIC_EXTRA_BITS +NETWORK_TOPIC_PREFIX_BITS: 6 +NETWORK_TOPIC_SHUFFLING_PREFIX_BITS: 3 # Deneb # `2**7` (=128) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index c61844788e..7170f3504d 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -118,7 +118,7 @@ GOSSIP_MAX_SIZE: 10485760 # `2**10` (= 1024) MAX_REQUEST_BLOCKS: 1024 # `2**8` (= 256) -EPOCHS_PER_SHARD_SUBSCRIPTION: 256 +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 # [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272 # `10 * 2**20` (=10485760, 10 MiB) @@ -136,11 +136,11 @@ MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 SUBNETS_PER_NODE: 2 # 2**8 (= 64) ATTESTATION_SUBNET_COUNT: 64 -NETWORK_SHARD_COUNT: 64 -NETWORK_SHARD_EXTRA_BITS: 0 -# ceillog2(NETWORK_SHARD_COUNT) + NETWORK_SHARD_EXTRA_BITS -NETWORK_SHARD_PREFIX_BITS: 6 -NETWORK_SHARD_SHUFFLING_PREFIX_BITS: 3 +NETWORK_TOPIC_COUNT: 64 +NETWORK_TOPIC_EXTRA_BITS: 0 +# ceillog2(NETWORK_TOPIC_COUNT) + NETWORK_TOPIC_EXTRA_BITS +NETWORK_TOPIC_PREFIX_BITS: 6 +NETWORK_TOPIC_SHUFFLING_PREFIX_BITS: 3 # Deneb # `2**7` (=128) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 6761c2dab8..3f3d0d7d2c 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -191,7 +191,7 @@ This section outlines configurations that are used in this spec. |---|---|---| | `GOSSIP_MAX_SIZE` | `10 * 2**20` (= 10485760, 10 MiB) | The maximum allowed size of uncompressed gossip messages. | | `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | -| `EPOCHS_PER_SHARD_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on a shard subscription (~27 hours) | +| `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on an attestation subscription (~27 hours) | | `MIN_EPOCHS_FOR_BLOCK_REQUESTS` | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) | The minimum epoch range over which a node must serve blocks | | `MAX_CHUNK_SIZE` | `10 * 2**20` (=10485760, 10 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | | `TTFB_TIMEOUT` | `5` | The maximum duration in **seconds** to wait for first byte of request response (time-to-first-byte). | @@ -201,12 +201,11 @@ This section outlines configurations that are used in this spec. | `MESSAGE_DOMAIN_INVALID_SNAPPY` | `DomainType('0x00000000')` | 4-byte domain for gossip message-id isolation of *invalid* snappy messages | | `MESSAGE_DOMAIN_VALID_SNAPPY` | `DomainType('0x01000000')` | 4-byte domain for gossip message-id isolation of *valid* snappy messages | | `ATTESTATION_SUBNET_COUNT` | `2**6` (= 64) | The number of attestation subnets used in the gossipsub protocol. | -| `NETWORK_SHARD_COUNT` | `2**6` (= 64) | The number of network shards. | -| `NETWORK_SHARD_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a network shard | -| `NETWORK_SHARD_PREFIX_BITS` | `int(ceillog2(NETWORK_SHARD_COUNT) + NETWORK_SHARD_EXTRA_BITS)` | -| `NETWORK_SHARD_SHUFFLING_PREFIX_BITS` | `3` | The number of bits used to shuffle nodes to a new shard within `EPOCHS_PER_SHARD_SUBSCRIPTION` | -| `SHARDS_PER_NODE` | `1` | The number of network shards assigned to each node-id | -| `SUBNETS_PER_SHARD` | `2` | The number of long-lived subnets a beacon node should be subscribed to per assigned shard. | +| `NETWORK_TOPIC_COUNT` | `ATTESTATION_SUBNET_COUNT` | The number of topics to generate for a node-id. Used in the gossipsub-protocol. | +| `NETWORK_TOPIC_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a gossipsub topic | +| `NETWORK_TOPIC_PREFIX_BITS` | `int(ceillog2(NETWORK_TOPIC_COUNT) + NETWORK_TOPIC_EXTRA_BITS)`. | +| `NETWORK_TOPIC_SHUFFLING_PREFIX_BITS` | `3` | The number of bits used to shuffle nodes to a new topic within `EPOCHS_PER_SUBNET_SUBSCRIPTION`. | +| `SUBNETS_PER_NODE` | `2` | The number of long-lived attestation subnets a beacon node should be subscribed to. | ### MetaData @@ -215,7 +214,7 @@ Clients MUST locally store the following `MetaData`: ``` ( seq_number: uint64 - shards: Bitvector[NETWORK_SHARD_COUNT] + attnets: Bitvector[ATTESTATION_SUBNET_COUNT] ) ``` @@ -223,7 +222,7 @@ Where - `seq_number` is a `uint64` starting at `0` used to version the node's metadata. If any other field in the local `MetaData` changes, the node MUST increment `seq_number` by 1. -- `shards` is a `Bitvector` representing the node's persistent attestation subnet subscriptions. +- `attnets` is a `Bitvector` representing the node's persistent attestation subnet subscriptions. *Note*: `MetaData.seq_number` is used for versioning of the node's metadata, is entirely independent of the ENR sequence number, @@ -959,16 +958,16 @@ Specifications of these parameters can be found in the [ENR Specification](http: ##### Attestation subnet bitfield -The ENR `shards` entry signifies the attestation subnet bitfield with the following form +The ENR `attnets` entry signifies the attestation subnet bitfield with the following form to more easily discover peers participating in particular attestation gossip subnets. | Key | Value | |:-------------|:-------------------------------------------------| -| `shards` | SSZ `Bitvector[NETWORK_SHARD_COUNT]` | +| `attnets` | SSZ `Bitvector[ATTESTATION_SUBNET_COUNT]` | -If a node's `MetaData.shards` has any non-zero bit, the ENR MUST include the `shards` entry with the same value as `MetaData.shards`. +If a node's `MetaData.attnets` has any non-zero bit, the ENR MUST include the `attnets` entry with the same value as `MetaData.attnets`. -If a node's `MetaData.shards` is composed of all zeros, the ENR MAY optionally include the `shards` entry or leave it out entirely. +If a node's `MetaData.attnets` is composed of all zeros, the ENR MAY optionally include the `attnets` entry or leave it out entirely. ##### `eth2` field @@ -1013,7 +1012,7 @@ Clients MAY connect to peers with the same `fork_digest` but a different `next_f Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`. -### Network Shards +### Topic Backbone In order for gossipsub to function, there must be a stable set of peers for each topic that remain subscribed for a long period of time (order 1 @@ -1023,81 +1022,84 @@ topics are transient for most nodes (i.e attestation subnets, DAS-related columns) it is necessary that we enforce each node on the network to facilitate the support of these topics by long-lived subscribing to them (and thereby validating and forwarding messages). -To this end we define the abstract concept of a "network shard". Each network -shard is mapped to one or many transient gossipsub topics that require a stable -set of subscribed peers. The primary advantage of this concept is that a node -need only to optimise their peer set to obtain a uniform set of peers on all -network shards, which will then guarantee a uniform set of peers on all transient -gossipsub topics (rather than trying to optimise for each individual set of -topics (i.e attestation_subnets, DAS-related columns). +To this end we define a mapping that relates a peer's node-id into one or many +gossipsub topics for a given epoch. Every node MUST subscribe to their associated +topic at any given epoch. -The mapping that links a node-id to a network shard is: +The mapping is based on the first few +`NETWORK_TOPIC_PREFIX_BITS` of a peer's node-id. Client implementors may wish +to keep a uniform distribution of peers with these prefix-bits in order to +maintain (on average) a uniform distribution of peers on the transient +gossipsub topics (i.e attestation subnets). + +The mapping that links a node-id to a topic id is: ```python -def compute_network_shard(node_id: NodeID, epoch: Epoch) -> ShardID: - # The main prefix bits to determine a network shard - shard_prefix = node_id >> (NODE_ID_BITS - NETWORK_SHARD_PREFIX_BITS) +def compute_network_topic(node_id: NodeID, epoch: Epoch, rotation_period: uint16) -> TopicID: + # NOTE: The rotation period is measured in epochs + # The main prefix bits to determine a network topic + topic_prefix = node_id >> (NODE_ID_BITS - NETWORK_TOPIC_PREFIX_BITS) # Used to extract the total prefix bits (prefix + shuffling_bits) shuffling_bit_size = ( NODE_ID_BITS - - NETWORK_SHARD_PREFIX_BITS - - NETWORK_SHARD_SHUFFLING_PREFIX_BITS + - NETWORK_TOPIC_PREFIX_BITS + - NETWORK_TOPIC_SHUFFLING_PREFIX_BITS ) - # The NETWORK_SHARD_SHUFFLING_PREFIX_BITS that trail shard_prefix. - # These are used to stagger the rotation of network shards so that all - # nodes do not rotate from shards all at once. - # The larger the NETWORK_SHARD_SHUFFLING_PREFIX_BITS the more granular the - # nodes will transition from one shard to another throughout a period. - shuffling_bits = (node_id >> shuffling_bit_size) % (1 << NETWORK_SHARD_SHUFFLING_PREFIX_BITS) + # The NETWORK_TOPIC_SHUFFLING_PREFIX_BITS that trail topic_prefix. + # These are used to stagger the rotation of network topics so that all + # nodes do not rotate from topics all at once. + # The larger the NETWORK_TOPIC_SHUFFLING_PREFIX_BITS the more granular the + # nodes will transition from one topic to another throughout a period. + shuffling_bits = (node_id >> shuffling_bit_size) % (1 << NETWORK_TOPIC_SHUFFLING_PREFIX_BITS) # Calculates a multiplier that scales the shuffling prefix (assumed smaller # than the rotation period) to be uniform throughout the entire rotation # period. Can also be calculated as: - # EPOCHS_PER_SHARD_SUBSCRIPTION // 1 >> NETWORK_SHARD_SHUFFLING_PREFIX_BITS - shuffling_multiplier = EPOCHS_PER_SHARD_SUBSCRIPTION >> NETWORK_SHARD_SHUFFLING_PREFIX_BITS - # The epoch at which this node will rotate to a new shard + # rotation_period // 1 >> NETWORK_TOPIC_SHUFFLING_PREFIX_BITS + shuffling_multiplier = rotation_period >> NETWORK_TOPIC_SHUFFLING_PREFIX_BITS + # The epoch at which this node will rotate to a new topic # This is distributed uniformly throughout the rotation period with a - # granularity based on the size of NETWORK_SHARD_SHUFFLING_PREFIX_BITS + # granularity based on the size of NETWORK_TOPIC_SHUFFLING_PREFIX_BITS epoch_transition = ( - (shard_prefix + (shuffling_bits * shuffling_multiplier)) % EPOCHS_PER_SHARD_SUBSCRIPTION + (topic_prefix + (shuffling_bits * shuffling_multiplier)) % rotation_period ) - # A seed which changes every rotation period (EPOCHS_PER_SHARD_SUBSCRIPTION) + # A seed which changes every rotation period # This enforces the rotation period and is staggered for each prefix so - # that nodes do not rotate from shards all at once - permutation_seed = hash(uint_to_bytes(uint64((epoch + epoch_transition) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) - # The resulting value that ultimately defines the network shard. + # that nodes do not rotate from topics all at once + permutation_seed = hash(uint_to_bytes(uint64((epoch + epoch_transition) // rotation_period))) + # The resulting value that ultimately defines the network topic. permutated_prefix = compute_shuffled_index( - shard_prefix, - 1 << NETWORK_SHARD_PREFIX_BITS, + topic_prefix, + 1 << NETWORK_TOPIC_PREFIX_BITS, permutation_seed, ) - return ShardID(permutated_prefix % NETWORK_SHARD_COUNT) + return TopicID(permutated_prefix % NETWORK_TOPIC_COUNT) ``` -The `compute_network_shard` function is designed with the following +The `compute_network_topic` function is designed with the following desirable properties: -* It uses only the first set of bits (defined by NETWORK_SHARD_PREFIX_BITS and - NETWORK_SHARD_SHUFFLING_PREFIX_BITS) of the - node-id. This allows for efficient discovery searches, by allowing nodes to - search for specific nodes of network shards based on the kademilia XOR +* It uses only the first set of bits (defined by NETWORK_TOPIC_PREFIX_BITS and + NETWORK_TOPIC_SHUFFLING_PREFIX_BITS) of the + node-id. This allows for efficient discovery searches, by allowing clients to + search for specific nodes linked to a gossipsub topic based on the kademilia XOR metric. -* Nodes will maintain a shard for EPOCHS_PER_SHARD_SUBSCRIPTION before - rotating to a new shard. -* The rotation is staggered uniformly throughout the rotation period per shard. -* No individual shard will suddenly rotate to another, rather a subset of nodes - per shard transition gradually throughout the rotation period. +* Nodes will maintain a topic for the `rotation_period` before + rotating to a new topic. +* The rotation is staggered uniformly throughout the rotation period per topic. +* No individual topic will suddenly rotate to another, rather a subset of nodes + per topic transition gradually throughout the rotation period. * The function is feasibly reversible. Prefixes can be calculated for desired - network shards for any given epoch, allowing nodes to search for these + network topics for any given epoch, allowing nodes to search for these prefixes. ### Attestation Subnets -The backbone structure for attestation subnets can be calculated from their -network shard via the following: +The backbone structure for attestation subnets can be calculated from the +network topic function above via the following: ```python -def compute_subscribed_subnets(node_id: NodeID, epoch: Epoch) -> Sequence[SubnetID]: - network_shard = compute_network_shard(node_id, epoch) - return [SubnetId(network_shard + index) % ATTESTATION_SUBNET_COUNT for index in range(SUBNETS_PER_SHARD)] +def compute_subscribed_topic(node_id: NodeID, epoch: Epoch, rotation_period: EPOCHS_PER_SUBNET_SUBSCRIPTION) -> Sequence[SubnetID]: + network_topic_id = compute_network_topic(node_id, epoch, rotation_period) + return [SubnetId(network_topic_id + index) % ATTESTATION_SUBNET_COUNT for index in range(SUBNETS_PER_NODE)] ``` *Note*: When preparing for a hard fork, a node must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. @@ -1414,7 +1416,7 @@ due to not being fully synced to ensure that such (amplified) DOS attacks are no #### How are we going to discover peers in a gossipsub topic? -In Phase 0, peers for attestation subnets will be found using the `shards` entry in the ENR. +In Phase 0, peers for attestation subnets will be found using the `attnets` entry in the ENR. Although this method will be sufficient for early upgrade of the beacon chain, we aim to use the more appropriate discv5 topics for this and other similar tasks in the future. ENRs should ultimately not be used for this purpose.