Initial impl of state network bridge 1902 (#1948)

Makefile

@@ -68,6 +68,7 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
 FLUFFY_TOOLS := \
 	portal_bridge \
 	beacon_lc_bridge \
+	state_bridge \
 	eth_data_exporter \
 	content_verifier \
 	blockwalk \
@@ -76,6 +77,7 @@ FLUFFY_TOOLS := \
 FLUFFY_TOOLS_DIRS := \
 	fluffy/tools/beacon_lc_bridge \
 	fluffy/tools/portal_bridge \
+	fluffy/tools/state_bridge \
 	fluffy/tools
 # comma-separated values for the "clean" target
 FLUFFY_TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(FLUFFY_TOOLS))

fluffy/network/state/state_content.nim

@@ -1,5 +1,5 @@
 # Fluffy
-# Copyright (c) 2021-2023 Status Research & Development GmbH
+# Copyright (c) 2023-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -11,12 +11,29 @@
 {.push raises: [].}
 
 import
-  nimcrypto/[hash, sha2, keccak], stew/[objects, results, endians2], stint,
+  nimcrypto/[hash, sha2, keccak], stew/[objects, results], stint,
   ssz_serialization,
   ../../common/common_types
 
 export ssz_serialization, common_types, hash, results
 
+type JsonAccount* = object
+  nonce*: int
+  balance*: string
+  storage_hash*: string
+  code_hash*: string
+
+type JsonProof* = object
+  address*: string
+  state*: JsonAccount
+  proof*: seq[string]
+
+type JsonProofVector* = object
+  `block`*: int
+  block_hash*: string
+  state_root*: string
+  proofs*: seq[JsonProof]
+
 type
   NodeHash* = MDigest[32 * 8] # keccak256
   CodeHash* = MDigest[32 * 8] # keccak256
@@ -61,6 +78,9 @@ type
     address*: Address
     codeHash*: CodeHash
 
+  WitnessNode* = ByteList
+  AccountTrieProof* = List[WitnessNode, 32]
+
   ContentKey* = object
     case contentType*: ContentType
     of unused:

fluffy/network/state/state_network.nim

@@ -1,12 +1,15 @@
 # Fluffy
-# Copyright (c) 2021-2023 Status Research & Development GmbH
+# Copyright (c) 2021-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
+  std/[sequtils, sugar],
   stew/results, chronos, chronicles,
+  eth/[rlp, common],
+  eth/trie/hexary_proof_verification,
   eth/p2p/discoveryv5/[protocol, enr],
   ../../database/content_db,
   ../wire/[portal_protocol, portal_stream, portal_protocol_config],
@@ -59,8 +62,59 @@ proc getContent*(n: StateNetwork, key: ContentKey):
   # domain types.
   return Opt.some(contentResult.content)
 
-proc validateContent(content: openArray[byte], contentKey: ByteList): bool =
-  true
+proc validateContent(
+    n: StateNetwork,
+    contentKey: ByteList,
+    contentValue: seq[byte]): Future[bool] {.async.} =
+  let key = contentKey.decode().valueOr:
+    return false
+
+  case key.contentType:
+    of unused:
+      warn "Received content with unused content type"
+      false
+    of accountTrieNode:
+      true
+    of contractStorageTrieNode:
+      true
+    of accountTrieProof:
+      let decodedProof = decodeSsz(contentValue, AccountTrieProof).valueOr:
+        warn "Received invalid account trie proof", error
+        return false
+      let
+        proof = decodedProof.asSeq().map((p: ByteList) => p.toSeq())
+        trieKey = keccakHash(key.accountTrieProofKey.address).data.toSeq()
+        value = proof[^1].decode(seq[seq[byte]])[^1]
+        stateRoot = MDigest[256](data: key.accountTrieProofKey.stateRoot)
+        verificationResult = verifyMptProof(proof, stateRoot, trieKey, value)
+      case verificationResult.kind:
+        of ValidProof:
+          true
+        else:
+          warn "Received invalid account trie proof"
+          false
+    of contractStorageTrieProof:
+      true
+    of contractBytecode:
+      true
+
+proc validateContent(
+    n: StateNetwork,
+    contentKeys: ContentKeysList,
+    contentValues: seq[seq[byte]]): Future[bool] {.async.} =
+  for i, contentValue in contentValues:
+    let contentKey = contentKeys[i]
+    if await n.validateContent(contentKey, contentValue):
+      let contentId = n.portalProtocol.toContentId(contentKey).valueOr:
+        error "Received offered content with invalid content key", contentKey
+        return false
+
+      n.portalProtocol.storeContent(contentKey, contentId, contentValue)
+
+      info "Received offered content validated successfully", contentKey
+    else:
+      error "Received offered content failed validation", contentKey
+      return false
 
 proc new*(
     T: type StateNetwork,
@@ -91,8 +145,11 @@ proc new*(
 proc processContentLoop(n: StateNetwork) {.async.} =
   try:
     while true:
-      # Just dropping state date for now
-      discard await n.contentQueue.popFirst()
+      let (maybeContentId, contentKeys, contentValues) = await n.contentQueue.popFirst()
+      if await n.validateContent(contentKeys, contentValues):
+        asyncSpawn n.portalProtocol.neighborhoodGossipDiscardPeers(
+          maybeContentId, contentKeys, contentValues
+        )
   except CancelledError:
     trace "processContentLoop canceled"
 

fluffy/tests/all_fluffy_tests.nim

@@ -9,8 +9,9 @@
 
 import
   ./test_portal_wire_protocol,
-  ./test_state_distance,
-  ./test_state_network,
+  ./state_network_tests/test_state_distance,
+  ./state_network_tests/test_state_network,
+  ./state_network_tests/test_state_content,
   ./test_state_proof_verification,
   ./test_accumulator,
   ./test_history_network,

fluffy/tests/state_network_tests/test_state_content.nim

@@ -0,0 +1,44 @@
+# Fluffy
+# Copyright (c) 2023-2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+import
+  std/[os, json, sequtils],
+  testutils/unittests,
+  stew/[byteutils, io2],
+  eth/keys,
+  ../../network/state/state_content
+
+const testVectorDir =
+  "./vendor/portal-spec-tests/tests/mainnet/state/"
+
+procSuite "State Content":
+  let rng = newRng()
+
+  test "Encode/decode accountTrieProof":
+    let file = testVectorDir & "/proofs.full.block.0.json"
+    let content = readAllFile(file).valueOr:
+      quit(1)
+
+    let decoded =
+      try:
+        Json.decode(content, state_content.JsonProofVector)
+      except SerializationError:
+        quit(1)
+
+    let proof = decoded.proofs[0].proof.map(hexToSeqByte)
+
+    var accountTrieProof = AccountTrieProof(@[])
+    for witness in proof:
+      let witnessNode = ByteList(witness)
+      discard accountTrieProof.add(witnessNode)
+
+    let
+      encodedProof = SSZ.encode(accountTrieProof)
+      decodedProof = decodeSsz(encodedProof, AccountTrieProof).get()
+
+    check decodedProof == accountTrieProof
+

fluffy/tests/test_state_distance.nim → fluffy/tests/state_network_tests/test_state_distance.nim

@@ -10,7 +10,7 @@
 import
   std/sequtils,
   stint, unittest2,
-  ../network/state/state_distance
+  ../../network/state/state_distance
 
 suite "State network custom distance function":
   test "Calculate distance according to spec":

fluffy/tests/test_state_network.nim → fluffy/tests/state_network_tests/test_state_network.nim

@@ -6,16 +6,23 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  std/os,
+  std/[os, json, sequtils, strutils, sugar],
+  stew/[byteutils, io2],
+  nimcrypto/hash,
   testutils/unittests, chronos,
-  eth/[common/eth_hash, keys],
+  eth/trie/hexary_proof_verification,
+  eth/keys,
+  eth/common/[eth_types, eth_hash],
   eth/p2p/discoveryv5/protocol as discv5_protocol, eth/p2p/discoveryv5/routing_table,
-  ../../nimbus/[config, db/core_db, db/state_db],
-  ../../nimbus/common/[chain_config, genesis],
-  ../network/wire/[portal_protocol, portal_stream],
-  ../network/state/[state_content, state_network],
-  ../database/content_db,
-  ./test_helpers
+  ../../../nimbus/[config, db/core_db, db/state_db],
+  ../../../nimbus/common/[chain_config, genesis],
+  ../../network/wire/[portal_protocol, portal_stream],
+  ../../network/state/[state_content, state_network],
+  ../../database/content_db,
+  .././test_helpers
+
+const testVectorDir =
+  "./vendor/portal-spec-tests/tests/mainnet/state/"
 
 proc genesisToTrie(filePath: string): CoreDbMptRef =
   # TODO: Doing our best here with API that exists, to be improved.
@@ -30,8 +37,84 @@ proc genesisToTrie(filePath: string): CoreDbMptRef =
 
   sdb.getTrie
 
-procSuite "State Content Network":
+procSuite "State Network":
   let rng = newRng()
+
+  test "Test account state proof":
+    let file = testVectorDir & "/proofs.full.block.0.json"
+    let content = readAllFile(file).valueOr:
+      quit(1)
+
+    let decoded =
+      try:
+        Json.decode(content, state_content.JsonProofVector)
+      except SerializationError:
+        quit(1)
+    let
+      proof = decoded.proofs[0].proof.map(hexToSeqByte)
+      stateRoot = MDigest[256].fromHex(decoded.state_root)
+      address = hexToByteArray[20](decoded.proofs[0].address)
+      key = keccakHash(address).data.toSeq()
+      value = proof[^1].decode(seq[seq[byte]])[^1]
+      proofResult = verifyMptProof(proof, stateRoot, key, value)
+    check proofResult.kind == ValidProof
+
+  asyncTest "Decode and use proofs":
+    let file = testVectorDir & "/proofs.full.block.0.json"
+    let content = readAllFile(file).valueOr:
+      quit(1)
+
+    let decoded =
+      try:
+        Json.decode(content, state_content.JsonProofVector)
+      except SerializationError:
+        quit(1)
+
+    let
+      node1 = initDiscoveryNode(
+        rng, PrivateKey.random(rng[]), localAddress(20302))
+      sm1 = StreamManager.new(node1)
+      node2 = initDiscoveryNode(
+        rng, PrivateKey.random(rng[]), localAddress(20303))
+      sm2 = StreamManager.new(node2)
+
+      proto1 = StateNetwork.new(node1, ContentDB.new("", uint32.high, inMemory = true), sm1)
+      proto2 = StateNetwork.new(node2, ContentDB.new("", uint32.high, inMemory = true), sm2)
+
+      state_root = hexToByteArray[sizeof(state_content.AccountTrieProofKey.stateRoot)](decoded.state_root)
+
+    check proto2.portalProtocol.addNode(node1.localNode) == Added
+
+
+    for proof in decoded.proofs:
+      let
+        address = hexToByteArray[sizeof(state_content.Address)](proof.address)
+        key = AccountTrieProofKey(
+          address: address,
+          stateRoot: state_root)
+        contentKey = ContentKey(
+          contentType: state_content.ContentType.accountTrieProof,
+          accountTrieProofKey: key)
+
+      var accountTrieProof = AccountTrieProof(@[])
+      for witness in proof.proof:
+        let witnessNode = ByteList(hexToSeqByte(witness))
+        discard accountTrieProof.add(witnessNode)
+
+      let encodedValue = SSZ.encode(accountTrieProof)
+
+      discard proto1.contentDB.put(contentKey.toContentId(), encodedValue, proto1.portalProtocol.localNode.id)
+
+      let foundContent = await proto2.getContent(contentKey)
+
+      check foundContent.isSome()
+
+      check decodeSsz(foundContent.get(), AccountTrieProof).isOk()
+
+    await node1.closeWait()
+    await node2.closeWait()
+
+
   asyncTest "Test Share Full State":
     let
       trie = genesisToTrie("fluffy" / "tests" / "custom_genesis" / "chainid7.json")

fluffy/tools/state_bridge/nim.cfg

@@ -0,0 +1 @@
+-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"