From d721103f055fba44ab862aeec6437c064516eaf2 Mon Sep 17 00:00:00 2001 From: Bytecoin Developer Date: Wed, 12 Dec 2018 21:19:14 +0300 Subject: [PATCH] Add the batch of release v3.4.0-beta-20181212 commits --- CMakeLists.txt | 112 +- ReleaseNotes.md | 47 +- src/Core/Archive.cpp | 63 +- src/Core/Archive.hpp | 15 +- src/Core/BlockChain.cpp | 765 ++++---- src/Core/BlockChain.hpp | 111 +- src/Core/BlockChainFileFormat.cpp | 211 ++- src/Core/BlockChainFileFormat.hpp | 32 +- src/Core/BlockChainState.cpp | 1416 ++++++++------ src/Core/BlockChainState.hpp | 150 +- src/Core/Config.cpp | 65 +- src/Core/Config.hpp | 80 +- src/Core/CryptoNoteTools.cpp | 87 +- src/Core/CryptoNoteTools.hpp | 25 +- src/Core/Currency.cpp | 555 +++--- src/Core/Currency.hpp | 131 +- src/Core/Difficulty.cpp | 4 +- src/Core/Difficulty.hpp | 4 +- src/Core/Multicore.cpp | 264 ++- src/Core/Multicore.hpp | 77 +- src/Core/Node.cpp | 695 +++---- src/Core/Node.hpp | 393 ++-- src/Core/NodeDownloader.cpp | 485 ----- src/Core/NodeDownloaderV3.cpp | 619 ------- src/Core/NodeLegacyAPI.cpp | 118 +- src/Core/Node_P2PProtocolBytecoin.cpp | 816 ++++++-- src/Core/Node_P2PProtocolBytecoinNew.cpp | 159 -- src/Core/TransactionBuilder.cpp | 570 +++--- src/Core/TransactionBuilder.hpp | 87 +- src/Core/TransactionExtra.cpp | 94 +- src/Core/TransactionExtra.hpp | 34 +- src/Core/Wallet.cpp | 1196 ++++++++++-- src/Core/Wallet.hpp | 250 ++- src/Core/WalletNode.cpp | 486 +++-- src/Core/WalletNode.hpp | 70 +- src/Core/WalletSerializationV1.cpp | 34 +- src/Core/WalletSerializationV1.hpp | 18 +- src/Core/WalletState.cpp | 555 +++--- src/Core/WalletState.hpp | 71 +- src/Core/WalletStateBasic.cpp | 366 ++-- src/Core/WalletStateBasic.hpp | 62 +- src/Core/WalletSync.cpp | 182 +- src/Core/WalletSync.hpp | 16 +- src/CryptoNote.cpp | 660 +++++-- src/CryptoNote.hpp | 224 ++- src/CryptoNoteConfig.hpp | 138 +- src/common/BIPs.cpp | 287 +++ src/common/BIPs.hpp | 36 + src/common/Base58.cpp | 387 ++-- src/common/Base58.hpp | 15 +- src/common/Base64.cpp | 32 +- src/common/Base64.hpp | 6 +- src/common/BinaryArray.cpp | 155 +- src/common/BinaryArray.hpp | 219 +-- src/common/CRC32.cpp | 65 + src/common/CRC32.hpp | 28 + src/common/CRC32_generate.py | 29 + src/common/CommandLine.cpp | 14 +- src/common/CommandLine.hpp | 2 +- src/common/ConsoleTools.cpp | 8 +- src/common/ConsoleTools.hpp | 8 +- src/common/Int128.hpp | 25 +- src/common/Ipv4Address.cpp | 19 +- src/common/Ipv4Address.hpp | 38 +- src/common/JsonValue.cpp | 8 +- src/common/JsonValue.hpp | 6 +- src/common/Math.hpp | 4 +- src/common/MemoryStreams.hpp | 14 +- src/common/Nocopy.hpp | 2 +- src/common/ScopeExit.hpp | 6 +- src/common/Streams.cpp | 78 +- src/common/Streams.hpp | 61 +- src/common/StringTools.cpp | 16 +- src/common/StringTools.hpp | 61 +- src/common/StringView.cpp | 5 +- src/common/StringView.hpp | 10 +- src/common/Varint.cpp | 9 +- src/common/Varint.hpp | 35 +- src/common/Words.cpp | 751 ++++++++ src/common/Words.hpp | 17 + src/common/Words_generate.py | 69 + src/common/exception.cpp | 6 +- src/common/exception.hpp | 2 +- src/common/string.hpp | 4 +- src/common/words_english.txt | 2048 +++++++++++++++++++++ src/crypto/bernstein/c_types.h | 12 +- src/crypto/bernstein/chacha8.c | 6 +- src/crypto/bernstein/chacha8.h | 5 +- src/crypto/bernstein/crypto-ops-data.h | 4 +- src/crypto/bernstein/crypto-ops.c | 73 +- src/crypto/bernstein/crypto-ops.h | 39 +- src/crypto/chacha.cpp | 31 + src/crypto/chacha.hpp | 27 + src/crypto/chacha8.hpp | 41 - src/crypto/crypto-util.c | 11 - src/crypto/crypto-util.h | 3 - src/crypto/crypto.cpp | 1072 ++++++----- src/crypto/crypto.hpp | 113 +- src/crypto/generic-ops.hpp | 24 +- src/crypto/groestl/Groestl-opt.c | 1106 +++++------ src/crypto/groestl/Groestl-opt.h | 130 +- src/crypto/groestl/brg_endian.h | 266 +-- src/crypto/groestl/brg_types.h | 462 ++--- src/crypto/groestl/tables.h | 78 +- src/crypto/hash-blake.c | 5 +- src/crypto/hash-groestl.c | 5 +- src/crypto/hash-jh.c | 4 +- src/crypto/hash-keccak.c | 24 +- src/crypto/hash-skein.c | 4 +- src/crypto/hash.cpp | 80 +- src/crypto/hash.h | 31 +- src/crypto/hash.hpp | 27 +- src/crypto/initializer.h | 6 +- src/crypto/random.c | 25 +- src/crypto/random.h | 8 +- src/crypto/slow-hash_stdc.c | 17 +- src/crypto/slow-hash_x86.c | 14 +- src/crypto/slow-hash_x86.inl | 4 +- src/crypto/tree-hash.c | 44 +- src/crypto/tree-hash.h | 13 +- src/crypto/types.cpp | 26 + src/crypto/types.hpp | 83 +- src/http/Agent.cpp | 31 +- src/http/Agent.hpp | 17 +- src/http/BinaryRpc.hpp | 13 +- src/http/Client.cpp | 119 ++ src/http/Client.hpp | 51 + src/http/JsonRpc.cpp | 39 +- src/http/JsonRpc.hpp | 39 +- src/http/RequestParser.cpp | 4 +- src/http/RequestParser.hpp | 9 +- src/http/ResponseParser.cpp | 4 +- src/http/ResponseParser.hpp | 9 +- src/http/Server.cpp | 113 +- src/http/Server.hpp | 66 +- src/http/types.cpp | 17 +- src/http/types.hpp | 25 +- src/logging/CommonLogger.cpp | 4 +- src/logging/CommonLogger.hpp | 2 +- src/logging/ConsoleLogger.cpp | 3 +- src/logging/ConsoleLogger.hpp | 2 +- src/logging/FileLogger.cpp | 14 +- src/logging/FileLogger.hpp | 2 +- src/logging/ILogger.cpp | 2 +- src/logging/ILogger.hpp | 2 +- src/logging/LoggerGroup.cpp | 2 +- src/logging/LoggerGroup.hpp | 2 +- src/logging/LoggerManager.cpp | 100 +- src/logging/LoggerManager.hpp | 11 +- src/logging/LoggerMessage.cpp | 2 +- src/logging/LoggerMessage.hpp | 2 +- src/main_bytecoind.cpp | 39 +- src/main_tests.cpp | 50 +- src/main_walletd.cpp | 301 +-- src/p2p/CryptoNoteProtocolDefinitions.hpp | 93 - src/p2p/LevinProtocol.cpp | 92 +- src/p2p/LevinProtocol.hpp | 48 +- src/p2p/P2P.cpp | 98 +- src/p2p/P2P.hpp | 28 +- src/p2p/P2PClientBasic.cpp | 272 --- src/p2p/P2PClientBasic.hpp | 88 - src/p2p/P2PClientNew.cpp | 295 --- src/p2p/P2PClientNew.hpp | 82 - src/p2p/P2PProtocolBasic.cpp | 316 ++++ src/p2p/P2PProtocolBasic.hpp | 92 + src/p2p/P2pProtocolDefinitions.hpp | 306 +-- src/p2p/P2pProtocolNew.cpp | 144 -- src/p2p/P2pProtocolNew.hpp | 252 --- src/p2p/P2pProtocolTypes.hpp | 123 +- src/p2p/P2pSeria.cpp | 255 +-- src/p2p/PeerDB.cpp | 110 +- src/p2p/PeerDB.hpp | 8 +- src/platform/DB.hpp | 2 +- src/platform/DBlmdb.cpp | 12 +- src/platform/DBlmdb.hpp | 7 +- src/platform/DBsqlite3.cpp | 272 +-- src/platform/DBsqlite3.hpp | 49 +- src/platform/ExclusiveLock.cpp | 1 - src/platform/ExclusiveLock.hpp | 2 +- src/platform/Files.cpp | 34 +- src/platform/Files.hpp | 17 +- src/platform/Network.cpp | 57 +- src/platform/Network.hpp | 37 +- src/platform/PathTools.cpp | 35 +- src/platform/PathTools.hpp | 3 +- src/platform/PathTools.mm | 9 +- src/platform/PreventSleep.cpp | 2 +- src/platform/PreventSleep.hpp | 2 +- src/platform/Time.cpp | 3 - src/platform/Time.hpp | 2 +- src/rpc_api.cpp | 237 ++- src/rpc_api.hpp | 464 ++--- src/seria/BinaryInputStream.cpp | 68 +- src/seria/BinaryInputStream.hpp | 47 +- src/seria/BinaryOutputStream.cpp | 50 +- src/seria/BinaryOutputStream.hpp | 47 +- src/seria/ISeria.hpp | 111 +- src/seria/JsonInputStream.cpp | 50 +- src/seria/JsonInputStream.hpp | 49 +- src/seria/JsonOutputStream.cpp | 31 +- src/seria/JsonOutputStream.hpp | 98 +- src/seria/KVBinaryCommon.hpp | 4 +- src/seria/KVBinaryInputStream.cpp | 74 +- src/seria/KVBinaryInputStream.hpp | 8 +- src/seria/KVBinaryOutputStream.cpp | 70 +- src/seria/KVBinaryOutputStream.hpp | 65 +- src/version.hpp | 8 +- 207 files changed, 15634 insertions(+), 11311 deletions(-) delete mode 100644 src/Core/NodeDownloader.cpp delete mode 100644 src/Core/NodeDownloaderV3.cpp delete mode 100644 src/Core/Node_P2PProtocolBytecoinNew.cpp create mode 100644 src/common/BIPs.cpp create mode 100644 src/common/BIPs.hpp create mode 100644 src/common/CRC32.cpp create mode 100644 src/common/CRC32.hpp create mode 100644 src/common/CRC32_generate.py create mode 100644 src/common/Words.cpp create mode 100644 src/common/Words.hpp create mode 100644 src/common/Words_generate.py create mode 100644 src/common/words_english.txt create mode 100644 src/crypto/chacha.cpp create mode 100644 src/crypto/chacha.hpp delete mode 100644 src/crypto/chacha8.hpp create mode 100644 src/crypto/types.cpp create mode 100644 src/http/Client.cpp create mode 100644 src/http/Client.hpp delete mode 100644 src/p2p/CryptoNoteProtocolDefinitions.hpp delete mode 100644 src/p2p/P2PClientBasic.cpp delete mode 100644 src/p2p/P2PClientBasic.hpp delete mode 100644 src/p2p/P2PClientNew.cpp delete mode 100644 src/p2p/P2PClientNew.hpp create mode 100644 src/p2p/P2PProtocolBasic.cpp create mode 100644 src/p2p/P2PProtocolBasic.hpp delete mode 100644 src/p2p/P2pProtocolNew.cpp delete mode 100644 src/p2p/P2pProtocolNew.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c613e272..ef9728cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,33 +5,33 @@ cmake_minimum_required(VERSION 3.0) project(bytecoin) set(CMAKE_CXX_STANDARD 14) set(CMAKE_C_STANDARD 11) -# message(STATUS "Bytecoind profile: According to cmake, sizeof(void *) == " ${CMAKE_SIZEOF_VOID_P}) +set(CRYPTONOTE_NAME bytecoin) +add_definitions(-DCRYPTONOTE_NAME=\"${CRYPTONOTE_NAME}\") option(USE_INSTRUMENTATION "For testing - builds with address sanitizer instrument" OFF) option(WITH_THREAD_SANITIZER "For testing - builds with thread sanitizer instrument, USE_INSTRUMENTATION must be also set" OFF) option(BETTER_DEBUG "Disables optimizations. We do not use standard debug/realease configurations because they change too much" OFF) -option(USE_SSL "Builds with support of https between walletd and bytecoind" ON) +get_filename_component(PARENT_DIR ${CMAKE_SOURCE_DIR} DIRECTORY) + if(CMAKE_SIZEOF_VOID_P EQUAL 8) option(USE_SQLITE "Builds with SQLite instead of LMDB. 4x slower, but works on 32-bit and mobile platforms" OFF) - set(OPENSSL_ROOT ../openssl) + set(OPENSSL_ROOT ${PARENT_DIR}/openssl) else() option(USE_SQLITE "Builds with sqlite instead of LMDB. 4x slower, but works on 32-bit and mobile platforms" ON) - set(OPENSSL_ROOT ../openssl32) + set(OPENSSL_ROOT ${PARENT_DIR}/openssl32) endif() if(WIN32) -# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") -# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") add_definitions(-D_SCL_SECURE_NO_WARNINGS=1 -D_CRT_SECURE_NO_WARNINGS=1 -D_WIN32_WINNT=0x0501) else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maes -g -Wall -Wextra -Werror=return-type -Wno-unused-parameter") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes -g -Wall -Wextra -Werror=return-type -Wno-unused-parameter") if(BETTER_DEBUG) + message(STATUS "Using better debug: " ${BETTER_DEBUG}) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2") endif() - message(STATUS "Better debug: " ${BETTER_DEBUG}) message(STATUS "Instrumentation usage: " ${USE_INSTRUMENTATION}) message(STATUS "Thread sanitizer usage: " ${WITH_THREAD_SANITIZER}) if(USE_INSTRUMENTATION) @@ -44,30 +44,34 @@ else() endif() endif() endif() +include_directories(${PARENT_DIR}/sqlite) +if(NOT EXISTS "${PARENT_DIR}/sqlite/sqlite3.c") + message(FATAL_ERROR "${PARENT_DIR}/sqlite/sqlite3.c not found. Make sure amalgamated sqlite is in ../sqlite/") +endif() +set(SRC_WARNINGS_DB ../sqlite/sqlite3.c) +set(SRC_DB src/platform/DBsqlite3.cpp src/platform/DBsqlite3.hpp) if(USE_SQLITE) # Requires dl on Linux, we add it unconditionally for simplicity. - message(STATUS "Database selected: SQLite 3. Make sure it is put into ../sqlite/") - include_directories(../sqlite) - set(SRC_WARNINGS_DB ../sqlite/sqlite3.c) - set(SRC_DB src/platform/DBsqlite3.cpp src/platform/DBsqlite3.hpp) + message(STATUS "Database selected: SQLite 3") add_definitions(-Dplatform_USE_SQLITE=1) else() - message(STATUS "Database selected: LMDB. Make sure it is put into ../lmdb/") + message(STATUS "Database selected: LMDB") include_directories(../lmdb/libraries/liblmdb) - set(SRC_WARNINGS_DB ../lmdb/libraries/liblmdb/mdb.c ../lmdb/libraries/liblmdb/midl.c) - set(SRC_DB src/platform/DBlmdb.cpp src/platform/DBlmdb.hpp) -endif() -if(USE_SSL) - message(STATUS "SSL usage: ON. Make sure openssl headers are in " ${OPENSSL_ROOT} "/include and static libs are in " ${OPENSSL_ROOT} "/") - include_directories(${OPENSSL_ROOT}/include) - link_directories(${OPENSSL_ROOT}) # Must be placed before add_executable, add_library. - set(LINK_OPENSSL ssl crypto) - add_definitions(-Dplatform_USE_SSL=1) -else() - message(STATUS "SSL usage: OFF") + if(NOT EXISTS "${PARENT_DIR}/lmdb/libraries/liblmdb/mdb.c") + message(FATAL_ERROR "${PARENT_DIR}/lmdb/libraries/liblmdb/mdb.c not found. Make sure lmdb is cloned into ../lmdb/") + endif() + set(SRC_WARNINGS_DB ${SRC_WARNINGS_DB} ../lmdb/libraries/liblmdb/mdb.c ../lmdb/libraries/liblmdb/midl.c) + set(SRC_DB ${SRC_DB} src/platform/DBlmdb.cpp src/platform/DBlmdb.hpp) endif() + +message(STATUS "Make sure OpenSSL headers are in " ${OPENSSL_ROOT} "/include and static libs are in " ${OPENSSL_ROOT} "/") +include_directories(${OPENSSL_ROOT}/include) +link_directories(${OPENSSL_ROOT}) # Must be placed before add_executable, add_library. +set(LINK_OPENSSL ssl crypto) +add_definitions(-Dplatform_USE_SSL=1) + file(GLOB SRC_CRYPTO - src/crypto/*.cpp src/crypto/*.hpp src/crypto/*.h + src/crypto/*.cpp src/crypto/*.hpp src/crypto/*.c src/crypto/*.h src/crypto/bernstein/*.h src/crypto/bernstein/*.c src/crypto/blake/*.h src/crypto/blake/*.c @@ -78,7 +82,6 @@ file(GLOB SRC_CRYPTO src/crypto/skein/*.h src/crypto/skein/*.c ) file(GLOB SRC_COMMON src/common/*.cpp src/common/*.hpp) -file(GLOB SRC_SERIALIZATION src/Serialization/*.cpp src/Serialization/*.hpp) file(GLOB SRC_SERIA src/seria/*.cpp src/seria/*.hpp) file(GLOB SRC_LOGGING src/logging/*.cpp src/logging/*.hpp) file(GLOB SRC_P2P src/p2p/*.cpp src/p2p/*.hpp) @@ -93,29 +96,28 @@ file(GLOB SRC_PLATFORM src/platform/PreventSleep.cpp src/platform/PreventSleep.hpp src/platform/Windows.hpp src/platform/DB.hpp ) +# We compile those folders with full optimization even in debug mode, otherwise binaries will run much slower in debug if(WIN32) set_property(SOURCE ${SRC_CRYPTO} PROPERTY COMPILE_FLAGS -Ot) set_property(SOURCE ${SRC_DB} PROPERTY COMPILE_FLAGS -Ot) set_property(SOURCE ${SRC_WARNINGS_DB} PROPERTY COMPILE_FLAGS "-Ot -w") set_property(SOURCE ${SRC_COMMON} PROPERTY COMPILE_FLAGS -Ot) - set_property(SOURCE ${SRC_SERIALIZATION} PROPERTY COMPILE_FLAGS -Ot) set_property(SOURCE ${SRC_SERIA} PROPERTY COMPILE_FLAGS -Ot) else() set_property(SOURCE ${SRC_CRYPTO} PROPERTY COMPILE_FLAGS -O3) set_property(SOURCE ${SRC_DB} PROPERTY COMPILE_FLAGS -O3) set_property(SOURCE ${SRC_WARNINGS_DB} PROPERTY COMPILE_FLAGS "-O3 -w") set_property(SOURCE ${SRC_COMMON} PROPERTY COMPILE_FLAGS -O3) - set_property(SOURCE ${SRC_SERIALIZATION} PROPERTY COMPILE_FLAGS -O3) set_property(SOURCE ${SRC_SERIA} PROPERTY COMPILE_FLAGS -O3) endif() include_directories(src) +include_directories(${CMAKE_BINARY_DIR}) set(SOURCE_FILES ${SRC_DB} ${SRC_WARNINGS_DB} ${SRC_COMMON} ${SRC_HTTP} ${SRC_CORE} - ${SRC_SERIALIZATION} ${SRC_SERIA} ${SRC_LOGGING} ${SRC_PLATFORM} @@ -127,23 +129,15 @@ set(SOURCE_FILES ) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bin") set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/libs") -if(WIN32) - include_directories(SYSTEM ../boost) - if(CMAKE_SIZEOF_VOID_P EQUAL 8) - link_directories(SYSTEM ../boost/stage/lib) # must be before add_executable, add_library - else() - link_directories(SYSTEM ../boost/stage32/lib) # must be before add_executable, add_library - endif() -endif() add_library(bytecoin-crypto ${SRC_CRYPTO}) add_library(bytecoin-core ${SOURCE_FILES}) target_link_libraries(bytecoin-core bytecoin-crypto) if(WIN32) add_executable(walletd src/main_walletd.cpp src/bytecoin.rc) # .rc works only if referenced directly in add_executable - add_executable(bytecoind src/main_bytecoind.cpp src/bytecoin.rc) # .rc works only if referenced directly in add_executable + add_executable(${CRYPTONOTE_NAME}d src/main_bytecoind.cpp src/bytecoin.rc) # .rc works only if referenced directly in add_executable else() add_executable(walletd src/main_walletd.cpp) - add_executable(bytecoind src/main_bytecoind.cpp) + add_executable(${CRYPTONOTE_NAME}d src/main_bytecoind.cpp) endif() add_executable(tests src/main_tests.cpp tests/io.hpp tests/Random.hpp tests/blockchain/test_blockchain.cpp tests/blockchain/test_blockchain.hpp @@ -154,30 +148,32 @@ add_executable(tests src/main_tests.cpp tests/io.hpp tests/Random.hpp tests/wallet_file/test_wallet_file.cpp tests/wallet_file/test_wallet_file.hpp) set(Boost_USE_STATIC_LIBS ON) add_definitions(-DBOOST_BIND_NO_PLACEHOLDERS=1 -DBOOST_CONFIG_SUPPRESS_OUTDATED_MESSAGE=1) # boost::_1 conflicts with std::_1 -target_link_libraries(walletd bytecoin-crypto bytecoin-core) -target_link_libraries(bytecoind bytecoin-crypto bytecoin-core) -target_link_libraries(tests bytecoin-crypto bytecoin-core) -if(WIN32) +add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY=1 -DBOOST_SYSTEM_NO_DEPRECATED=1) # required for header-only compilation +add_definitions(-DBOOST_DATE_TIME_NO_LIB=1 -DBOOST_SYSTEM_NO_LIB=1 -DBOOST_REGEX_NO_LIB=1) # required for header-only compilation +set(Boost_USE_MULTITHREADED OFF) # all boost libraries are multithreaded since some version +find_package(Boost 1.65) +if(Boost_FOUND) + message( STATUS "Boost found by find_boost, Boost_INCLUDE_DIRS: " ${Boost_INCLUDE_DIRS}) else() - set(Boost_USE_MULTITHREADED OFF) # all boost libraries are multithreaded since some version - if(APPLE) - set(BOOST_ROOT ../boost) - find_package(Boost 1.62 REQUIRED COMPONENTS system) - message(STATUS "Boost_INCLUDE_DIRS: " ${Boost_INCLUDE_DIRS}) - include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) - #set(Boost_LIBRARIES "${Boost_LIBRARIES}") - else() - message(STATUS "Boost_INCLUDE_DIRS: ../boost") - include_directories(SYSTEM ../boost) - set(Boost_LIBRARIES "${CMAKE_CURRENT_SOURCE_DIR}/../boost/stage/lib/libboost_system.a") + if(NOT EXISTS "${PARENT_DIR}/boost/boost/version.hpp") + message(FATAL_ERROR "Boost not found, please download and unpack boost into ${PARENT_DIR}/boost") endif() - message(STATUS "Boost_LIBRARIES: " ${Boost_LIBRARIES}) + set(Boost_INCLUDE_DIRS ${PARENT_DIR}/boost) + message( STATUS "Using boost from local folder, Boost_INCLUDE_DIRS: " ${Boost_INCLUDE_DIRS}) +endif() +include_directories(${Boost_INCLUDE_DIRS}) + +target_link_libraries(walletd bytecoin-crypto bytecoin-core) +target_link_libraries(${CRYPTONOTE_NAME}d bytecoin-crypto bytecoin-core) +target_link_libraries(tests bytecoin-crypto bytecoin-core) + +if(NOT WIN32) if(APPLE) set(CMAKE_OSX_DEPLOYMENT_TARGET "10.11") target_link_libraries(walletd "-framework Foundation" "-framework IOKit") - target_link_libraries(bytecoind "-framework Foundation" "-framework IOKit") + target_link_libraries(${CRYPTONOTE_NAME}d "-framework Foundation" "-framework IOKit") endif() - target_link_libraries(walletd ${Boost_LIBRARIES} ${LINK_OPENSSL} dl pthread) - target_link_libraries(bytecoind ${Boost_LIBRARIES} ${LINK_OPENSSL} dl pthread) - target_link_libraries(tests ${Boost_LIBRARIES} ${LINK_OPENSSL} dl pthread) + target_link_libraries(walletd ${LINK_OPENSSL} dl pthread) + target_link_libraries(${CRYPTONOTE_NAME}d ${LINK_OPENSSL} dl pthread) + target_link_libraries(tests ${LINK_OPENSSL} dl pthread) endif() diff --git a/ReleaseNotes.md b/ReleaseNotes.md index c12edb60..7a2c40e8 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,12 +1,57 @@ ## Release Notes +### v3.4.0-beta-20181212 + +*Consensus update (hard fork)* +- The release starts immediate voting in the stagenet. Voting in the mainnet will start when v3.4.0 is released. +- Introduce new unlinkable addresses, stored in new HD wallet. Single mnemonic is enough to restore all wallet addresses. +- Destination addresses can now be derived from blockchain with wallet secrets, so saving history is now unnecessary. +- Greatly simplified maximum block size calculations. Miners will now explicitly vote for maximum block size (up to hard limit of 2 MB), depending on how many expensive transactions are in memory pool. Reward penalties are removed and most other unnecessary checks on block/transactions sizes are also removed from consensus. +- New HD wallet is encrypted with chacha20 with salt (major improvement to previous wallet format) +- New auditable addresses, guaranteed to always have balance exactly equal to what view-only version of the same wallet shows (useful for public deposits). +- Signatures are now half size. +- The requirement that coinbase transactions are locked for 10 blocks is removed from consensus. +- Creating dust (or other not round output amounts) are now prohibited by consensus rules. +- Minimum anonymity is now 3 (4 output references per input). This does not apply to dust or not round outputs. +- `bytecoind` now calculates output obfuscation value, and does not use less-than-ideal outputs for mix-in + +*Command line changes/additions* +- New walletd command-line parameter to set creation time when importing keys or mnemonic (which have no inherent timestamp) +- New walletd command-line parameter to generate mnemonics +- New walletd command-line parameter to create unlinkable wallet from mnemonics +- New walletd command-line parameter to enable getting secrets bia JSON RPC API + +*Specific API improvements* +- By default, getting secrets via JSON RPC API is disabled. +- New walletd method 'get_wallet_records' with optional 'create' parameter can be used to get (creating if needed) wallet records from linkable wallet +- New walletd method `set_address_label` to set labels for addresses (labels are returned by `get_wallet_records`) +- New error code `TOO_MUCH_ANONYMITY` (`-305`) can be returned from `create_transaction`. + +*API deprecations (will be removed in version 3.5.0)* +- `walletd` method `get_addresses` is marked as deprecated, use `get_wallet_records` instead. +- `walletd` method `get_view_key_pair` is marked as deprecated, use `get_wallet_info` with `need_secrets` set to true instead. +- `walletd` method 'create_transaction' has 'save_history' and 'save_history_error' deprecated, because history is now always implicitly saved to blockchain. +- `next_block_effective_median_size` renamed to `recommended_max_transaction_size` with `next_block_effective_median_size` deprecated. + +*Incompatible API changes* +- `parent_block` is renamed to `root_block` in all contexts without deprecation, due to widespread confusion. + +*Improvements to P2P protocol to handle large network load* +- Header relay instead of block body relay. In most cases receiver will reasemble block from memory pool. In rare case when this is not possible, the block body will be requested. +- Transaction description relay instead of transactions body relay. Only new transactions will be requested. Transaction descriptions contain most recent referenced block hash, so that receiver can easily distinguish between "wrong signatures due to malicious intent" and "wrong signatures due to chain reorganisations" +- Incremental memory pool sync with cut-off limit by fee/byte. Allows pools with huge assymetry in size to quickly get into stable state. + +*Other changes* +- Memory pool size increased to 4 MB. +- `walletd` will automatically rebuild wallet cache after upgrade to this version. This can take long time for large wallets. + ### v3.3.3 - Fixed bug when `walletd` fails to create transactions for certain coins. ### v3.3.2 -- Fixed bug when an invalid transaction may persist in the payment queue. +- Now all folders we get from user/system are normalized by removing excess slashes from the tail ### v3.3.1 diff --git a/src/Core/Archive.cpp b/src/Core/Archive.cpp index 59f31e71..7f1908ac 100644 --- a/src/Core/Archive.cpp +++ b/src/Core/Archive.cpp @@ -2,7 +2,6 @@ // Licensed under the GNU Lesser General Public License. See LICENSE for details. #include "Archive.hpp" -#include #include #include "CryptoNoteTools.hpp" #include "Currency.hpp" @@ -13,7 +12,7 @@ #include "seria/BinaryInputStream.hpp" #include "seria/BinaryOutputStream.hpp" -using namespace bytecoin; +using namespace cn; using namespace platform; static const std::string RECORDS_PREFIX = "r"; @@ -23,25 +22,20 @@ const std::string Archive::BLOCK("b"); const std::string Archive::TRANSACTION("t"); const std::string Archive::CHECKPOINT("c"); -// static const float DB_COMMIT_PERIOD = 60; // 1 minute sounds good for archive - -Archive::Archive(bool read_only, const std::string &path) - : read_only(read_only) -// , commit_timer(std::bind(&Archive::db_commit, this)) -{ +Archive::Archive(bool read_only, const std::string &path) : m_read_only(read_only) { #if !platform_USE_SQLITE try { - m_db = std::make_unique(read_only, path); - if (!m_db->get("$unique_id", unique_id)) { + m_db = std::make_unique(read_only ? platform::O_READ_EXISTING : platform::O_OPEN_ALWAYS, path); + if (!m_db->get("$unique_id", m_unique_id)) { DB::Cursor cur = m_db->begin(std::string()); if (!cur.end()) throw std::runtime_error("Archive database format unknown version, please delete " + m_db->get_path()); - unique_id = common::pod_to_hex(crypto::random_keypair().public_key); - m_db->put("$unique_id", unique_id, true); - std::cout << "Created archive with unique id: " << unique_id << std::endl; + m_unique_id = common::pod_to_hex(crypto::random_keypair().public_key); + m_db->put("$unique_id", m_unique_id, true); + std::cout << "Created archive with unique id: " << m_unique_id << std::endl; } - DB::Cursor cur2 = m_db->rbegin(RECORDS_PREFIX); - next_record_id = cur2.end() ? 0 : 1 + common::read_varint_sqlite4(cur2.get_suffix()); + DB::Cursor cur2 = m_db->rbegin(RECORDS_PREFIX); + m_next_record_id = cur2.end() ? 0 : 1 + common::read_varint_sqlite4(cur2.get_suffix()); } catch (const std::exception &) { if (read_only) m_db = nullptr; @@ -62,7 +56,7 @@ void Archive::add(const std::string &type, const BinaryArray &data, const Hash &hash, const std::string &source_address) { - if (!m_db || read_only || source_address.empty()) + if (!m_db || m_read_only || source_address.empty()) return; std::cout << "Adding to archive: " << type << " hash=" << hash << " size=" << data.size() << " source_address=" << source_address << std::endl; @@ -70,34 +64,32 @@ void Archive::add(const std::string &type, DB::Value value; if (!m_db->get(hash_key, value)) m_db->put(hash_key, data, true); - api::bytecoind::GetArchive::ArchiveRecord rec; + api::cnd::GetArchive::ArchiveRecord rec; rec.timestamp = now_unix_timestamp(&rec.timestamp_usec); rec.type = type; rec.hash = hash; rec.source_address = source_address; - m_db->put(RECORDS_PREFIX + common::write_varint_sqlite4(next_record_id), seria::to_binary(rec), true); - next_record_id += 1; + m_db->put(RECORDS_PREFIX + common::write_varint_sqlite4(m_next_record_id), seria::to_binary(rec), true); + m_next_record_id += 1; } void Archive::db_commit() { - if (!m_db || read_only) + if (!m_db || m_read_only) return; m_db->commit_db_txn(); - // commit_timer.once(DB_COMMIT_PERIOD); } -void Archive::read_archive(api::bytecoind::GetArchive::Request &&req, api::bytecoind::GetArchive::Response &resp) { - if (unique_id.empty()) - throw api::bytecoind::GetArchive::Error( - api::bytecoind::GetArchive::ARCHIVE_NOT_ENABLED, "Archive was never enabled on this bytecoind", unique_id); - if (req.archive_id != unique_id) - throw api::bytecoind::GetArchive::Error( - api::bytecoind::GetArchive::WRONG_ARCHIVE_ID, "Archive id changed", unique_id); +void Archive::read_archive(api::cnd::GetArchive::Request &&req, api::cnd::GetArchive::Response &resp) { + if (m_unique_id.empty()) + throw api::cnd::GetArchive::Error( + api::cnd::GetArchive::ARCHIVE_NOT_ENABLED, "Archive was never enabled on this node", m_unique_id); + if (req.archive_id != m_unique_id) + throw api::cnd::GetArchive::Error(api::cnd::GetArchive::WRONG_ARCHIVE_ID, "Archive id changed", m_unique_id); resp.from_record = req.from_record; - if (resp.from_record > next_record_id) - resp.from_record = next_record_id; - if (req.max_count > api::bytecoind::GetArchive::Request::MAX_COUNT) - req.max_count = api::bytecoind::GetArchive::Request::MAX_COUNT; + if (resp.from_record > m_next_record_id) + resp.from_record = m_next_record_id; + if (req.max_count > api::cnd::GetArchive::Request::MAX_COUNT) + req.max_count = api::cnd::GetArchive::Request::MAX_COUNT; if (!m_db) return; resp.records.reserve(static_cast(req.max_count)); @@ -105,7 +97,7 @@ void Archive::read_archive(api::bytecoind::GetArchive::Request &&req, api::bytec cur.next()) { if (resp.records.size() >= req.max_count) break; - api::bytecoind::GetArchive::ArchiveRecord rec; + api::cnd::GetArchive::ArchiveRecord rec; seria::from_binary(rec, cur.get_value_array()); resp.records.push_back(rec); if (req.records_only) @@ -116,11 +108,10 @@ void Archive::read_archive(api::bytecoind::GetArchive::Request &&req, api::bytec if (resp.blocks.count(str_hash) == 0) { BinaryArray data; invariant(m_db->get(hash_key, data), ""); - api::bytecoind::GetArchive::ArchiveBlock &bl = resp.blocks[str_hash]; + api::cnd::GetArchive::ArchiveBlock &bl = resp.blocks[str_hash]; RawBlock raw_block; seria::from_binary(raw_block, data); - Block block; - invariant(block.from_raw_block(raw_block), ""); + Block block(raw_block); bl.raw_header = block.header; bl.raw_transactions.reserve(block.transactions.size()); bl.transaction_binary_sizes.reserve(block.transactions.size() + 1); diff --git a/src/Core/Archive.hpp b/src/Core/Archive.hpp index 846ddca0..0544f247 100644 --- a/src/Core/Archive.hpp +++ b/src/Core/Archive.hpp @@ -4,24 +4,23 @@ #pragma once #include "platform/DB.hpp" -//#include "platform/Network.hpp" #include "rpc_api.hpp" -namespace bytecoin { +namespace cn { class Archive { - const bool read_only; + const bool m_read_only; std::unique_ptr m_db; - uint64_t next_record_id = 0; - std::string unique_id; + uint64_t m_next_record_id = 0; + std::string m_unique_id; // platform::Timer commit_timer; public: explicit Archive(bool read_only, const std::string &path); - std::string get_unique_id() const { return unique_id; } + std::string get_unique_id() const { return m_unique_id; } void add( const std::string &type, const common::BinaryArray &data, const Hash &hash, const std::string &source_address); - void read_archive(api::bytecoind::GetArchive::Request &&req, api::bytecoind::GetArchive::Response &resp); + void read_archive(api::cnd::GetArchive::Request &&req, api::cnd::GetArchive::Response &resp); void db_commit(); static const std::string BLOCK; @@ -29,4 +28,4 @@ class Archive { static const std::string CHECKPOINT; }; -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/BlockChain.cpp b/src/Core/BlockChain.cpp index a73784e4..6c4e85e3 100644 --- a/src/Core/BlockChain.cpp +++ b/src/Core/BlockChain.cpp @@ -3,7 +3,6 @@ #include "BlockChain.hpp" -#include #include #include "Config.hpp" #include "CryptoNoteTools.hpp" @@ -17,10 +16,10 @@ #include "seria/BinaryInputStream.hpp" #include "seria/BinaryOutputStream.hpp" -using namespace bytecoin; +using namespace cn; using namespace platform; -const std::string BlockChain::version_current = "5"; +const std::string BlockChain::version_current = "6"; // We increment when making incompatible changes to indices. // We use suffixes so all keys related to the same block are close to each other in DB @@ -28,7 +27,7 @@ static const std::string BLOCK_PREFIX = "b"; static const std::string BLOCK_SUFFIX = "b"; static const std::string HEADER_PREFIX = "b"; static const std::string HEADER_SUFFIX = "h"; -static const std::string TRANSATION_PREFIX = "t"; +static const std::string TRANSACTION_PREFIX = "t"; static const std::string TIP_CHAIN_PREFIX = "c"; static const std::string TIMESTAMP_BLOCK_PREFIX = "T"; static const std::string CHECKPOINT_PREFIX_STABLE = "CS"; @@ -39,63 +38,25 @@ static const std::string CD_TIPS_PREFIX = "x-tips/"; // We store bid->children counter, with counter=1 default (absent from index) // We store cumulative_difficulty->bid for bids with no children -static const size_t COMMIT_EVERY_N_BLOCKS = 50000; // We do not want to create too large DB transactions - -static const std::string delete_blockchain_message = "database corrupted, please delete "; - -bool Block::from_raw_block(const RawBlock &raw_block) { - try { - BlockTemplate &bheader = header; - seria::from_binary(bheader, raw_block.block); - transactions.resize(0); - transactions.reserve(raw_block.transactions.size()); - for (auto &&raw_transaction : raw_block.transactions) { - Transaction transaction; - seria::from_binary(transaction, raw_transaction); - transactions.push_back(std::move(transaction)); - } - } catch (...) { - return false; - } - return true; -} - -bool Block::to_raw_block(RawBlock &raw_block) const { - try { - const BlockTemplate &bheader = header; - raw_block.block = seria::to_binary(bheader); - raw_block.transactions.resize(0); - raw_block.transactions.reserve(transactions.size()); - for (auto &&transaction : transactions) { - BinaryArray raw_transaction = seria::to_binary(transaction); - raw_block.transactions.push_back(std::move(raw_transaction)); - } - } catch (...) { - return false; +Block::Block(const RawBlock &rb) { + BlockTemplate &bheader = header; + seria::from_binary(bheader, rb.block); + transactions.reserve(rb.transactions.size()); + for (auto &&raw_transaction : rb.transactions) { + Transaction transaction; + seria::from_binary(transaction, raw_transaction); + transactions.push_back(std::move(transaction)); } - return true; } PreparedBlock::PreparedBlock(BinaryArray &&ba, const Currency ¤cy, crypto::CryptoNightContext *context) : block_data(std::move(ba)) { try { seria::from_binary(raw_block, block_data); - if (!block.from_raw_block(raw_block)) { - error_text = "from_raw_block failed"; - return; - } - auto body_proxy = get_body_proxy_from_template(block.header); - bid = bytecoin::get_block_hash(block.header, body_proxy); - if (block.header.is_merge_mined()) - parent_block_size = seria::binary_size(block.header.parent_block); - coinbase_tx_size = seria::binary_size(block.header.base_transaction); - base_transaction_hash = get_transaction_hash(block.header.base_transaction); - if (context) { - auto ba = currency.get_block_long_hashing_data(block.header, body_proxy); - long_block_hash = context->cn_slow_hash(ba.data(), ba.size()); - } + prepare(currency, context); } catch (const std::exception &ex) { - error_text = common::what(ex); + error = ConsensusError{common::what(ex)}; + return; } } @@ -103,28 +64,43 @@ PreparedBlock::PreparedBlock(RawBlock &&rba, const Currency ¤cy, crypto::C : raw_block(rba) { try { block_data = seria::to_binary(raw_block); - if (!block.from_raw_block(raw_block)) { - error_text = "from_raw_block failed"; + prepare(currency, context); + } catch (const std::exception &ex) { + error = ConsensusError{common::what(ex)}; + return; + } +} + +void PreparedBlock::prepare(const Currency ¤cy, crypto::CryptoNightContext *context) { + block = Block{raw_block}; + auto body_proxy = get_body_proxy_from_template(block.header); + bid = cn::get_block_hash(block.header, body_proxy); + if (block.header.is_merge_mined()) + parent_block_size = seria::binary_size(block.header.root_block); + coinbase_tx_size = seria::binary_size(block.header.base_transaction); + block_header_size = seria::binary_size(static_cast(block.header)); + base_transaction_hash = get_transaction_hash(block.header.base_transaction); + if (context) { + auto ba = currency.get_block_long_hashing_data(block.header, body_proxy); + long_block_hash = context->cn_slow_hash(ba.data(), ba.size()); + } + if (block.header.transaction_hashes.size() != raw_block.transactions.size()) { + error = ConsensusError{"Wrong transcation count in block template"}; + return; + } + // Transactions are in block + for (size_t i = 0; i != block.transactions.size(); ++i) { + Hash tid = get_transaction_hash(block.transactions.at(i)); + if (tid != block.header.transaction_hashes.at(i)) { + error = ConsensusError{"Transaction from block template absent in block"}; return; } - auto body_proxy = get_body_proxy_from_template(block.header); - bid = bytecoin::get_block_hash(block.header, body_proxy); - if (block.header.is_merge_mined()) - parent_block_size = seria::binary_size(block.header.parent_block); - coinbase_tx_size = seria::binary_size(block.header.base_transaction); - base_transaction_hash = get_transaction_hash(block.header.base_transaction); - if (context) { - auto ba = currency.get_block_long_hashing_data(block.header, body_proxy); - long_block_hash = context->cn_slow_hash(ba.data(), ba.size()); - } - } catch (const std::exception &ex) { - error_text = common::what(ex); } } BlockChain::BlockChain(logging::ILogger &log, const Config &config, const Currency ¤cy, bool read_only) : m_genesis_bid(currency.genesis_block_hash) - , m_db(read_only, config.get_data_folder() + "/blockchain") + , m_db(read_only ? platform::O_READ_EXISTING : platform::O_OPEN_ALWAYS, config.get_data_folder() + "/blockchain") , m_archive(read_only || !config.is_archive, config.get_data_folder() + "/archive") , m_log(log, "BlockChainState") , m_config(config) @@ -141,95 +117,103 @@ BlockChain::BlockChain(logging::ILogger &log, const Config &config, const Curren if (version != version_current) return; // BlockChainState will upgrade DB, we must not continue or risk crashing Hash stored_genesis_bid; - if (read_chain(0, &stored_genesis_bid)) { + if (get_chain(0, &stored_genesis_bid)) { if (stored_genesis_bid != m_genesis_bid) throw std::runtime_error("Database starts with different genesis_block"); - read_tip(); + DB::Cursor cur2 = m_db.rbegin(TIP_CHAIN_PREFIX); + m_tip_height = cur2.end() ? -1 : common::integer_cast(common::read_varint_sqlite4(cur2.get_suffix())); + seria::from_binary(m_tip_bid, cur2.get_value_array()); + api::BlockHeader tip_header = read_header(m_tip_bid); + m_tip_cumulative_difficulty = tip_header.cumulative_difficulty; + m_header_tip_window.push_back(tip_header); } BinaryArray cha; - if (m_db.get("internal_import_chain", cha)) + if (m_db.get("internal_import_chain", cha)) { seria::from_binary(m_internal_import_chain, cha); + m_log(logging::INFO) << "BlockChain continue internal import of blocks, count=" + << m_internal_import_chain.size() << std::endl; + } } void BlockChain::db_commit() { m_log(logging::INFO) << "BlockChain::db_commit started... tip_height=" << m_tip_height - << " header_cache.size=" << header_cache.size() << std::endl; + << " m_header_cache.size=" << m_header_cache.size() << std::endl; m_db.commit_db_txn(); - header_cache.clear(); // Most simple cache policy ever + m_header_cache.clear(); // Most simple cache policy ever m_archive.db_commit(); m_log(logging::INFO) << "BlockChain::db_commit finished..." << std::endl; } -BroadcastAction BlockChain::add_block( - const PreparedBlock &pb, api::BlockHeader *info, const std::string &source_address) { +bool BlockChain::add_block(const PreparedBlock &pb, api::BlockHeader *info, const std::string &source_address) { *info = api::BlockHeader(); - bool have_header = read_header(pb.bid, info); + bool have_header = get_header(pb.bid, info); bool have_block = has_block(pb.bid); if (have_block && have_header) { - if (info->height > m_currency.last_sw_checkpoint().height) + if (info->height > m_currency.last_hard_checkpoint().height) m_archive.add(Archive::BLOCK, pb.block_data, pb.bid, source_address); - return BroadcastAction::NOTHING; + return false; } api::BlockHeader prev_info; prev_info.height = -1; - if (pb.bid != m_genesis_bid && !read_header(pb.block.header.previous_block_hash, &prev_info)) - return BroadcastAction::NOTHING; // Not interested in orphan headers + if (pb.bid != m_genesis_bid && !get_header(pb.block.header.previous_block_hash, &prev_info)) + return false; // Not interested in orphan headers info->major_version = pb.block.header.major_version; info->minor_version = pb.block.header.minor_version; info->timestamp = pb.block.header.timestamp; info->previous_block_hash = pb.block.header.previous_block_hash; - info->nonce = pb.block.header.nonce; + info->binary_nonce = pb.block.header.nonce; info->hash = pb.bid; info->height = prev_info.height + 1; // Rest fields are filled by check_standalone_consensus - std::string check_error = check_standalone_consensus(pb, info, prev_info, true); - if (!check_error.empty()) - return BroadcastAction::BAN; // TODO - return check_error - if (!add_blod(*info)) { // Has parent that does not pass through last SW checkpoint - if (info->height > m_currency.last_sw_checkpoint().height) + check_standalone_consensus(pb, info, prev_info, true); // throws ConsensusError + if (!add_blod(*info)) { // Has parent that does not pass through last hard checkpoint + if (info->height > m_currency.last_hard_checkpoint().height) m_archive.add(Archive::BLOCK, pb.block_data, pb.bid, source_address); - return BroadcastAction::NOTHING; + return false; } try { if (!have_block) { // have block, but not header during internal_import store_block(pb.bid, pb.block_data); // Do not commit between here and // reorganize_blocks or invariant might be dead - if (info->height > m_currency.last_sw_checkpoint().height) + if (info->height > m_currency.last_hard_checkpoint().height) m_archive.add(Archive::BLOCK, pb.block_data, pb.bid, source_address); } store_header(pb.bid, *info); if (pb.bid == m_genesis_bid) { - invariant(redo_block(pb.bid, pb.block_data, pb.raw_block, pb.block, *info, pb.base_transaction_hash), - "Failed to apply genesis block"); + redo_block(pb.bid, pb.block_data, pb.raw_block, pb.block, *info, pb.base_transaction_hash); push_chain(*info); - // debug_check_transaction_invariants(pb.raw_block, pb.block, *info, pb.base_transaction_hash); + if (m_config.paranoid_checks) + debug_check_transaction_invariants(pb.raw_block, pb.block, *info, pb.base_transaction_hash); } else { modify_children_counter(prev_info.cumulative_difficulty, pb.block.header.previous_block_hash, 1); } check_children_counter(info->cumulative_difficulty, pb.bid, 1); modify_children_counter(info->cumulative_difficulty, pb.bid, -1); // -1 from default 1 gives 0 - if (info->hash == m_currency.last_sw_checkpoint().hash) + if (info->hash == m_currency.last_hard_checkpoint().hash) build_blods(); auto tip_check_cd = get_checkpoint_difficulty(get_tip_bid()); auto bid_check_cd = get_checkpoint_difficulty(info->hash); if (compare(bid_check_cd, info->cumulative_difficulty, tip_check_cd, get_tip_cumulative_difficulty()) > 0) { if (get_tip_bid() == pb.block.header.previous_block_hash) { // most common case optimization - if (!redo_block(pb.bid, pb.block_data, pb.raw_block, pb.block, *info, pb.base_transaction_hash)) - return BroadcastAction::BAN; + redo_block(pb.bid, pb.block_data, pb.raw_block, pb.block, *info, pb.base_transaction_hash); push_chain(*info); - // debug_check_transaction_invariants(pb.raw_block, pb.block, *info, - // pb.base_transaction_hash); + if (m_config.paranoid_checks) + debug_check_transaction_invariants(pb.raw_block, pb.block, *info, pb.base_transaction_hash); } else reorganize_blocks(pb.bid, pb, *info); } + } catch (const ConsensusError &) { + throw; // The only exception which is safe here } catch (const std::exception &ex) { m_log(logging::ERROR) << "Exception while reorganizing blockchain, probably out of disk space ex.what=" - << common::what(ex) << ", " << delete_blockchain_message << m_db.get_path() << std::endl; + << common::what(ex) << ", database corrupted, please delete " << m_db.get_path() + << std::endl; std::exit(api::BYTECOIND_DATABASE_ERROR); } - if (get_tip_height() % COMMIT_EVERY_N_BLOCKS == COMMIT_EVERY_N_BLOCKS - 1) // no commit on genesis + if (get_tip_height() % m_config.db_commit_every_n_blocks == + m_config.db_commit_every_n_blocks - 1) // no commit on genesis db_commit(); - return info->hash == get_tip_bid() ? BroadcastAction::BROADCAST_ALL : BroadcastAction::NOTHING; + return info->hash == get_tip_bid(); } void BlockChain::debug_check_transaction_invariants(const RawBlock &raw_block, const Block &block, @@ -239,13 +223,13 @@ void BlockChain::debug_check_transaction_invariants(const RawBlock &raw_block, c Height bhe; Hash bha; size_t iib; - invariant(read_transaction(base_transaction_hash, &binary_tx, &bhe, &bha, &iib), "tx index invariant failed 1"); + invariant(get_transaction(base_transaction_hash, &binary_tx, &bhe, &bha, &iib), "tx index invariant failed 1"); seria::from_binary(rtx, binary_tx); invariant(get_transaction_hash(rtx) == base_transaction_hash && bhe == info.height && bha == info.hash && iib == 0, "tx index invariant failed 2"); for (size_t tx_index = 0; tx_index != block.transactions.size(); ++tx_index) { Hash tid = block.header.transaction_hashes.at(tx_index); - invariant(read_transaction(tid, &binary_tx, &bhe, &bha, &iib), "tx index invariant failed 3"); + invariant(get_transaction(tid, &binary_tx, &bhe, &bha, &iib), "tx index invariant failed 3"); seria::from_binary(rtx, binary_tx); invariant(seria::to_binary(rtx) == raw_block.transactions.at(tx_index) && bhe == info.height && bha == info.hash && iib == tx_index + 1 && @@ -265,71 +249,69 @@ bool BlockChain::reorganize_blocks(const Hash &switch_to_chain, return false; // Full new chain not yet downloaded } std::map> undone_transactions; - bool undone_blocks = false; + size_t undone_transactions_binary_size = 0; + bool undone_blocks = false; while (get_tip_bid() != common) { RawBlock raw_block; - Block block; - invariant(read_block(get_tip_bid(), &raw_block) && block.from_raw_block(raw_block), + invariant(get_block(get_tip_bid(), &raw_block), "Block to undo not found or failed to convert" + common::pod_to_hex(get_tip_bid())); + Block block(raw_block); undone_blocks = true; undo_block(get_tip_bid(), raw_block, block, m_tip_height); - for (size_t tx_index = 0; tx_index != block.transactions.size(); ++tx_index) { - Hash tid = block.header.transaction_hashes.at(tx_index); - undone_transactions.insert(std::make_pair(tid, std::make_pair(std::move(block.transactions.at(tx_index)), - std::move(raw_block.transactions.at(tx_index))))); - } + if (undone_transactions_binary_size < m_config.max_undo_transactions_size) + for (size_t tx_index = 0; tx_index != block.transactions.size(); ++tx_index) { + Hash tid = block.header.transaction_hashes.at(tx_index); + undone_transactions_binary_size += raw_block.transactions.at(tx_index).size(); + undone_transactions.insert( + std::make_pair(tid, std::make_pair(std::move(block.transactions.at(tx_index)), + std::move(raw_block.transactions.at(tx_index))))); + } pop_chain(block.header.previous_block_hash); tip_changed(); } // Now redo all blocks we have in storage, will ask for the rest of blocks - bool result = true; - while (!chain2.empty()) { - Hash chha = chain2.back(); - chain2.pop_back(); - if (chha == recent_pb.bid) { - invariant( - recent_pb.block.header.previous_block_hash == get_tip_bid(), "Unexpected block prev, invariant dead"); - if (!redo_block(recent_pb.bid, recent_pb.block_data, recent_pb.raw_block, recent_pb.block, recent_info, - recent_pb.base_transaction_hash)) { - // invalid block on longest subchain, make no attempt to download the - // rest - // we will forever stuck on this block until longer chain appears, that - // does not include it - result = false; - break; - } - push_chain(recent_info); - for (auto &&tid : recent_pb.block.header.transaction_hashes) - undone_transactions.erase(tid); - if (m_config.paranoid_checks) - debug_check_transaction_invariants( - recent_pb.raw_block, recent_pb.block, recent_info, recent_pb.base_transaction_hash); - } else { - BinaryArray block_data; - RawBlock raw_block; - Block block; - if (!read_block(chha, &block_data, &raw_block) || !block.from_raw_block(raw_block)) { - result = false; - break; // Strange, we checked has_block, somehow "bad block" got into DB. TODO - throw? - } - invariant(block.header.previous_block_hash == get_tip_bid(), "Unexpected block prev, invariant dead"); - api::BlockHeader info = read_header(chha); - Hash base_transaction_hash = get_transaction_hash(block.header.base_transaction); - // if redo fails, we will forever stuck on this block until longer chain - // appears, that does not include it - if (!redo_block(chha, block_data, raw_block, block, info, base_transaction_hash)) { - result = false; - break; + // We catch consensus error from redo_block + // when invalid block on longest subchain, we should make no attempt to download the rest + // we will forever stuck on this block until longer chain appears, that does not include it + try { + while (!chain2.empty()) { + Hash chha = chain2.back(); + chain2.pop_back(); + if (chha == recent_pb.bid) { + invariant(recent_pb.block.header.previous_block_hash == get_tip_bid(), + "Unexpected block prev, invariant dead"); + redo_block(recent_pb.bid, recent_pb.block_data, recent_pb.raw_block, recent_pb.block, recent_info, + recent_pb.base_transaction_hash); + push_chain(recent_info); + for (auto &&tid : recent_pb.block.header.transaction_hashes) + undone_transactions.erase(tid); + if (m_config.paranoid_checks) + debug_check_transaction_invariants( + recent_pb.raw_block, recent_pb.block, recent_info, recent_pb.base_transaction_hash); + } else { + BinaryArray block_data; + RawBlock raw_block; + invariant(get_block(chha, &block_data, &raw_block), ""); + Block block(raw_block); + invariant(block.header.previous_block_hash == get_tip_bid(), "Unexpected block prev, invariant dead"); + api::BlockHeader info = read_header(chha); + Hash base_transaction_hash = get_transaction_hash(block.header.base_transaction); + + redo_block(chha, block_data, raw_block, block, info, base_transaction_hash); + push_chain(info); + for (auto &&tid : block.header.transaction_hashes) + undone_transactions.erase(tid); + if (m_config.paranoid_checks) + debug_check_transaction_invariants(raw_block, block, info, base_transaction_hash); } - push_chain(info); - for (auto &&tid : block.header.transaction_hashes) - undone_transactions.erase(tid); - if (m_config.paranoid_checks) - debug_check_transaction_invariants(raw_block, block, info, base_transaction_hash); } + } catch (const ConsensusError &) { + // The only exception which is safe here + on_reorganization(undone_transactions, undone_blocks); + return false; } on_reorganization(undone_transactions, undone_blocks); - return result; + return true; } Hash BlockChain::get_common_block( @@ -368,70 +350,39 @@ Hash BlockChain::get_common_block( return hid1; } -std::vector BlockChain::get_sparse_chain() const { +std::vector BlockChain::get_sparse_chain(Height max_jump) const { std::vector result; - auto tip_path = get_sparse_chain(m_genesis_bid, m_tip_bid); + auto tip_path = get_sparse_chain(m_genesis_bid, m_tip_bid, max_jump); for (const auto &el : tip_path) result.push_back(el.hash); return result; - // Height jump = 0; - // while (m_tip_height >= jump) { - // tip_path.push_back(read_chain(m_tip_height - jump)); - // if (tip_path.size() <= 10) - // jump += 1; - // else - // jump += (1 << (tip_path.size() - 10)); - // } - // if (tip_path.back() != m_genesis_bid) - // tip_path.push_back(m_genesis_bid); - // return tip_path; } -std::vector BlockChain::get_sparse_chain(Hash start, Hash end) const { - std::vector tip_path; +std::vector BlockChain::get_sparse_chain(Hash start, Hash end, Height max_jump) const { + std::vector tip_path; api::BlockHeader header_end; - if (!read_header(end, &header_end) || !in_chain(header_end.height, header_end.hash)) + if (!get_header(end, &header_end) || !in_chain(header_end.height, header_end.hash)) return tip_path; api::BlockHeader header_start; - if (!read_header(start, &header_start) || !in_chain(header_start.height, header_start.hash)) + if (!get_header(start, &header_start) || !in_chain(header_start.height, header_start.hash)) return tip_path; - Height jump = 0; - while (header_end.height >= jump + header_start.height) { - tip_path.push_back(SWCheckpoint{header_end.height - jump, read_chain(header_end.height - jump)}); - if (tip_path.size() <= 10) - jump += 1; - else - jump += (1 << (tip_path.size() - 10)); + Height jump = 1; + Height delta = 0; + while (header_end.height >= delta + header_start.height) { + tip_path.push_back(HardCheckpoint{header_end.height - delta, read_chain(header_end.height - delta)}); + if (tip_path.size() >= 10) { + jump *= 2; + if (tip_path.back().height > m_currency.last_hard_checkpoint().height && jump > max_jump) + jump = max_jump; // no big jumps above last hard checkpoint + } + delta += jump; } if (tip_path.back().hash != start) - tip_path.push_back(SWCheckpoint{header_start.height, header_start.hash}); + tip_path.push_back(HardCheckpoint{header_start.height, header_start.hash}); return tip_path; } -std::vector BlockChain::get_sync_headers(const std::vector &locator, size_t max_count) const { - std::vector result; - Height start_height = 0; - std::vector chain = get_sync_headers_chain(locator, &start_height, max_count); - result.reserve(chain.size()); - for (auto &&c : chain) { - result.push_back(read_header(c)); - } - return result; -} - -uint32_t BlockChain::find_blockchain_supplement(const std::vector &remote_block_ids) const { - for (auto &&lit : remote_block_ids) { - api::BlockHeader header; - if (!read_header(lit, &header)) - continue; - if (header.height > m_tip_height) - continue; - return header.height; - } - return 0; // Not possible if genesis blocks match -} - Height BlockChain::get_timestamp_lower_bound_height(Timestamp ts) const { auto middle = common::write_varint_sqlite4(ts); DB::Cursor cur = m_db.begin(TIMESTAMP_BLOCK_PREFIX, middle); @@ -443,52 +394,34 @@ Height BlockChain::get_timestamp_lower_bound_height(Timestamp ts) const { return common::integer_cast(common::read_varint_sqlite4(be, en)); } -std::vector BlockChain::get_sync_headers_chain(const std::vector &locator, - Height *start_height, - size_t max_count) const { +std::vector BlockChain::get_sync_headers_chain( + const std::vector &locator, Height *start_height, size_t max_count) const { std::vector result; for (auto &&lit : locator) { api::BlockHeader header; - if (!read_header(lit, &header)) + if (!get_header(lit, &header)) continue; while (header.height != 0) { Hash ha; - if (read_chain(header.height, &ha) && ha == header.hash) + if (get_chain(header.height, &ha) && ha == header.hash) break; header = read_header(header.previous_block_hash); } - uint32_t min_height = header.height; - *start_height = min_height; + Height min_height = header.height; + *start_height = min_height; for (; result.size() < max_count && min_height <= m_tip_height; min_height += 1) { result.push_back(read_chain(min_height)); } return result; } - *start_height = m_tip_height + 1; - return result; -} - -struct APIRawBlockHeightDifficulty { - RawBlock &raw_block; - Height &height; - Difficulty &cd; - APIRawBlockHeightDifficulty(RawBlock &raw_block, Height &height, Difficulty &cd) - : raw_block(raw_block), height(height), cd(cd) {} -}; - -namespace seria { -void ser_members(APIRawBlockHeightDifficulty &v, ISeria &s) { - seria_kv("height", v.height, s); - seria_kv("cd", v.cd, s); - seria_kv("raw_block", v.raw_block, s); + throw std::runtime_error("No common block found in get_sync_headers_chain"); } -} // namespace seria struct APITransactionPos { - Height height = 0; - uint32_t offset = 0; - uint32_t size = 0; - uint32_t index = 0; + Height height = 0; + size_t offset = 0; + size_t size = 0; + size_t index = 0; }; namespace seria { @@ -500,9 +433,15 @@ void ser_members(APITransactionPos &v, ISeria &s) { } } // namespace seria -bool BlockChain::read_transaction( +bool BlockChain::has_transaction(const Hash &tid) const { + DB::Value value; + auto txkey = TRANSACTION_PREFIX + DB::to_binary_key(tid.data, sizeof(tid.data)); + return m_db.get(txkey, value); +} + +bool BlockChain::get_transaction( const Hash &tid, BinaryArray *binary_tx, Height *block_height, Hash *block_hash, size_t *index_in_block) const { - auto txkey = TRANSATION_PREFIX + DB::to_binary_key(tid.data, sizeof(tid.data)); + auto txkey = TRANSACTION_PREFIX + DB::to_binary_key(tid.data, sizeof(tid.data)); BinaryArray ba; if (!m_db.get(txkey, ba)) return false; @@ -520,38 +459,36 @@ bool BlockChain::read_transaction( return true; } -bool BlockChain::redo_block(const Hash &bhash, const BinaryArray &block_data, const RawBlock &raw_block, +void BlockChain::redo_block(const Hash &bhash, const BinaryArray &block_data, const RawBlock &raw_block, const Block &block, const api::BlockHeader &info, const Hash &base_transaction_hash) { - if (!redo_block(bhash, block, info)) - return false; + redo_block(bhash, block, info); auto tikey = TIMESTAMP_BLOCK_PREFIX + common::write_varint_sqlite4(info.timestamp) + common::write_varint_sqlite4(info.height); m_db.put(tikey, std::string(), true); APITransactionPos tpos; tpos.height = info.height; - auto bkey = TRANSATION_PREFIX + DB::to_binary_key(base_transaction_hash.data, sizeof(base_transaction_hash.data)); - tpos.index = 0; + auto bkey = TRANSACTION_PREFIX + DB::to_binary_key(base_transaction_hash.data, sizeof(base_transaction_hash.data)); + tpos.index = 0; BinaryArray coinbase_ba = seria::to_binary(block.header.base_transaction); auto ptr = common::slow_memmem(block_data.data() + tpos.offset + tpos.size, - block_data.size() - tpos.offset - tpos.size, coinbase_ba.data(), coinbase_ba.size()); + block_data.size() - tpos.offset - tpos.size, coinbase_ba.data(), coinbase_ba.size()); invariant(ptr, "binary coinbase tx not found in binary block"); - tpos.offset = static_cast(ptr - block_data.data()); - tpos.size = static_cast(coinbase_ba.size()); + tpos.offset = ptr - block_data.data(); + tpos.size = coinbase_ba.size(); m_db.put(bkey, seria::to_binary(tpos), true); for (size_t tx_index = 0; tx_index != block.transactions.size(); ++tx_index) { Hash tid = block.header.transaction_hashes.at(tx_index); - tpos.index = static_cast(tx_index + 1); - bkey = TRANSATION_PREFIX + DB::to_binary_key(tid.data, sizeof(tid.data)); + tpos.index = tx_index + 1; + bkey = TRANSACTION_PREFIX + DB::to_binary_key(tid.data, sizeof(tid.data)); const auto &binary_tx = raw_block.transactions.at(tx_index); ptr = common::slow_memmem(block_data.data() + tpos.offset + tpos.size, - block_data.size() - tpos.offset - tpos.size, binary_tx.data(), binary_tx.size()); + block_data.size() - tpos.offset - tpos.size, binary_tx.data(), binary_tx.size()); invariant(ptr, "binary tx not found in binary block"); - tpos.offset = static_cast(ptr - block_data.data()); - tpos.size = static_cast(binary_tx.size()); + tpos.offset = ptr - block_data.data(); + tpos.size = binary_tx.size(); m_db.put(bkey, seria::to_binary(tpos), true); } - return true; } void BlockChain::undo_block(const Hash &bhash, const RawBlock &, const Block &block, Height height) { // if (!m_tip_segment.empty()) @@ -563,11 +500,11 @@ void BlockChain::undo_block(const Hash &bhash, const RawBlock &, const Block &bl m_db.del(tikey, true); Hash tid = get_transaction_hash(block.header.base_transaction); - auto bkey = TRANSATION_PREFIX + DB::to_binary_key(tid.data, sizeof(tid.data)); + auto bkey = TRANSACTION_PREFIX + DB::to_binary_key(tid.data, sizeof(tid.data)); m_db.del(bkey, true); for (size_t tx_index = 0; tx_index != block.transactions.size(); ++tx_index) { tid = block.header.transaction_hashes.at(tx_index); - bkey = TRANSATION_PREFIX + DB::to_binary_key(tid.data, sizeof(tid.data)); + bkey = TRANSACTION_PREFIX + DB::to_binary_key(tid.data, sizeof(tid.data)); m_db.del(bkey, true); } } @@ -577,7 +514,7 @@ void BlockChain::store_block(const Hash &bid, const BinaryArray &block_data) { m_db.put(key, block_data, true); } -bool BlockChain::read_block(const Hash &bid, BinaryArray *block_data, RawBlock *raw_block) const { +bool BlockChain::get_block(const Hash &bid, BinaryArray *block_data, RawBlock *raw_block) const { BinaryArray rb; auto key = BLOCK_PREFIX + DB::to_binary_key(bid.data, sizeof(bid.data)) + BLOCK_SUFFIX; if (!m_db.get(key, rb)) @@ -588,9 +525,9 @@ bool BlockChain::read_block(const Hash &bid, BinaryArray *block_data, RawBlock * return true; } -bool BlockChain::read_block(const Hash &bid, RawBlock *raw_block) const { +bool BlockChain::get_block(const Hash &bid, RawBlock *raw_block) const { BinaryArray rb; - return read_block(bid, &rb, raw_block); + return get_block(bid, &rb, raw_block); } bool BlockChain::has_block(const Hash &bid) const { @@ -607,50 +544,43 @@ void BlockChain::store_header(const Hash &bid, const api::BlockHeader &header) { m_db.put(key, ba, true); } -bool BlockChain::read_header(const Hash &bid, api::BlockHeader *header, Height hint) const { +const api::BlockHeader *BlockChain::read_header_fast(const Hash &bid, Height hint) const { if (get_tip_height() != Height(-1) && hint <= get_tip_height() && hint >= get_tip_height() - m_header_tip_window.size() + 1) { const auto &candidate = m_header_tip_window.at(m_header_tip_window.size() - 1 - (get_tip_height() - hint)); if (candidate.hash == bid) { - *header = candidate; // fastest lookup is in tip window - return true; + return &candidate; // fastest lookup is in tip window } } - auto cit = header_cache.find(bid); - if (cit != header_cache.end()) { - *header = cit->second; - return true; + auto cit = m_header_cache.find(bid); + if (cit != m_header_cache.end()) { + return &cit->second; } - if (header_cache.size() > m_currency.largest_window() * 20) { + Hash bbid = bid; // next lines can modify bid, because it can be reference to header inside cache + if (m_header_cache.size() > m_currency.largest_window() * 20) { m_log(logging::INFO) << "BlockChain header cache reached max size and cleared" << std::endl; - header_cache.clear(); // very simple policy + m_header_cache.clear(); // very simple policy } BinaryArray rb; - auto key = HEADER_PREFIX + DB::to_binary_key(bid.data, sizeof(bid.data)) + HEADER_SUFFIX; + auto key = HEADER_PREFIX + DB::to_binary_key(bbid.data, sizeof(bbid.data)) + HEADER_SUFFIX; if (!m_db.get(key, rb)) - return false; - Hash bbid = bid; // next line can modify bid, because it can be reference to header.previous_block_hash - seria::from_binary(*header, rb); - header_cache.insert(std::make_pair(bbid, *header)); - return true; + return nullptr; + api::BlockHeader header; + seria::from_binary(header, rb); + cit = m_header_cache.insert(std::make_pair(bbid, std::move(header))).first; + return &cit->second; } -void BlockChain::fix_block_sizes(api::BlockHeader *info) const { - DB::Value binary_block; - auto key = BLOCK_PREFIX + DB::to_binary_key(info->hash.data, sizeof(info->hash.data)) + BLOCK_SUFFIX; - invariant(m_db.get(key, binary_block), "Block must be there"); - common::MemoryInputStream stream(binary_block.data(), binary_block.size()); - seria::BinaryInputStream ba_stream(stream); - uint32_t raw_block_block_size = 0; - ser(raw_block_block_size, ba_stream); - auto coinbase_size = info->block_size - info->transactions_size; - info->transactions_size = info->block_size; - info->block_size = info->transactions_size + raw_block_block_size - coinbase_size; +bool BlockChain::get_header(const Hash &bid, api::BlockHeader *header, Height hint) const { + const api::BlockHeader *result = read_header_fast(bid, hint); + if (result) + *header = *result; + return result != nullptr; } api::BlockHeader BlockChain::read_header(const Hash &bid, Height hint) const { api::BlockHeader result; - invariant(read_header(bid, &result, hint), "Expected header was not found" + common::pod_to_hex(bid)); + invariant(get_header(bid, &result, hint), "Expected header was not found" + common::pod_to_hex(bid)); return result; } @@ -659,34 +589,35 @@ const api::BlockHeader &BlockChain::get_tip() const { return m_header_tip_window.back(); } -std::vector BlockChain::get_tip_segment( - const api::BlockHeader &prev_info, Height window, bool add_genesis) const { - std::vector result; - result.reserve(window); +void BlockChain::for_each_reversed_tip_segment(const api::BlockHeader &prev_info, Height window, bool add_genesis, + std::function &&fun) const { if (prev_info.height == Height(-1)) - return result; - api::BlockHeader pi = prev_info; - while (result.size() < window && pi.height != 0) { - result.push_back(pi); - pi = read_header(pi.previous_block_hash, pi.height - 1); + return; + const api::BlockHeader *header = &prev_info; + size_t count = 0; + while (count < window && header->height != 0) { + fun(*header); + count += 1; + header = read_header_fast(header->previous_block_hash, header->height - 1); + invariant(header, ""); } - if (result.size() < window && add_genesis) { - invariant(pi.height == 0, "Invariant dead - window size not reached, but genesis not found in get_tip_segment"); - result.push_back(pi); + if (count < window && add_genesis) { + invariant( + header->height == 0, "Invariant dead - window size not reached, but genesis not found in get_tip_segment"); + fun(*header); + count += 1; } - std::reverse(result.begin(), result.end()); - return result; } -void BlockChain::read_tip() { - DB::Cursor cur2 = m_db.rbegin(TIP_CHAIN_PREFIX); - m_tip_height = cur2.end() ? -1 : common::integer_cast(common::read_varint_sqlite4(cur2.get_suffix())); - seria::from_binary(m_tip_bid, cur2.get_value_array()); - api::BlockHeader tip_header = read_header(m_tip_bid); - m_tip_cumulative_difficulty = tip_header.cumulative_difficulty; - m_header_tip_window.clear(); - m_header_tip_window.push_back(tip_header); -} +// std::vector BlockChain::get_tip_segment( +// const api::BlockHeader &prev_info, Height window, bool add_genesis) const { +// std::vector result; +// result.reserve(window); +// for_each_reversed_tip_segment( +// prev_info, window, add_genesis, [&](const api::BlockHeader &header) { result.push_back(header); }); +// std::reverse(result.begin(), result.end()); +// return result; +//} void BlockChain::push_chain(const api::BlockHeader &header) { m_tip_height += 1; @@ -707,7 +638,7 @@ void BlockChain::pop_chain(const Hash &new_tip_bid) { m_tip_height -= 1; m_tip_bid = new_tip_bid; invariant(read_chain(m_tip_height) == m_tip_bid, - "After undo tip does not match read_chain " + common::pod_to_hex(m_tip_bid)); + "After undo tip does not match get_chain " + common::pod_to_hex(m_tip_bid)); if (m_header_tip_window.empty()) { api::BlockHeader tip_header = read_header(m_tip_bid); m_header_tip_window.push_back(tip_header); @@ -716,7 +647,7 @@ void BlockChain::pop_chain(const Hash &new_tip_bid) { } // After upgrading to future versions, remove version from index key -bool BlockChain::read_chain(uint32_t height, Hash *bid) const { +bool BlockChain::get_chain(Height height, Hash *bid) const { BinaryArray ba; if (!m_db.get(TIP_CHAIN_PREFIX + common::write_varint_sqlite4(height), ba)) return false; @@ -726,12 +657,16 @@ bool BlockChain::read_chain(uint32_t height, Hash *bid) const { bool BlockChain::in_chain(Height height, Hash bid) const { Hash ha; - return read_chain(height, &ha) && ha == bid; + return get_chain(height, &ha) && ha == bid; +} +bool BlockChain::in_chain(Hash bid) const { + auto header = read_header_fast(bid, 0); + return header ? in_chain(header->height, bid) : false; } -Hash BlockChain::read_chain(uint32_t height) const { +Hash BlockChain::read_chain(Height height) const { Hash ha; - invariant(read_chain(height, &ha), "read_header_chain failed"); + invariant(get_chain(height, &ha), "read_header_chain failed"); return ha; } @@ -752,7 +687,7 @@ void BlockChain::modify_children_counter(CumulativeDifficulty cd, const Hash &bi auto key = CHILDREN_PREFIX + DB::to_binary_key(bid.data, sizeof(bid.data)); auto cd_key = CD_TIPS_PREFIX + common::write_varint_sqlite4(cd.hi) + common::write_varint_sqlite4(cd.lo) + DB::to_binary_key(bid.data, sizeof(bid.data)); - uint32_t counter = 1; // default is 1 when not stored in db + size_t counter = 1; // default is 1 when not stored in db BinaryArray rb; if (m_db.get(key, rb)) seria::from_binary(counter, rb); @@ -784,7 +719,7 @@ bool BlockChain::get_oldest_tip(CumulativeDifficulty *cd, Hash *bid) const { return true; } -void BlockChain::for_each_tip(std::function fun) const { +void BlockChain::for_each_tip(std::function &&fun) const { for (DB::Cursor cur = m_db.rbegin(CD_TIPS_PREFIX); !cur.end(); cur.next()) { const std::string &suf = cur.get_suffix(); const char *be = suf.data(); @@ -809,14 +744,14 @@ bool BlockChain::prune_branch(CumulativeDifficulty cd, Hash bid) { for (auto cit = checkpoints.begin(); cit != checkpoints.end(); ++cit) if (cit->is_enabled() && cit->hash == bid) return false; - auto bit = blods.find(bid); - if (bit != blods.end()) { + auto bit = m_blods.find(bid); + if (bit != m_blods.end()) { if (!bit->second.children.empty() || !bit->second.parent) return false; auto rit = std::remove(bit->second.parent->children.begin(), bit->second.parent->children.end(), &bit->second); invariant(rit != bit->second.parent->children.end(), ""); bit->second.parent->children.erase(rit, bit->second.parent->children.end()); - blods.erase(bit); + m_blods.erase(bit); } api::BlockHeader me = read_header(bid); api::BlockHeader pa = read_header(me.previous_block_hash); @@ -829,7 +764,7 @@ bool BlockChain::prune_branch(CumulativeDifficulty cd, Hash bid) { m_header_tip_window.clear(); api::BlockHeader tip_header = read_header(m_tip_bid); m_header_tip_window.push_back(tip_header); - header_cache.erase(bid); + m_header_cache.erase(bid); return true; } @@ -849,7 +784,7 @@ void BlockChain::test_print_tips() const { for (DB::Cursor cur = m_db.begin(CHILDREN_PREFIX); !cur.end(); cur.next()) { Hash bid; DB::from_binary_key(cur.get_suffix(), 0, bid.data, sizeof(bid.data)); - uint32_t counter = 1; + size_t counter = 1; seria::from_binary(counter, cur.get_value_array()); api::BlockHeader info = read_header(bid); std::cout << "branch height=" << info.height << " bid=" << bid << " children=" << counter << std::endl; @@ -870,7 +805,7 @@ void BlockChain::test_print_structure(Height n_confirmations) const { for (DB::Cursor cur = m_db.begin(CHILDREN_PREFIX); !cur.end(); cur.next()) { Hash bid; DB::from_binary_key(cur.get_suffix(), 0, bid.data, sizeof(bid.data)); - uint32_t counter = 1; + size_t counter = 1; seria::from_binary(counter, cur.get_value_array()); std::cout << "children=" << counter << " bid=" << bid << std::endl; @@ -893,10 +828,10 @@ void BlockChain::test_print_structure(Height n_confirmations) const { std::cout << " sideblock height=" << header.height << " depth=" << t_height - header.height << (confirmed ? " (confirmed)" : "") << " bid=" << bid << std::endl; RawBlock rb; - Block block; if (confirmed) { total_forked_blocks += 1; - if (read_block(bid, &rb) && block.from_raw_block(rb)) { + if (get_block(bid, &rb)) { + Block block(rb); for (size_t tx_pos = 0; tx_pos != block.header.transaction_hashes.size(); ++tx_pos) { Hash tid = block.header.transaction_hashes.at(tx_pos); total_forked_transactions += 1; @@ -904,11 +839,11 @@ void BlockChain::test_print_structure(Height n_confirmations) const { Height height = 0; Hash block_hash; size_t index_in_block = 0; - if (!read_transaction(tid, &binary_tx, &height, &block_hash, &index_in_block)) { + if (!get_transaction(tid, &binary_tx, &height, &block_hash, &index_in_block)) { Amount input_amount = 0; for (const auto &input : block.transactions.at(tx_pos).inputs) - if (input.type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(input); + if (input.type() == typeid(InputKey)) { + const InputKey &in = boost::get(input); input_amount += in.amount; } total_possible_ds_transactions += 1; @@ -932,42 +867,6 @@ void BlockChain::test_print_structure(Height n_confirmations) const { << total_possible_ds_transactions << " total amount=" << total_possible_ds_amount << std::endl; } -void BlockChain::upgrade_5_to_6() { - m_log(logging::INFO) << "Blockchain database needs to recalculate block sizes..." << std::endl; - size_t processed = 0; - std::set hashes; - DB::Cursor cur2 = m_db.rbegin(TIP_CHAIN_PREFIX); - Height tip_height = cur2.end() ? -1 : common::integer_cast(common::read_varint_sqlite4(cur2.get_suffix())); - for (DB::Cursor cur = m_db.rbegin(BLOCK_PREFIX); !cur.end(); cur.next()) { - if (cur.get_suffix().substr(cur.get_suffix().size() - HEADER_SUFFIX.size()) != HEADER_SUFFIX) - continue; - if (processed++ % 50000 == 0) - m_log(logging::INFO) << "Processed " << processed << "/" << tip_height * 11 / 10 << " blocks" << std::endl; - Hash bid; - DB::from_binary_key(cur.get_suffix(), 0, bid.data, sizeof(bid.data)); - api::BlockHeader header; - seria::from_binary(header, cur.get_value_array()); - invariant(bid == header.hash, "DB corrupted"); - invariant(hashes.insert(bid).second, ""); - // RawBlock rb; - // invariant(read_block(bid, &rb), "Block must be there"); - DB::Value binary_block; - auto key = BLOCK_PREFIX + DB::to_binary_key(bid.data, sizeof(bid.data)) + BLOCK_SUFFIX; - invariant(m_db.get(key, binary_block), "Block must be there"); - common::MemoryInputStream stream(binary_block.data(), binary_block.size()); - seria::BinaryInputStream ba_stream(stream); - uint32_t raw_block_block_size = 0; - ser(raw_block_block_size, ba_stream); - auto coinbase_size = header.block_size - header.transactions_size; - header.transactions_size = header.block_size; - header.block_size = header.transactions_size + raw_block_block_size - coinbase_size; - key = HEADER_PREFIX + DB::to_binary_key(bid.data, sizeof(bid.data)) + HEADER_SUFFIX; - BinaryArray ba = seria::to_binary(header); - m_db.put(key, ba, false); - } - m_log(logging::INFO) << "Recalulating block sizes finished" << std::endl; -} - void BlockChain::start_internal_import() { m_log(logging::INFO) << "Blockchain database has old format, preparing for internal block import..." << std::endl; if (m_internal_import_chain.empty()) { @@ -1006,7 +905,7 @@ void BlockChain::start_internal_import() { continue; // block in main chain } BinaryArray block_data; - if (read_block(bid, &block_data, nullptr)) + if (get_block(bid, &block_data, nullptr)) m_archive.add(Archive::BLOCK, block_data, bid, "start_internal_import"); } cur.erase(); @@ -1024,19 +923,19 @@ bool BlockChain::internal_import() { break; const Hash bid = m_internal_import_chain.at(get_tip_height() + 1); RawBlock rb; - if (!read_block(bid, &rb)) { + if (!get_block(bid, &rb)) { m_log(logging::WARNING) << "Block not found during internal import for height=" << get_tip_height() + 1 << " bid=" << bid << std::endl; break; } PreparedBlock pb(std::move(rb), m_currency, nullptr); api::BlockHeader info; - if (add_block(pb, &info, "internal_import") != BroadcastAction::BROADCAST_ALL) { - m_log(logging::WARNING) << "Block corrupted during internal import for height=" << get_tip_height() + 1 + if (!add_block(pb, &info, "internal_import")) { + m_log(logging::WARNING) << "Block corrupted during internal import for height=" << get_tip_height() + 1 << " bid=" << bid << std::endl; break; } - // if (get_tip_height() % COMMIT_EVERY_N_BLOCKS == 0) + // if (get_tip_height() % m_config.db_commit_every_n_blocks == 0) // db_commit(); auto idea_ms = std::chrono::duration_cast( std::chrono::high_resolution_clock::now() - idea_start); @@ -1055,17 +954,17 @@ bool BlockChain::internal_import() { } void BlockChain::test_undo_everything(Height new_tip_height) { - while (get_tip_height() > new_tip_height) { + while (true) { // get_tip_height() > new_tip_height RawBlock raw_block; - Block block; - if (!read_block(get_tip_bid(), &raw_block) || !block.from_raw_block(raw_block)) + if (!get_block(get_tip_bid(), &raw_block)) break; + Block block(raw_block); undo_block(get_tip_bid(), raw_block, block, m_tip_height); if (get_tip_bid() == m_genesis_bid) break; pop_chain(block.header.previous_block_hash); tip_changed(); - if (get_tip_height() % COMMIT_EVERY_N_BLOCKS == 1) + if (get_tip_height() % m_config.db_commit_every_n_blocks == 1) db_commit(); } std::cout << "---- After undo everything ---- " << std::endl; @@ -1083,8 +982,6 @@ void BlockChain::test_undo_everything(Height new_tip_height) { continue; if (cur.get_suffix().find(HEADER_PREFIX) == 0) continue; - if (cur.get_suffix().find("f") == 0) // TODO - we have no index starting with "f" - continue; std::cout << DB::clean_key(cur.get_suffix()) << std::endl; if (counter++ > 1000) break; @@ -1110,7 +1007,7 @@ std::vector BlockChain::get_stable_checkpoints() const { } bool BlockChain::add_checkpoint(const SignedCheckpoint &checkpoint, const std::string &source_address) { - if (checkpoint.height <= m_currency.last_sw_checkpoint().height && checkpoint.is_enabled()) + if (checkpoint.height <= m_currency.last_hard_checkpoint().height && checkpoint.is_enabled()) return false; // Height is ignored when disabling key_id PublicKey public_key = m_currency.get_checkpoint_public_key(checkpoint.key_id); // returns empty key if out of range @@ -1139,8 +1036,8 @@ bool BlockChain::add_checkpoint(const SignedCheckpoint &checkpoint, const std::s m_archive.add(Archive::CHECKPOINT, binary_checkpoint, crypto::cn_fast_hash(binary_checkpoint.data(), binary_checkpoint.size()), source_address); if (checkpoint.is_enabled()) { - auto bit = blods.find(checkpoint.hash); - if (bit == blods.end()) + auto bit = m_blods.find(checkpoint.hash); + if (bit == m_blods.end()) return true; // orphan checkpoint } m_db.put(key_stable, binary_checkpoint, false); @@ -1152,7 +1049,7 @@ bool BlockChain::add_checkpoint(const SignedCheckpoint &checkpoint, const std::s return true; api::BlockHeader header = read_header(bid); RawBlock raw_block; - if (!read_block(bid, &raw_block)) + if (!get_block(bid, &raw_block)) return true; PreparedBlock pb(std::move(raw_block), m_currency, nullptr); reorganize_blocks(bid, pb, header); @@ -1176,22 +1073,21 @@ int BlockChain::compare( } bool BlockChain::add_blod_impl(const api::BlockHeader &header) { - auto bit = blods.find(header.hash); - if (bit != blods.end()) + auto bit = m_blods.find(header.hash); + if (bit != m_blods.end()) return true; // Strange, but nop - bit = blods.find(header.previous_block_hash); - if (bit == blods.end()) + bit = m_blods.find(header.previous_block_hash); + if (bit == m_blods.end()) return false; - Blod &blod = blods[header.hash]; + Blod &blod = m_blods[header.hash]; blod.height = header.height; blod.hash = header.hash; blod.parent = &bit->second; bit->second.children.push_back(&blod); blod.checkpoint_difficulty = blod.parent->checkpoint_difficulty; - if (m_currency.upgrade_from_major_version) { - blod.vote_for_upgrade = uint8_t(header.major_version == m_currency.upgrade_from_major_version && - header.minor_version == m_currency.upgrade_indicator_minor_version); + if (m_currency.upgrade_desired_major_version) { + blod.vote_for_upgrade = uint8_t(m_currency.is_upgrade_vote(header.major_version, header.minor_version)); blod.upgrade_decided_height = blod.parent->upgrade_decided_height; if (!blod.upgrade_decided_height) { blod.votes_for_upgrade_in_voting_window = blod.parent->votes_for_upgrade_in_voting_window; @@ -1199,14 +1095,13 @@ bool BlockChain::add_blod_impl(const api::BlockHeader &header) { if (blod.votes_for_upgrade_in_voting_window.size() > m_currency.upgrade_voting_window) blod.votes_for_upgrade_in_voting_window.pop_front(); invariant(blod.votes_for_upgrade_in_voting_window.size() <= m_currency.upgrade_voting_window, ""); - auto count = std::count(blod.votes_for_upgrade_in_voting_window.begin(), + size_t count = std::count(blod.votes_for_upgrade_in_voting_window.begin(), blod.votes_for_upgrade_in_voting_window.end(), uint8_t(1)); // if(count != 0) // std::cout << "Votes for version=" << int(m_currency.upgrade_desired_major_version) << " - // height=" - //<< header.height << " votes=" << count << std::endl; - if (count >= m_currency.upgrade_votes_required) { - blod.upgrade_decided_height = blod.height + m_currency.upgrade_blocks_after_voting; + // height=" << header.height << " votes=" << count << std::endl; + if (count >= m_currency.upgrade_votes_required()) { + blod.upgrade_decided_height = blod.height + m_currency.upgrade_window; m_log(logging::INFO) << "Consensus upgrade decided on height=" << header.height << " decided_height=" << blod.upgrade_decided_height << " bid=" << header.hash << std::endl; @@ -1218,7 +1113,7 @@ bool BlockChain::add_blod_impl(const api::BlockHeader &header) { } bool BlockChain::add_blod(const api::BlockHeader &header) { - if (blods.empty()) // Allow any blocks if main does not pass through last sw checkpoint yet + if (m_blods.empty()) // Allow any blocks if main does not pass through last sw checkpoint yet return true; add_blod_impl(header); // We inherit from parent and rebuild only if we pass through one of checlpoints @@ -1236,13 +1131,13 @@ bool BlockChain::add_blod(const api::BlockHeader &header) { } void BlockChain::build_blods() { - if (!blods.empty()) + if (!m_blods.empty()) return; // build only once per daemon launch - api::BlockHeader last_sw_checkpoint_header; - if (!read_header(m_currency.last_sw_checkpoint().hash, &last_sw_checkpoint_header)) + api::BlockHeader last_hard_checkpoint_header; + if (!get_header(m_currency.last_hard_checkpoint().hash, &last_hard_checkpoint_header)) return; - std::set bad_header_hashes; // sidechains that do not pass through last SW checkpoint - std::set good_header_hashes; // sidechains that pass through last SW checkpoint + std::set bad_header_hashes; // sidechains that do not pass through last hard checkpoint + std::set good_header_hashes; // sidechains that pass through last hard checkpoint std::vector good_headers; for_each_tip([&](CumulativeDifficulty cd, Hash tip_bid) -> bool { std::vector side_chain; @@ -1261,11 +1156,11 @@ void BlockChain::build_blods() { bad_header_hashes.insert(ha.hash); break; } - if (header.height < m_currency.last_sw_checkpoint().height) + if (header.height < m_currency.last_hard_checkpoint().height) break; side_chain.push_back(header); - if (header.height == m_currency.last_sw_checkpoint().height) { - if (header.hash == m_currency.last_sw_checkpoint().hash) { + if (header.height == m_currency.last_hard_checkpoint().height) { + if (header.hash == m_currency.last_hard_checkpoint().hash) { std::reverse(side_chain.begin(), side_chain.end()); for (auto &&ha : side_chain) { good_header_hashes.insert(ha.hash); @@ -1283,88 +1178,88 @@ void BlockChain::build_blods() { }); for (auto &&ha : good_headers) { // They are conveniently sorted parent to child and can be applied sequentially if (!add_blod_impl(ha)) { - invariant(ha.hash == m_currency.last_sw_checkpoint().hash, ""); - Blod &blod = blods[ha.hash]; + invariant(ha.hash == m_currency.last_hard_checkpoint().hash, ""); + Blod &blod = m_blods[ha.hash]; blod.height = ha.height; blod.hash = ha.hash; - if (m_currency.upgrade_from_major_version) { - blod.vote_for_upgrade = uint8_t(ha.major_version == m_currency.upgrade_from_major_version && - ha.minor_version == m_currency.upgrade_indicator_minor_version); + if (m_currency.upgrade_desired_major_version) { + blod.vote_for_upgrade = uint8_t(m_currency.is_upgrade_vote(ha.major_version, ha.minor_version)); Height first_height = - m_currency.last_sw_checkpoint().height < m_currency.upgrade_voting_window - 1 + m_currency.last_hard_checkpoint().height < m_currency.upgrade_voting_window - 1 ? 0 - : m_currency.last_sw_checkpoint().height - (m_currency.upgrade_voting_window - 1); - for (Height h = first_height; h != m_currency.last_sw_checkpoint().height; ++h) { + : m_currency.last_hard_checkpoint().height - (m_currency.upgrade_voting_window - 1); + for (Height h = first_height; h != m_currency.last_hard_checkpoint().height; ++h) { auto header = read_header(read_chain(h), h); - auto vote = uint8_t(header.major_version == m_currency.upgrade_from_major_version && - header.minor_version == m_currency.upgrade_indicator_minor_version); + auto vote = uint8_t(m_currency.is_upgrade_vote(header.major_version, header.minor_version)); blod.votes_for_upgrade_in_voting_window.push_back(vote); } blod.votes_for_upgrade_in_voting_window.push_back(blod.vote_for_upgrade); - auto count = std::count(blod.votes_for_upgrade_in_voting_window.begin(), + size_t count = std::count(blod.votes_for_upgrade_in_voting_window.begin(), blod.votes_for_upgrade_in_voting_window.end(), uint8_t(1)); invariant( - count < m_currency.upgrade_votes_required, "Upgrade happened before or on last SW checkpoint"); + count < m_currency.upgrade_votes_required(), "Upgrade happened before or on last hard checkpoint"); } } } update_key_count_max_heights(); } -void BlockChain::fill_statistics(api::bytecoind::GetStatistics::Response &res) const { +void BlockChain::fill_statistics(api::cnd::GetStatistics::Response &res) const { res.checkpoints = get_latest_checkpoints(); - if (!m_currency.upgrade_from_major_version) + if (!m_currency.upgrade_desired_major_version) return; - auto bit = blods.find(get_tip_bid()); - if (bit == blods.end()) + auto bit = m_blods.find(get_tip_bid()); + if (bit == m_blods.end()) return; - res.upgrade_decided_height = bit->second.upgrade_decided_height; - auto count = std::count(bit->second.votes_for_upgrade_in_voting_window.begin(), - bit->second.votes_for_upgrade_in_voting_window.end(), uint8_t(1)); + res.upgrade_decided_height = bit->second.upgrade_decided_height; + auto count = std::count(bit->second.votes_for_upgrade_in_voting_window.begin(), + bit->second.votes_for_upgrade_in_voting_window.end(), uint8_t(1)); res.upgrade_votes_in_top_block = static_cast(count); } bool BlockChain::fill_next_block_versions( - const api::BlockHeader &prev_info, bool cooperative, uint8_t *major, uint8_t *minor) const { - *major = m_currency.get_block_major_version_for_height(prev_info.height + 1); - if (*major == m_currency.upgrade_from_major_version) + const api::BlockHeader &prev_info, bool cooperative, uint8_t *major_mm, uint8_t *major_cm, uint8_t *minor) const { + *major_cm = *major_mm = m_currency.get_block_major_version_for_height(prev_info.height + 1); + if (*major_cm >= m_currency.amethyst_block_version) + *major_cm += 1; + *minor = 0; + if (*major_mm == m_currency.upgrade_from_major_version) *minor = m_currency.upgrade_indicator_minor_version; - else - *minor = 0; - // TODO - for now we only observe voting, do not create new blocks yet - // if( !m_currency.upgrade_desired_major_version ) - // return true; - // if(blods.empty()) - // return true; - // auto bit = blods.find(prev_info.hash); - // if (bit == blods.end()) - // return false; - // if(!bit->second.upgrade_decided_height || prev_info.height + 1 < bit->second.upgrade_decided_height) - // return true; - // *major = m_currency.upgrade_desired_major_version; - // *minor = 0; + if (!m_currency.upgrade_desired_major_version) + return true; + if (m_blods.empty()) + return true; + auto bit = m_blods.find(prev_info.hash); + if (bit == m_blods.end()) + return false; + if (!bit->second.upgrade_decided_height || prev_info.height + 1 < bit->second.upgrade_decided_height) + return true; + *major_cm = *major_mm = m_currency.upgrade_desired_major_version; + if (*major_cm >= m_currency.amethyst_block_version) + *major_cm += 1; + *minor = 0; return true; } void BlockChain::update_key_count_max_heights() { // We use simplest O(n) algo, will optimize later and use this one as a reference - for (auto &&bit : blods) { + for (auto &&bit : m_blods) { bit.second.checkpoint_key_ids.reset(); bit.second.checkpoint_difficulty = CheckpointDifficulty{}; } auto checkpoints = get_stable_checkpoints(); for (const auto &cit : checkpoints) if (cit.is_enabled()) { - auto bit = blods.find(cit.hash); - if (bit == blods.end()) + auto bit = m_blods.find(cit.hash); + if (bit == m_blods.end()) continue; for (Blod *b = &bit->second; b; b = b->parent) b->checkpoint_key_ids.set(cit.key_id); } std::vector to_visit; - auto bit = blods.find(m_currency.last_sw_checkpoint().hash); - if (bit != blods.end()) + auto bit = m_blods.find(m_currency.last_hard_checkpoint().hash); + if (bit != m_blods.end()) to_visit.push_back(&bit->second); while (!to_visit.empty()) { Blod *blod = to_visit.back(); @@ -1379,8 +1274,8 @@ void BlockChain::update_key_count_max_heights() { } BlockChain::CheckpointDifficulty BlockChain::get_checkpoint_difficulty(Hash hash) const { - auto bit = blods.find(hash); - if (bit == blods.end()) + auto bit = m_blods.find(hash); + if (bit == m_blods.end()) return CheckpointDifficulty{}; return bit->second.checkpoint_difficulty; } diff --git a/src/Core/BlockChain.hpp b/src/Core/BlockChain.hpp index 5dbd73e8..5b263551 100644 --- a/src/Core/BlockChain.hpp +++ b/src/Core/BlockChain.hpp @@ -10,24 +10,41 @@ #include "CryptoNote.hpp" #include "logging/LoggerMessage.hpp" #include "platform/DB.hpp" -#include "platform/ExclusiveLock.hpp" #include "rpc_api.hpp" namespace crypto { class CryptoNightContext; } -namespace bytecoin { +namespace cn { class Config; class Currency; -enum class BroadcastAction { BROADCAST_ALL, NOTHING, BAN }; -enum class AddTransactionResult { - BAN, - BROADCAST_ALL, - ALREADY_IN_POOL, - INCREASE_FEE, - FAILED_TO_REDO, - OUTPUT_ALREADY_SPENT +class ConsensusError : public std::runtime_error { +public: + ConsensusError(const std::string &str) : runtime_error(str) {} +}; + +class ConsensusErrorOutputDoesNotExist : public ConsensusError { +public: + size_t input_index = 0; + size_t output_index = 0; + ConsensusErrorOutputDoesNotExist(const std::string &str, size_t input_index, size_t output_index) + : ConsensusError(str), input_index(input_index), output_index(output_index) {} +}; + +class ConsensusErrorBadOutputOrSignature : public ConsensusError { +public: + Height conflict_height = 0; + ConsensusErrorBadOutputOrSignature(const std::string &str, Height conflict_height) + : ConsensusError(str), conflict_height(conflict_height) {} +}; + +class ConsensusErrorOutputSpent : public ConsensusError { +public: + KeyImage key_image; + Height conflict_height = 0; + ConsensusErrorOutputSpent(const std::string &str, const KeyImage &key_image, Height conflict_height) + : ConsensusError(str), key_image(key_image), conflict_height(conflict_height) {} }; struct PreparedBlock { @@ -37,14 +54,18 @@ struct PreparedBlock { Hash bid; Hash base_transaction_hash; size_t coinbase_tx_size = 0; + size_t block_header_size = 0; size_t parent_block_size = 0; - Hash long_block_hash; // only if context != nullptr - std::string error_text; // empty when no error + Hash long_block_hash; // only if passed context != nullptr + boost::optional error; explicit PreparedBlock(BinaryArray &&ba, const Currency ¤cy, crypto::CryptoNightContext *context); explicit PreparedBlock( RawBlock &&rba, const Currency ¤cy, crypto::CryptoNightContext *context); // we get raw blocks from p2p PreparedBlock() = default; + +private: + void prepare(const Currency ¤cy, crypto::CryptoNightContext *context); }; class BlockChain { @@ -66,30 +87,30 @@ class BlockChain { template void get_txs(const std::vector &, T &) const; - std::vector get_tip_segment( - const api::BlockHeader &prev_info, Height window, bool add_genesis) const; + void for_each_reversed_tip_segment(const api::BlockHeader &prev_info, Height window, bool add_genesis, + std::function &&fun) const; - bool read_chain(Height height, Hash *bid) const; + bool get_chain(Height height, Hash *bid) const; bool in_chain(Height height, Hash bid) const; - bool read_block(const Hash &bid, RawBlock *rb) const; - bool read_block(const Hash &bid, BinaryArray *block_data, RawBlock *rb) const; // rb can be null here - bool has_block(const Hash &bid) const; - bool read_header(const Hash &bid, api::BlockHeader *info, Height hint = 0) const; - void fix_block_sizes(api::BlockHeader *info) const; // TODO - remove after correct sizes are in DB - bool read_transaction( + bool in_chain(Hash bid) const; + bool get_block(const Hash &bid, RawBlock *rb) const; + bool get_block(const Hash &bid, BinaryArray *block_data, RawBlock *rb) const; // rb can be null here + bool has_header(const Hash &bid) const { return read_header_fast(bid, 0) != nullptr; } + bool get_header(const Hash &bid, api::BlockHeader *info, Height hint = 0) const; + bool get_transaction( const Hash &tid, BinaryArray *binary_tx, Height *block_height, Hash *block_hash, size_t *index_in_block) const; + bool has_transaction(const Hash &tid) const; // Modify blockchain state. bytecoin header does not contain enough info for consensus calcs, so we cannot have // header chain without block chain - BroadcastAction add_block(const PreparedBlock &pb, api::BlockHeader *info, const std::string &source_address); + bool add_block(const PreparedBlock &pb, api::BlockHeader *info, const std::string &source_address); // Facilitate sync and download - std::vector get_sparse_chain() const; - std::vector get_sparse_chain(Hash start, Hash end) const; - std::vector get_sync_headers(const std::vector &sparse_chain, size_t max_count) const; - std::vector get_sync_headers_chain( - const std::vector &sparse_chain, Height *start_height, size_t max_count) const; + std::vector get_sparse_chain(Height max_jump = std::numeric_limits::max()) const; + std::vector get_sparse_chain( + Hash start, Hash end, Height max_jump = std::numeric_limits::max()) const; + std::vector get_sync_headers_chain(const std::vector &sparse_chain, Height *start_height, + size_t max_count) const; // throws if no common blocks - Height find_blockchain_supplement(const std::vector &remote_block_ids) const; Height get_timestamp_lower_bound_height(Timestamp) const; void test_undo_everything(Height new_tip_height); @@ -106,23 +127,25 @@ class BlockChain { std::vector get_stable_checkpoints() const; bool add_checkpoint(const SignedCheckpoint &checkpoint, const std::string &source_address); - void read_archive(api::bytecoind::GetArchive::Request &&req, api::bytecoind::GetArchive::Response &resp) { + void read_archive(api::cnd::GetArchive::Request &&req, api::cnd::GetArchive::Response &resp) { m_archive.read_archive(std::move(req), resp); } - virtual void fill_statistics(api::bytecoind::GetStatistics::Response &res) const; + virtual void fill_statistics(api::cnd::GetStatistics::Response &res) const; typedef std::array CheckpointDifficulty; // size must be == m_currency.get_checkpoint_keys_count() protected: + bool has_block(const Hash &bid) const; + std::vector m_internal_import_chain; void start_internal_import(); - void upgrade_5_to_6(); // We will perform it on next version change - virtual std::string check_standalone_consensus( + virtual void check_standalone_consensus( const PreparedBlock &pb, api::BlockHeader *info, const api::BlockHeader &prev_info, bool check_pow) const = 0; - virtual bool redo_block(const Hash &bhash, const Block &block, const api::BlockHeader &info) = 0; - virtual void undo_block(const Hash &bhash, const Block &block, Height height) = 0; - bool redo_block(const Hash &bhash, const BinaryArray &block_data, const RawBlock &raw_block, const Block &block, - const api::BlockHeader &info, const Hash &base_transaction_hash); + virtual void redo_block( + const Hash &bhash, const Block &block, const api::BlockHeader &info) = 0; // throws ConsensusError + virtual void undo_block(const Hash &bhash, const Block &block, Height height) = 0; + void redo_block(const Hash &bhash, const BinaryArray &block_data, const RawBlock &raw_block, const Block &block, + const api::BlockHeader &info, const Hash &base_transaction_hash); // throws ConsensusError void debug_check_transaction_invariants(const RawBlock &raw_block, const Block &block, const api::BlockHeader &info, const Hash &base_transaction_hash) const; void undo_block(const Hash &bhash, const RawBlock &raw_block, const Block &block, Height height); @@ -146,15 +169,15 @@ class BlockChain { private: Hash m_tip_bid; CumulativeDifficulty m_tip_cumulative_difficulty{}; - Height m_tip_height = -1; - void read_tip(); + Height m_tip_height = -1; // We use overflow to 0 to apply genesis block in constructor void push_chain(const api::BlockHeader &header); void pop_chain(const Hash &new_tip_bid); Hash read_chain(Height height) const; - mutable std::unordered_map header_cache; + mutable std::unordered_map m_header_cache; std::deque m_header_tip_window; // We cache recent headers for quick calculation in block windows + const api::BlockHeader *read_header_fast(const Hash &bid, Height hint) const; api::BlockHeader read_header(const Hash &bid, Height hint = 0) const; void store_block(const Hash &bid, const BinaryArray &block_data); @@ -169,7 +192,7 @@ class BlockChain { void modify_children_counter(CumulativeDifficulty cd, const Hash &bid, int delta); bool get_oldest_tip(CumulativeDifficulty *cd, Hash *bid) const; bool prune_branch(CumulativeDifficulty cd, Hash bid); - void for_each_tip(std::function fun) const; + void for_each_tip(std::function &&fun) const; static int compare( const CheckpointDifficulty &a, CumulativeDifficulty ca, const CheckpointDifficulty &b, CumulativeDifficulty cb); @@ -185,7 +208,7 @@ class BlockChain { std::deque votes_for_upgrade_in_voting_window; Height upgrade_decided_height = 0; }; - std::map blods; + std::map m_blods; void update_key_count_max_heights(); bool add_blod_impl(const api::BlockHeader &header); bool add_blod(const api::BlockHeader &header); @@ -193,8 +216,8 @@ class BlockChain { protected: void build_blods(); - bool fill_next_block_versions( - const api::BlockHeader &prev_info, bool cooperative, uint8_t *major, uint8_t *minor) const; + bool fill_next_block_versions(const api::BlockHeader &prev_info, bool cooperative, uint8_t *major_mm, + uint8_t *major_cm, uint8_t *minor) const; }; -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/BlockChainFileFormat.cpp b/src/Core/BlockChainFileFormat.cpp index 20093f10..95749969 100644 --- a/src/Core/BlockChainFileFormat.cpp +++ b/src/Core/BlockChainFileFormat.cpp @@ -2,7 +2,6 @@ // Licensed under the GNU Lesser General Public License. See LICENSE for details. #include "BlockChainFileFormat.hpp" -//#include "crypto/crypto-ops.h" #include "BlockChainState.hpp" #include "common/Math.hpp" #include "platform/PathTools.hpp" @@ -10,7 +9,7 @@ #include "seria/BinaryOutputStream.hpp" using namespace common; -using namespace bytecoin; +using namespace cn; // Example // LegacyBlockChainReader reader(import_path + "/blockindexes.bin", import_path + "/blocks.bin"); @@ -20,13 +19,15 @@ using namespace bytecoin; // std::cout << "Block tx count=" << pb.block.transactions.size() << std::endl; // } +const bool multicore = true; + LegacyBlockChainReader::LegacyBlockChainReader(const Currency ¤cy, const std::string &index_file_name, const std::string &item_file_name) - : currency(currency) { + : m_currency(currency) { try { - m_indexes_file = std::make_unique(index_file_name, platform::FileStream::READ_EXISTING); - m_items_file = std::make_unique(item_file_name, platform::FileStream::READ_EXISTING); + m_indexes_file = std::make_unique(index_file_name, platform::O_READ_EXISTING); + m_items_file = std::make_unique(item_file_name, platform::O_READ_EXISTING); m_indexes_file->seek(0, SEEK_END); uint64_t m_indexesFileSize = m_indexes_file->tellp(); @@ -41,12 +42,12 @@ LegacyBlockChainReader::LegacyBlockChainReader(const Currency ¤cy, LegacyBlockChainReader::~LegacyBlockChainReader() { { - std::unique_lock lock(mu); - quit = true; - have_work.notify_all(); + std::unique_lock lock(m_mu); + m_quit = true; + m_have_work.notify_all(); } - if (th.joinable()) - th.join(); + if (m_th.joinable()) + m_th.join(); } void LegacyBlockChainReader::load_offsets() { @@ -71,64 +72,74 @@ void LegacyBlockChainReader::load_offsets() { BinaryArray LegacyBlockChainReader::get_block_data_by_index(Height i) { load_offsets(); - size_t si = common::integer_cast(m_offsets.at(i + 1) - m_offsets.at(i)); - m_items_file->seek(m_offsets.at(i), SEEK_SET); - BinaryArray data_cache(si); - m_items_file->read(reinterpret_cast(data_cache.data()), si); - return data_cache; + if (i + 1 >= m_offsets.size()) + return BinaryArray{}; + try { + size_t si = common::integer_cast(m_offsets.at(i + 1) - m_offsets.at(i)); + m_items_file->seek(m_offsets.at(i), SEEK_SET); + BinaryArray data_cache(si); + m_items_file->read(reinterpret_cast(data_cache.data()), si); + return data_cache; + } catch (const std::runtime_error &) { + return BinaryArray{}; + } } -const size_t MAX_PRELOAD_BLOCKS = 100; -const size_t MAX_PRELOAD_TOTAL_SIZE = 50 * 1024 * 1024; +const size_t MAX_PRELOAD_BLOCKS = 100; +// const size_t MAX_PRELOAD_TOTAL_SIZE = 50 * 1024 * 1024; void LegacyBlockChainReader::thread_run() { while (true) { Height to_load = 0; { - std::unique_lock lock(mu); - if (quit) + std::unique_lock lock(m_mu); + if (m_quit) return; - if (next_load_height == 0) - next_load_height = last_load_height; - if (next_load_height > last_load_height + MAX_PRELOAD_BLOCKS || - total_prepared_data_size > MAX_PRELOAD_TOTAL_SIZE) { - have_work.wait(lock); + if (m_blocks_to_load.empty()) { + m_have_work.wait(lock); continue; } - to_load = next_load_height++; + to_load = *m_blocks_to_load.begin(); } BinaryArray rba = get_block_data_by_index(to_load); - PreparedBlock pb(std::move(rba), currency, nullptr); + PreparedBlock pb(std::move(rba), m_currency, nullptr); { - std::unique_lock lock(mu); - total_prepared_data_size += pb.block_data.size(); - prepared_blocks[to_load] = std::move(pb); - prepared_blocks_ready.notify_all(); + std::unique_lock lock(m_mu); + m_blocks_to_load.erase(to_load); + m_total_prepared_data_size += pb.block_data.size(); + m_prepared_blocks[to_load] = std::move(pb); + m_prepared_blocks_ready.notify_all(); } } } -static size_t max_ps = 0; -PreparedBlock LegacyBlockChainReader::get_prepared_block_by_index(Height i) { +PreparedBlock LegacyBlockChainReader::get_prepared_block_by_index(Height height) { load_offsets(); + if (!multicore) { + BinaryArray rba = get_block_data_by_index(height); + PreparedBlock pb(std::move(rba), m_currency, nullptr); + return pb; + } { - std::unique_lock lock(mu); - if (!th.joinable()) - th = std::thread(&LegacyBlockChainReader::thread_run, this); - last_load_height = i; - have_work.notify_all(); + std::unique_lock lock(m_mu); + if (!m_th.joinable()) + m_th = std::thread(&LegacyBlockChainReader::thread_run, this); + for (Height i = height; i != height + MAX_PRELOAD_BLOCKS; ++i) { + if (m_prepared_blocks.count(i) == 0) + m_blocks_to_load.insert(i); + } + m_have_work.notify_all(); } while (true) { - std::unique_lock lock(mu); - auto pit = prepared_blocks.find(i); - if (pit == prepared_blocks.end()) { - prepared_blocks_ready.wait(lock); + std::unique_lock lock(m_mu); + auto pit = m_prepared_blocks.find(height); + if (pit == m_prepared_blocks.end()) { + m_prepared_blocks_ready.wait(lock); continue; } PreparedBlock result = std::move(pit->second); - pit = prepared_blocks.erase(pit); - max_ps = std::max(max_ps, total_prepared_data_size); - total_prepared_data_size -= result.block_data.size(); + pit = m_prepared_blocks.erase(pit); + m_total_prepared_data_size -= result.block_data.size(); return result; } } @@ -139,9 +150,9 @@ bool LegacyBlockChainReader::import_blocks(BlockChainState *block_chain) { // size_t bs_count = std::min(block_chain.get_tip_height() + 1 + count, get_block_count()); while (block_chain->get_tip_height() + 1 < get_block_count()) { BinaryArray rba = get_block_data_by_index(block_chain->get_tip_height() + 1); - PreparedBlock pb(std::move(rba), currency, nullptr); + PreparedBlock pb(std::move(rba), m_currency, nullptr); api::BlockHeader info; - if (block_chain->add_block(pb, &info, "blocks_file") != BroadcastAction::BROADCAST_ALL) { + if (!block_chain->add_block(pb, &info, "blocks_file")) { std::cout << "block_chain.add_block !BROADCAST_ALL block=" << block_chain->get_tip_height() + 1 << std::endl; block_chain->db_commit(); @@ -162,44 +173,53 @@ bool LegacyBlockChainReader::import_blocks(BlockChainState *block_chain) { return block_chain->get_tip_height() + 1 < get_block_count(); // Not finished } -bool LegacyBlockChainReader::import_blockchain2(const std::string &coin_folder, - BlockChainState *block_chain, - Height max_height) { - LegacyBlockChainReader reader( - block_chain->get_currency(), coin_folder + "/blockindexes.bin", coin_folder + "/blocks.bin"); - const size_t import_height = std::min(max_height, reader.get_block_count() + 1); - if (block_chain->get_tip_height() > import_height) { - // std::cout << "Skipping block chain import - we have more blocks than " - // "blocks.bin tip_height=" - // << block_chain->get_tip_height() << " bs_count=" << bs_count << std::endl; - return true; - } - std::cout << "Importing blocks up to height " << import_height << std::endl; - auto idea_start = std::chrono::high_resolution_clock::now(); - auto start_block = block_chain->get_tip_height(); - // api::BlockHeader prev_info; - while (block_chain->get_tip_height() < import_height) { - PreparedBlock pb = reader.get_prepared_block_by_index(block_chain->get_tip_height() + 1); - api::BlockHeader info; - if (block_chain->add_block(pb, &info, "blocks_file") != BroadcastAction::BROADCAST_ALL) { - std::cout << "block_chain.add_block !BROADCAST_ALL block=" << block_chain->get_tip_height() + 1 - << std::endl; - block_chain->db_commit(); - return false; +bool LegacyBlockChainReader::import_blockchain2(const std::string &index_file_name, const std::string &item_file_name, + BlockChainState *block_chain, Height max_height) { + auto idea_start = std::chrono::high_resolution_clock::now(); + Height start_block = 0; + try { + LegacyBlockChainReader reader(block_chain->get_currency(), index_file_name, item_file_name); + if (reader.get_block_count() == 0) + return true; + const size_t import_height = std::min(max_height, reader.get_block_count() - 1); + if (block_chain->get_tip_height() >= import_height) { + // std::cout << "Skipping block chain import - we have more blocks than " + // "blocks.bin tip_height=" + // << block_chain->get_tip_height() << " bs_count=" << bs_count << std::endl; + return true; } - // if (block_chain->get_tip_height() % 50000 == 0) - // block_chain->db_commit(); - // ts_file << info.timestamp << "\t" << info.timestamp_median << "\t" << - // info.timestamp_unlock << "\t" - // << int64_t(info.timestamp) - - // int64_t(prev_info.timestamp) << "\t" - // << int64_t(info.timestamp_median) - - // int64_t(prev_info.timestamp_median) << "\t" - // << int64_t(info.timestamp) - - // int64_t(info.timestamp_median) << std::endl; - // prev_info = info; - // if (block_chain->get_tip_height() == 1370000) // 1370000 - // break; + std::cout << "Importing blocks up to height " << import_height << std::endl; + start_block = block_chain->get_tip_height(); + // api::BlockHeader prev_info; + while (block_chain->get_tip_height() < import_height) { + PreparedBlock pb = reader.get_prepared_block_by_index(block_chain->get_tip_height() + 1); + api::BlockHeader info; + if (!block_chain->add_block(pb, &info, "blocks_file")) { + std::cout << "block_chain.add_block !BROADCAST_ALL block=" << block_chain->get_tip_height() + 1 + << std::endl; + block_chain->db_commit(); + return false; + } + // if (block_chain->get_tip_height() % 50000 == 0) + // block_chain->db_commit(); + // ts_file << info.timestamp << "\t" << info.timestamp_median << "\t" << + // info.timestamp_unlock << "\t" + // << int64_t(info.timestamp) - + // int64_t(prev_info.timestamp) << "\t" + // << int64_t(info.timestamp_median) - + // int64_t(prev_info.timestamp_median) << "\t" + // << int64_t(info.timestamp) - + // int64_t(info.timestamp_median) << std::endl; + // prev_info = info; + // if (block_chain->get_tip_height() == 1370000) // 1370000 + // break; + } + } catch (const std::exception &ex) { + std::cout << "Exception while importing blockchain file, what=" << common::what(ex) << std::endl; + return false; + } catch (...) { + std::cout << "Unknown exception while importing blockchain file" << std::endl; + return false; } block_chain->db_commit(); auto idea_ms = @@ -212,8 +232,8 @@ bool LegacyBlockChainReader::import_blockchain2(const std::string &coin_folder, LegacyBlockChainWriter::LegacyBlockChainWriter(const std::string &index_file_name, const std::string &item_file_name, uint64_t count) - : m_items_file(item_file_name, platform::FileStream::TRUNCATE_READ_WRITE) - , m_indexes_file(index_file_name, platform::FileStream::TRUNCATE_READ_WRITE) { + : m_items_file(item_file_name, platform::O_CREATE_ALWAYS) + , m_indexes_file(index_file_name, platform::O_CREATE_ALWAYS) { m_indexes_file.write(&count, sizeof(count)); } @@ -224,27 +244,28 @@ void LegacyBlockChainWriter::write_block(const RawBlock &raw_block) { m_indexes_file.write(&si, sizeof si); } -bool LegacyBlockChainWriter::export_blockchain2(const std::string &export_folder, const BlockChainState &block_chain) { +bool LegacyBlockChainWriter::export_blockchain2( + const std::string &index_file_name, const std::string &item_file_name, const BlockChainState &block_chain) { auto idea_start = std::chrono::high_resolution_clock::now(); std::cout << "Start exporting blocks" << std::endl; - LegacyBlockChainWriter writer( - export_folder + "/blockindexes.bin", export_folder + "/blocks.bin", block_chain.get_tip_height() + 1); + LegacyBlockChainWriter writer(index_file_name, item_file_name, block_chain.get_tip_height() + 1); for (Height ha = 0; ha != block_chain.get_tip_height() + 1; ++ha) { Hash bid{}; BinaryArray block_data; RawBlock raw_block; - if (!block_chain.read_chain(ha, &bid) || !block_chain.read_block(bid, &block_data, &raw_block)) - throw std::runtime_error("block_chain.read_block failed"); + if (!block_chain.get_chain(ha, &bid) || !block_chain.get_block(bid, &block_data, &raw_block)) + throw std::runtime_error("block_chain.get_block failed"); writer.write_block(raw_block); if (ha % 10000 == 0) std::cout << "Exporting block " << ha << "/" << block_chain.get_tip_height() << std::endl; - // Block block; - // if (!block.from_raw_block(raw_block)) - // throw std::runtime_error("from_raw_block failed"); + // Block block(raw_block); + // if(block.header.is_merge_mined() && block.header.root_block.base_transaction.version >= 2) + // std::cout << "Base transaction with strange version found, height=" << ha << " version=" << + // block.header.root_block.base_transaction.version << std::endl; // for (auto &&tr : block.transactions) { // for (auto &&input : tr.inputs) - // if (input.type() == typeid(KeyInput)) { - // const KeyInput &in = boost::get(input); + // if (input.type() == typeid(InputKey)) { + // const InputKey &in = boost::get(input); // } // } } diff --git a/src/Core/BlockChainFileFormat.hpp b/src/Core/BlockChainFileFormat.hpp index bb693712..86cdb6cb 100644 --- a/src/Core/BlockChainFileFormat.hpp +++ b/src/Core/BlockChainFileFormat.hpp @@ -12,28 +12,27 @@ #include "CryptoNote.hpp" #include "platform/Files.hpp" -namespace bytecoin { +namespace cn { class BlockChainState; -// TODO - convert all read/writes to little endian +// TODO - convert all read/writes to explicit little endian class LegacyBlockChainReader { - const Currency ¤cy; + const Currency &m_currency; std::unique_ptr m_items_file; std::unique_ptr m_indexes_file; Height m_count = 0; std::vector m_offsets; // we artifically add offset of the end of file void load_offsets(); - std::thread th; - std::mutex mu; - std::condition_variable have_work; - std::condition_variable prepared_blocks_ready; - bool quit = false; + std::thread m_th; + std::mutex m_mu; + std::condition_variable m_have_work; + std::condition_variable m_prepared_blocks_ready; + bool m_quit = false; - std::map prepared_blocks; - size_t total_prepared_data_size = 0; - Height last_load_height = 0; - Height next_load_height = 0; + std::map m_prepared_blocks; + std::set m_blocks_to_load; + size_t m_total_prepared_data_size = 0; void thread_run(); public: @@ -47,8 +46,8 @@ class LegacyBlockChainReader { bool import_blocks(BlockChainState *block_chain); // return false when no more blocks remain - static bool import_blockchain2(const std::string &coin_folder, BlockChainState *block_chain, - Height max_height = std::numeric_limits::max()); + static bool import_blockchain2(const std::string &index_file_name, const std::string &item_file_name, + BlockChainState *block_chain, Height max_height = std::numeric_limits::max()); }; class LegacyBlockChainWriter { @@ -59,7 +58,8 @@ class LegacyBlockChainWriter { LegacyBlockChainWriter(const std::string &index_file_name, const std::string &item_file_name, uint64_t count); void write_block(const RawBlock &raw_block); - static bool export_blockchain2(const std::string &export_folder, const BlockChainState &block_chain); + static bool export_blockchain2( + const std::string &index_file_name, const std::string &item_file_name, const BlockChainState &block_chain); }; -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/BlockChainState.cpp b/src/Core/BlockChainState.cpp index 4c49febe..6ec22c6e 100644 --- a/src/Core/BlockChainState.cpp +++ b/src/Core/BlockChainState.cpp @@ -4,6 +4,7 @@ #include "BlockChainState.hpp" #include #include +#include #include "Config.hpp" #include "CryptoNoteTools.hpp" #include "Currency.hpp" @@ -21,27 +22,34 @@ static const std::string AMOUNT_OUTPUT_PREFIX = "a"; static const std::string BLOCK_GLOBAL_INDICES_PREFIX = "b"; static const std::string BLOCK_GLOBAL_INDICES_SUFFIX = "g"; -static const std::string UNLOCK_BLOCK_PREFIX = "u"; -static const std::string UNLOCK_TIME_PREFIX = "U"; -// We store locked outputs in separate indexes +static const std::string DIN_PREFIX = "D"; -const size_t MAX_POOL_SIZE = 2000000; // ~1000 "normal" transactions with 10 inputs and 10 outputs +const int chain_reaction = 2; -using namespace bytecoin; +using namespace cn; using namespace platform; +typedef std::pair, std::vector> InputDesc; + namespace seria { void ser_members(IBlockChainState::UnlockTimePublickKeyHeightSpent &v, ISeria &s) { seria_kv("unlock_block_or_timestamp", v.unlock_block_or_timestamp, s); seria_kv("public_key", v.public_key, s); seria_kv("height", v.height, s); + seria_kv("auditable", v.auditable, s); seria_kv("spent", v.spent, s); + seria_kv("dins", v.dins, s); } } // namespace seria -BlockChainState::PoolTransaction::PoolTransaction( - const Transaction &tx, const BinaryArray &binary_tx, Amount fee, Timestamp timestamp) - : tx(tx), binary_tx(binary_tx), fee(fee), timestamp(timestamp) {} +BlockChainState::PoolTransaction::PoolTransaction(const Transaction &tx, const BinaryArray &binary_tx, Amount fee, + Timestamp timestamp, const Hash &newest_referenced_block) + : tx(tx) + , binary_tx(binary_tx) + , amount(get_tx_sum_outputs(tx)) + , fee(fee) + , timestamp(timestamp) + , newest_referenced_block(newest_referenced_block) {} void BlockChainState::DeltaState::store_keyimage(const KeyImage &key_image, Height height) { invariant(m_keyimages.insert(std::make_pair(key_image, height)).second, common::pod_to_hex(key_image)); @@ -59,61 +67,66 @@ bool BlockChainState::DeltaState::read_keyimage(const KeyImage &key_image, Heigh return true; } -uint32_t BlockChainState::DeltaState::push_amount_output( - Amount amount, BlockOrTimestamp unlock_time, Height block_height, const PublicKey &pk) { - uint32_t pg = m_parent_state->next_global_index_for_amount(amount); - auto &ga = m_global_amounts[amount]; - ga.push_back(std::make_pair(unlock_time, pk)); - return pg + static_cast(ga.size()) - 1; +size_t BlockChainState::DeltaState::push_amount_output( + Amount amount, BlockOrTimestamp unlock_time, Height block_height, const PublicKey &pk, bool is_auditable) { + auto pg = m_parent_state->next_global_index_for_amount(amount); + auto &ga = m_global_amounts[amount]; + ga.push_back(std::make_tuple(unlock_time, pk, is_auditable)); + return pg + ga.size() - 1; } -void BlockChainState::DeltaState::pop_amount_output(Amount amount, BlockOrTimestamp unlock_time, const PublicKey &pk) { - std::vector> &el = m_global_amounts[amount]; +void BlockChainState::DeltaState::pop_amount_output( + Amount amount, BlockOrTimestamp unlock_time, const PublicKey &pk, bool is_auditable) { + std::vector> &el = m_global_amounts[amount]; invariant(!el.empty(), "DeltaState::pop_amount_output underflow"); - invariant(el.back().first == unlock_time && el.back().second == pk, "DeltaState::pop_amount_output wrong element"); + invariant( + std::get<0>(el.back()) == unlock_time && std::get<1>(el.back()) == pk && std::get<2>(el.back()) == is_auditable, + "DeltaState::pop_amount_output wrong element"); el.pop_back(); } -uint32_t BlockChainState::DeltaState::next_global_index_for_amount(Amount amount) const { - uint32_t pg = m_parent_state->next_global_index_for_amount(amount); - auto git = m_global_amounts.find(amount); - return (git == m_global_amounts.end()) ? pg : static_cast(git->second.size()) + pg; +size_t BlockChainState::DeltaState::next_global_index_for_amount(Amount amount) const { + auto pg = m_parent_state->next_global_index_for_amount(amount); + auto git = m_global_amounts.find(amount); + return (git == m_global_amounts.end()) ? pg : git->second.size() + pg; } bool BlockChainState::DeltaState::read_amount_output( - Amount amount, uint32_t global_index, UnlockTimePublickKeyHeightSpent *unp) const { - uint32_t pg = m_parent_state->next_global_index_for_amount(amount); - if (global_index < pg) - return m_parent_state->read_amount_output(amount, global_index, unp); - global_index -= pg; - auto git = m_global_amounts.find(amount); - if (git == m_global_amounts.end() || global_index >= git->second.size()) - return false; - unp->unlock_block_or_timestamp = git->second[global_index].first; - unp->public_key = git->second[global_index].second; - unp->height = m_block_height; - unp->spent = false; // Spending just created outputs inside mempool or block is prohibited, simplifying logic - return true; -} -void BlockChainState::DeltaState::spend_output(Amount amount, uint32_t global_index) { - m_spent_outputs.push_back(std::make_pair(amount, global_index)); + Amount amount, size_t global_index, UnlockTimePublickKeyHeightSpent *unp) const { + // uint32_t pg = m_parent_state->next_global_index_for_amount(amount); + // if (global_index < pg) + return m_parent_state->read_amount_output(amount, global_index, unp); + // global_index -= pg; + // auto git = m_global_amounts.find(amount); + // if (git == m_global_amounts.end() || global_index >= git->second.size()) + // return false; + // unp->unlock_block_or_timestamp = std::get<0>(git->second[global_index]); + // unp->public_key = std::get<1>(git->second[global_index]); + // *is_white = std::get<2>(git->second[global_index]); + // unp->height = m_block_height; + // unp->spent = false; // Spending just created outputs inside mempool or block is prohibited, simplifying logic + // return true; } +// void BlockChainState::DeltaState::spend_output(Amount amount, size_t global_index) { +// m_spent_outputs.push_back(std::make_pair(amount, global_index)); +//} void BlockChainState::DeltaState::apply(IBlockChainState *parent_state) const { for (auto &&ki : m_keyimages) parent_state->store_keyimage(ki.first, ki.second); for (auto && : m_global_amounts) for (auto &&el : amp.second) - parent_state->push_amount_output(amp.first, el.first, m_block_height, el.second); - for (auto &&mo : m_spent_outputs) - parent_state->spend_output(mo.first, mo.second); + parent_state->push_amount_output( + amp.first, std::get<0>(el), m_block_height, std::get<1>(el), std::get<2>(el)); + // for (auto &&mo : m_spent_outputs) + // parent_state->spend_output(mo.first, mo.second); } void BlockChainState::DeltaState::clear(Height new_block_height) { m_block_height = new_block_height; m_keyimages.clear(); m_global_amounts.clear(); - m_spent_outputs.clear(); + // m_spent_outputs.clear(); } api::BlockHeader BlockChainState::fill_genesis(Hash genesis_bid, const BlockTemplate &g) { @@ -122,72 +135,91 @@ api::BlockHeader BlockChainState::fill_genesis(Hash genesis_bid, const BlockTemp result.minor_version = g.minor_version; result.previous_block_hash = g.previous_block_hash; result.timestamp = g.timestamp; - result.nonce = g.nonce; + result.binary_nonce = g.nonce; result.hash = genesis_bid; return result; } -static std::string validate_semantic(bool generating, const Transaction &tx, uint64_t *fee, bool check_output_key) { +// returns reward for coinbase transaction or fee for non-coinbase one +static Amount validate_semantic(const Currency ¤cy, uint8_t block_major_version, bool generating, + const Transaction &tx, bool check_output_key) { if (tx.inputs.empty()) - return "EMPTY_INPUTS"; - uint64_t summary_output_amount = 0; + throw ConsensusError("Empty inputs"); + // TODO - uncomment during next hard fork, finally prohibiting old signatures, outputs without secrets + // We cannot do it at once, because mem pool will have v1 transactions during switch + + // if(block_major_version >= currency.amethyst_block_version && tx.version < + // currency.amethyst_transaction_version && !generating) // for compatibility, we create v1 coinbase + // transaction if mining on legacy address + // return "WRONG_TRANSACTION_VERSION"; + if (block_major_version < currency.amethyst_block_version && tx.version >= currency.amethyst_transaction_version) + throw ConsensusError(common::to_string( + "Wrong transaction version", int(tx.version), "in block version", int(block_major_version))); + Amount summary_output_amount = 0; for (const auto &output : tx.outputs) { - if (output.amount == 0) - return "OUTPUT_ZERO_AMOUNT"; - if (output.target.type() == typeid(KeyOutput)) { - const KeyOutput &key_output = boost::get(output.target); + Amount amount = 0; + if (!currency.amount_allowed_in_output(block_major_version, amount)) + throw ConsensusError(common::to_string("Not round amount", amount)); + if (output.type() == typeid(OutputKey)) { + const auto &key_output = boost::get(output); + amount = key_output.amount; if (check_output_key && !key_isvalid(key_output.public_key)) - return "OUTPUT_INVALID_KEY"; + throw ConsensusError(common::to_string("Output key not valid elliptic point", key_output.public_key)); + if (tx.version < currency.amethyst_transaction_version && key_output.is_auditable) + throw ConsensusError( + common::to_string("Transaction version", tx.version, "insufficient for output with audit")); } else - return "OUTPUT_UNKNOWN_TYPE"; - if (std::numeric_limits::max() - output.amount < summary_output_amount) - return "OUTPUTS_AMOUNT_OVERFLOW"; - summary_output_amount += output.amount; + throw ConsensusError("Output type unknown"); + if (amount == 0) + throw ConsensusError("Output amount 0"); + // if (std::numeric_limits::max() - amount < summary_output_amount) + // throw ConsensusError("Outputs amounts overflow"); + if (!add_amount(summary_output_amount, amount)) + throw ConsensusError("Outputs amounts overflow"); } - uint64_t summary_input_amount = 0; + Amount summary_input_amount = 0; std::unordered_set ki; - std::set> outputs_usage; for (const auto &input : tx.inputs) { - uint64_t amount = 0; - if (input.type() == typeid(CoinbaseInput)) { + Amount amount = 0; + if (input.type() == typeid(InputCoinbase)) { if (!generating) - return "INPUT_UNKNOWN_TYPE"; - } else if (input.type() == typeid(KeyInput)) { + throw ConsensusError("Coinbase input in non-coinbase transaction"); + } else if (input.type() == typeid(InputKey)) { if (generating) - return "INPUT_UNKNOWN_TYPE"; - const KeyInput &in = boost::get(input); + throw ConsensusError("Key input in coinbase transaction"); + const InputKey &in = boost::get(input); amount = in.amount; if (!ki.insert(in.key_image).second) - return "INPUT_IDENTICAL_KEYIMAGES"; - if (in.output_indexes.empty()) - return "INPUT_EMPTY_OUTPUT_USAGE"; - // output_indexes are packed here, first is absolute, others are offsets to - // previous, so first can be zero, others can't - if (std::find(++std::begin(in.output_indexes), std::end(in.output_indexes), 0) != - std::end(in.output_indexes)) { - return "INPUT_IDENTICAL_OUTPUT_INDEXES"; - } + throw ConsensusError(common::to_string("Keyimage used twice in same transaction", in.key_image)); + std::vector global_indexes; + if (!relative_output_offsets_to_absolute(&global_indexes, in.output_indexes)) + throw ConsensusError("Output indexes invalid in input"); } else - return "INPUT_UNKNOWN_TYPE"; - if (std::numeric_limits::max() - amount < summary_input_amount) - return "INPUTS_AMOUNT_OVERFLOW"; - summary_input_amount += amount; + throw ConsensusError("Input type unknown"); + // if (std::numeric_limits::max() - amount < summary_input_amount) + // throw ConsensusError("Inputs amounts overflow"); + if (!add_amount(summary_input_amount, amount)) + throw ConsensusError("Outputs amounts overflow"); } if (summary_output_amount > summary_input_amount && !generating) - return "WRONG_AMOUNT"; - if (tx.signatures.size() != tx.inputs.size() && !generating) - return "INPUT_UNKNOWN_TYPE"; - if (!tx.signatures.empty() && generating) - return "INPUT_UNKNOWN_TYPE"; - *fee = summary_input_amount - summary_output_amount; - return std::string(); + throw ConsensusError("Sum of outputs > sum of inputs in non-coinbase transaction"); + // Types/count of signatures will be checked as a part of signatures check + // if (tx.signatures.size() != tx.inputs.size() && !generating) + // return "INPUT_UNKNOWN_TYPE"; + // if (!tx.signatures.empty() && generating) + // return "INPUT_UNKNOWN_TYPE"; + if (generating) + return summary_output_amount; + return summary_input_amount - summary_output_amount; } BlockChainState::BlockChainState(logging::ILogger &log, const Config &config, const Currency ¤cy, bool read_only) - : BlockChain(log, config, currency, read_only), log_redo_block_timestamp(std::chrono::steady_clock::now()) { + : BlockChain(log, config, currency, read_only) + , m_max_pool_size(config.max_pool_size) + , m_log_redo_block_timestamp(std::chrono::steady_clock::now()) { std::string version; m_db.get("$version", version); - if (version == "B" || version == "1" || version == "2" || version == "3" || version == "4") { + if (version == "B" || version == "1" || version == "2" || version == "3" || version == "4" || version == "5") { start_internal_import(); version = version_current; m_db.put("$version", version, false); @@ -198,338 +230,325 @@ BlockChainState::BlockChainState(logging::ILogger &log, const Config &config, co throw std::runtime_error("Blockchain database format unknown (version=" + version + "), please delete " + config.get_data_folder() + "/blockchain"); if (get_tip_height() == (Height)-1) { - Block genesis_block; - genesis_block.header = currency.genesis_block_template; + // Block genesis_block; + // genesis_block.header = currency.genesis_block_template; RawBlock raw_block; - invariant(genesis_block.to_raw_block(raw_block), "Genesis block failed to convert into raw block"); + raw_block.block = seria::to_binary(currency.genesis_block_template); + // invariant(genesis_block.to_raw_block(raw_block), "Genesis block failed to convert into raw block"); PreparedBlock pb(std::move(raw_block), m_currency, nullptr); api::BlockHeader info; - invariant(add_block(pb, &info, std::string()) != BroadcastAction::BAN, "Genesis block failed to add"); + invariant(add_block(pb, &info, std::string()), "Genesis block failed to add"); } BlockChainState::tip_changed(); m_log(logging::INFO) << "BlockChainState::BlockChainState height=" << get_tip_height() << " cumulative_difficulty=" << get_tip_cumulative_difficulty() << " bid=" << get_tip_bid() << std::endl; - /* RawBlock rb; - Block bb; - invariant(read_block(common::pfh("ec140124695fbe90929a3f49dbe4d2b88fa0ad5ae271aedf0ebcd9f55b6bd3d7"), - &rb), ""); - invariant(bb.from_raw_block(rb), ""); - Hash ha1 = get_auxiliary_block_header_hash(bb.header); - Hash ha2 = get_block_hash(bb.header); - auto ba3 = currency.get_block_long_hashing_data(bb.header); - auto bbh = seria::to_binary(bb.header); - std::cout << common::to_hex(rb.block.data(), rb.block.size()) << std::endl; - std::cout << common::to_hex(bbh.data(), bbh.size()) << std::endl; - invariant(bbh == rb.block, "");*/ - /* size_t templates_size = 0; - size_t true_headers_size = 0; - Height start = 400000; - Height ha = start; - for(; ha < start + 10000; ++ha ){ - Hash bid; - if(!read_chain(ha, &bid)) - break; - RawBlock rb; - Block bb; - invariant(read_block(bid, &rb), ""); - invariant(bb.from_raw_block(rb), ""); - BinaryArray tba = seria::to_binary(static_cast(bb.header)); - BinaryArray tba2 = seria::to_binary(get_body_proxy_from_template(bb.header)); - templates_size += rb.block.size(); - true_headers_size += tba.size() + tba2.size(); - } - std::cout << "from height " << start << " to " << ha << " block templates size=" << templates_size << " true - headers + body proxies size=" << true_headers_size << std::endl;*/ - /*BlockHeader tr; - tr.major_version = 104; - tr.cm_merkle_branch.push_back(crypto::rand()); - tr.cm_merkle_branch.push_back(crypto::rand()); - tr.cm_merkle_branch.push_back(Hash{}); - tr.cm_merkle_branch.push_back(Hash{}); - tr.cm_merkle_branch.push_back(Hash{}); - tr.cm_merkle_branch.push_back(crypto::rand()); - tr.cm_merkle_branch.push_back(Hash{}); - tr.cm_merkle_branch.push_back(Hash{}); - tr.cm_merkle_branch.push_back(Hash{}); - tr.cm_merkle_branch.push_back(crypto::rand()); - BinaryArray ba = seria::to_binary(tr); - std::cout << seria::to_json_value(tr).to_string() << std::endl; - BlockHeader tr2; - seria::from_binary(tr2, ba); - invariant(tr.cm_merkle_branch == tr2.cm_merkle_branch, "");*/ build_blods(); + DB::Cursor cur2 = m_db.rbegin(DIN_PREFIX); + m_next_nz_input_index = + cur2.end() ? 0 : common::integer_cast(common::read_varint_sqlite4(cur2.get_suffix())) + 1; } -std::string BlockChainState::check_standalone_consensus( +void BlockChainState::check_standalone_consensus( const PreparedBlock &pb, api::BlockHeader *info, const api::BlockHeader &prev_info, bool check_pow) const { + if (pb.error) // Some semantic checks are in PreparedBlock::prepare + throw pb.error.get(); const auto &block = pb.block; if (block.transactions.size() != block.header.transaction_hashes.size() || block.transactions.size() != pb.raw_block.transactions.size()) - return "WRONG_TRANSACTIONS_COUNT"; - info->size_median = m_next_median_size; - info->timestamp_median = m_next_median_timestamp; - // info->timestamp_unlock = m_next_unlock_timestamp; - - if (get_tip_bid() != prev_info.hash) // Optimization for most common case - calculate_consensus_values(prev_info, &info->size_median, &info->timestamp_median); - - auto next_minimum_size_median = m_currency.get_minimum_size_median(block.header.major_version); - info->effective_size_median = std::max(info->size_median, next_minimum_size_median); - + throw ConsensusError("Wrong transaction count in block template"); + // Timestamps are within reason + if (get_tip_bid() == prev_info.hash) // Optimization for most common case + info->timestamp_median = m_next_median_timestamp; + else + info->timestamp_median = calculate_next_median_timestamp(prev_info); + auto now = platform::now_unix_timestamp(); // It would be better to pass now through Node + if (block.header.timestamp > now + m_currency.block_future_time_limit) + throw ConsensusError("Timestamp too far in future"); + if (block.header.timestamp < info->timestamp_median) + throw ConsensusError("Timestamp too far in past"); + // Block versions + const bool is_amethyst = block.header.major_version >= m_currency.amethyst_block_version; + const auto body_proxy = get_body_proxy_from_template(block.header); + + uint8_t should_be_major_mm = 0, should_be_major_cm = 0, might_be_minor = 0; + if (!fill_next_block_versions(prev_info, false, &should_be_major_mm, &should_be_major_cm, &might_be_minor)) + throw ConsensusError("Block does not pass through last hard checkpoint"); + if (block.header.major_version != should_be_major_mm && block.header.major_version != should_be_major_cm) + throw ConsensusError(common::to_string("Block version wrong", int(block.header.major_version), "instead of", + int(should_be_major_mm), "or", int(should_be_major_cm))); + + // Object sizes ok size_t cumulative_size = 0; - for (size_t i = 0; i != pb.raw_block.transactions.size(); ++i) { - if (pb.raw_block.transactions.at(i).size() > - m_currency.max_transaction_allowed_size(info->effective_size_median)) { - // log(Logging::INFO) << "Raw transaction size " << - // binary_transaction.size() << " is too big."; - return "RAW_TRANSACTION_SIZE_TOO_BIG"; - } + for (size_t i = 0; i != pb.raw_block.transactions.size(); ++i) cumulative_size += pb.raw_block.transactions.at(i).size(); - Hash tid = get_transaction_hash(pb.block.transactions.at(i)); - if (tid != pb.block.header.transaction_hashes.at(i)) - return "TRANSACTION_ABSENT_IN_POOL"; + if (is_amethyst) { // We care only about single limit - block size + if (!extra_get_block_capacity_vote(block.header.base_transaction.extra, &info->block_capacity_vote)) + throw ConsensusError("No block capacity vote"); + if (info->block_capacity_vote < m_currency.block_capacity_vote_min || + info->block_capacity_vote > m_currency.block_capacity_vote_max) + throw ConsensusError(common::to_string("Invalid block capacity vote", info->block_capacity_vote, + "should be >=", m_currency.block_capacity_vote_min, "and <=", m_currency.block_capacity_vote_max)); + if (get_tip_bid() == prev_info.hash) // Optimization for most common case + info->block_capacity_vote_median = m_next_median_block_capacity_vote; + else + info->block_capacity_vote_median = calculate_next_median_block_capacity_vote(prev_info); + info->transactions_size = pb.coinbase_tx_size + cumulative_size; + if (info->transactions_size > info->block_capacity_vote_median) + throw ConsensusError(common::to_string("Block size too big, transactions size", info->transactions_size, + "should be <=", info->block_capacity_vote_median)); + if (pb.block_header_size > m_currency.max_header_size) + throw ConsensusError(common::to_string( + "Header size too big,", pb.block_header_size, "should be <=", m_currency.max_header_size)); + info->block_size = pb.block_header_size + common::get_varint_data(pb.raw_block.transactions.size()).size(); + info->block_size += info->transactions_size; + } else { + if (get_tip_bid() == prev_info.hash) // Optimization for most common case + info->size_median = m_next_median_size; + else + info->size_median = calculate_next_median_size(prev_info); + auto next_minimum_size_median = m_currency.get_minimum_size_median(block.header.major_version); + info->effective_size_median = std::max(info->size_median, next_minimum_size_median); + + info->transactions_size = pb.coinbase_tx_size + cumulative_size; + info->block_size = pb.raw_block.block.size() + cumulative_size; + // block_size not used in consensus calcs, we would change it to match definition in if() above + // but then some block explorers will not be happy due to change. + + const auto max_transactions_cumulative_size = m_currency.max_block_transactions_cumulative_size(info->height); + if (info->transactions_size > max_transactions_cumulative_size) + throw ConsensusError(common::to_string("Cumulative block transactions size too big,", + info->transactions_size, "should be <=", max_transactions_cumulative_size)); + if (info->transactions_size > info->effective_size_median * 2) + throw ConsensusError(common::to_string("Cumulative block transactions size too big,", + info->transactions_size, "should be <=", info->effective_size_median * 2)); + if (block.header.is_merge_mined() && pb.parent_block_size > m_currency.max_header_size) + throw ConsensusError(common::to_string( + "Root block size too big,", pb.parent_block_size, "should be <=", m_currency.max_header_size)); } - info->block_size = static_cast(pb.coinbase_tx_size + cumulative_size); - auto max_block_cumulative_size = m_currency.max_block_cumulative_size(info->height); - if (info->block_size > max_block_cumulative_size) - return "CUMULATIVE_BLOCK_SIZE_TOO_BIG"; - - uint8_t should_be_major = 0, might_be_minor = 0; - if (!fill_next_block_versions(prev_info, false, &should_be_major, &might_be_minor)) - return "DOES_NOT_PASS_THROUGH_LAST_SW_CHECKPOINT"; - if (block.header.major_version != should_be_major) - return "WRONG_VERSION"; - if (block.header.is_merge_mined()) { - if (block.header.major_version == 2 && block.header.parent_block.major_version > 1) - return "PARENT_BLOCK_WRONG_VERSION"; - size_t pasi = pb.parent_block_size; - if (pasi > 2048) - return "PARENT_BLOCK_SIZE_TOO_BIG"; + TransactionExtraMergeMiningTag mm_tag; + if (!extra_get_merge_mining_tag(block.header.root_block.base_transaction.extra, mm_tag)) + throw ConsensusError("No merge mining tag"); + if (mm_tag.depth != block.header.root_block.blockchain_branch.size()) + throw ConsensusError(common::to_string("Wrong merge mining depth,", mm_tag.depth, " should be ", + block.header.root_block.blockchain_branch.size())); + if (block.header.root_block.blockchain_branch.size() > 8 * sizeof(Hash)) + throw ConsensusError(common::to_string("Too big merge mining depth,", + block.header.root_block.blockchain_branch.size(), "should be <=", 8 * sizeof(Hash))); + Hash aux_blocks_merkle_root = crypto::tree_hash_from_branch(block.header.root_block.blockchain_branch.data(), + block.header.root_block.blockchain_branch.size(), get_auxiliary_block_header_hash(block.header, body_proxy), + &m_currency.genesis_block_hash); + if (aux_blocks_merkle_root != mm_tag.merkle_root) + throw ConsensusError(common::to_string( + "Wrong merge mining merkle root, tag", mm_tag.merkle_root, "actual", aux_blocks_merkle_root)); + } + if (block.header.is_cm_mined()) { + if (!crypto::cm_branch_valid(block.header.cm_merkle_branch)) + throw ConsensusError("CM branch invalid"); } - auto now = platform::now_unix_timestamp(); // It would be better to pass now through Node - if (block.header.timestamp > now + m_currency.block_future_time_limit) - return "TIMESTAMP_TOO_FAR_IN_FUTURE"; - if (block.header.timestamp < info->timestamp_median) - return "TIMESTAMP_TOO_FAR_IN_PAST"; - if (block.header.base_transaction.inputs.size() != 1) - return "INPUT_WRONG_COUNT"; - - if (block.header.base_transaction.inputs[0].type() != typeid(CoinbaseInput)) - return "INPUT_UNEXPECTED_TYPE"; - - if (boost::get(block.header.base_transaction.inputs[0]).height != info->height) - return "BASE_INPUT_WRONG_BLOCK_INDEX"; - - if (block.header.base_transaction.unlock_block_or_timestamp != info->height + m_currency.mined_money_unlock_window) - return "WRONG_TRANSACTION_UNLOCK_TIME"; - - const bool check_keys = m_config.paranoid_checks || !m_currency.is_in_sw_checkpoint_zone(info->height); - uint64_t miner_reward = 0; - for (const auto &output : block.header.base_transaction.outputs) { // TODO - call validate_semantic - if (output.amount == 0) - return "OUTPUT_ZERO_AMOUNT"; - if (output.target.type() == typeid(KeyOutput)) { - if (check_keys && !key_isvalid(boost::get(output.target).public_key)) - return "OUTPUT_INVALID_KEY"; - } else - return "OUTPUT_UNKNOWN_TYPE"; - - if (std::numeric_limits::max() - output.amount < miner_reward) - return "OUTPUTS_AMOUNT_OVERFLOW"; - miner_reward += output.amount; + throw ConsensusError(common::to_string( + "Coinbase transaction input count wrong,", block.header.base_transaction.inputs.size(), "should be 1")); + if (block.header.base_transaction.inputs[0].type() != typeid(InputCoinbase)) + throw ConsensusError("Coinbase transaction input type wrong"); + { + const auto coinbase_input = boost::get(block.header.base_transaction.inputs[0]); + if (coinbase_input.height != info->height) + throw ConsensusError(common::to_string( + "Coinbase transaction wrong input height,", coinbase_input.height, "should be", info->height)); } + if (!is_amethyst && block.header.base_transaction.unlock_block_or_timestamp != + info->height + m_currency.mined_money_unlock_window) { + throw ConsensusError(common::to_string("Coinbase transaction wrong unlock time,", + block.header.base_transaction.unlock_block_or_timestamp, "should be", + info->height + m_currency.mined_money_unlock_window)); + } + const bool check_keys = m_config.paranoid_checks || !m_currency.is_in_hard_checkpoint_zone(info->height); + const Amount miner_reward = + validate_semantic(m_currency, block.header.major_version, true, block.header.base_transaction, check_keys); { std::vector timestamps; std::vector difficulties; - Height blocks_count = std::min(prev_info.height, m_currency.difficulty_blocks_count()); - auto timestamps_window = get_tip_segment(prev_info, blocks_count, false); // TODO - excess min - size_t actual_count = timestamps_window.size(); - timestamps.resize(actual_count); - difficulties.resize(actual_count); - size_t pos = 0; - for (auto it = timestamps_window.begin(); it != timestamps_window.end(); ++it, ++pos) { - timestamps.at(pos) = it->timestamp; - difficulties.at(pos) = it->cumulative_difficulty; - } + const Height blocks_count = m_currency.difficulty_windows_plus_lag(); + timestamps.reserve(blocks_count); + difficulties.reserve(blocks_count); + for_each_reversed_tip_segment(prev_info, blocks_count, false, [&](const api::BlockHeader &header) { + timestamps.push_back(header.timestamp); + difficulties.push_back(header.cumulative_difficulty); + }); + std::reverse(timestamps.begin(), timestamps.end()); + std::reverse(difficulties.begin(), difficulties.end()); info->difficulty = m_currency.next_effective_difficulty(block.header.major_version, timestamps, difficulties); info->cumulative_difficulty = prev_info.cumulative_difficulty + info->difficulty; } - // if (info->difficulty == 0) - // return "DIFFICULTY_OVERHEAD"; - - Amount transactions_fee = 0; - for (const auto &tx : block.transactions) { - Amount fee = 0; - if (!get_tx_fee(tx, &fee)) - return "WRONG_AMOUNT"; - transactions_fee += fee; + info->transactions_fee = 0; + for (auto &&tx : pb.block.transactions) { + const Amount tx_fee = validate_semantic(m_currency, block.header.major_version, false, tx, check_keys); + info->transactions_fee += tx_fee; } - int64_t emission_change = 0; - auto already_generated_coins = prev_info.already_generated_coins; - - if (info->block_size > info->effective_size_median * 2) - return "CUMULATIVE_BLOCK_SIZE_TOO_BIG"; - - m_currency.get_block_reward(block.header.major_version, info->effective_size_median, 0, already_generated_coins, 0, - &info->base_reward, &emission_change); - - m_currency.get_block_reward(block.header.major_version, info->effective_size_median, info->block_size, - already_generated_coins, transactions_fee, &info->reward, &emission_change); - - if (miner_reward != info->reward) { - // log(Logging::WARNING) << "Block reward mismatch for block " << - // hash << ". Expected reward: " << reward << ", got reward: " << - // miner_reward; - return "BLOCK_REWARD_MISMATCH"; + if (is_amethyst) { + info->base_reward = m_currency.get_base_block_reward( + block.header.major_version, info->height, prev_info.already_generated_coins); + info->reward = info->base_reward + info->transactions_fee; + info->already_generated_coins = prev_info.already_generated_coins + info->base_reward; + } else { + SignedAmount emission_change = 0; + info->base_reward = m_currency.get_block_reward(block.header.major_version, info->height, + info->effective_size_median, 0, prev_info.already_generated_coins, 0, &emission_change); + info->reward = + m_currency.get_block_reward(block.header.major_version, info->height, info->effective_size_median, + info->transactions_size, prev_info.already_generated_coins, info->transactions_fee, &emission_change); + info->already_generated_coins = prev_info.already_generated_coins + emission_change; } - info->already_generated_coins = prev_info.already_generated_coins + emission_change; + + if (miner_reward != info->reward) + throw ConsensusError(common::to_string("Block reward mismatch,", miner_reward, "should be", info->reward)); info->already_generated_transactions = prev_info.already_generated_transactions + block.transactions.size() + 1; - info->transactions_fee = transactions_fee; - info->transactions_size = static_cast(cumulative_size); - for (auto &&tx : pb.block.transactions) { - Amount tx_fee = 0; - std::string tx_result = validate_semantic(false, tx, &tx_fee, m_config.paranoid_checks || check_keys); - if (!tx_result.empty()) - return tx_result; - } - if (m_currency.is_in_sw_checkpoint_zone(info->height)) { + if (m_currency.is_in_hard_checkpoint_zone(info->height)) { bool is_checkpoint; - if (!m_currency.check_sw_checkpoint(info->height, info->hash, is_checkpoint)) - return "CHECKPOINT_BLOCK_HASH_MISMATCH"; - } else { - if (!check_pow && !m_config.paranoid_checks) - return std::string(); - Hash long_hash = pb.long_block_hash; - if (long_hash == Hash{}) { - auto body_proxy = get_body_proxy_from_template(block.header); - auto ba = m_currency.get_block_long_hashing_data(block.header, body_proxy); - long_hash = m_hash_crypto_context.cn_slow_hash(ba.data(), ba.size()); - } - if (!check_hash(long_hash, info->difficulty)) - return "PROOF_OF_WORK_TOO_WEAK"; + if (!m_currency.check_hard_checkpoint(info->height, info->hash, is_checkpoint)) + throw ConsensusError( + common::to_string("Block does not pass through hard checkpoint at height", info->height)); + return; + } + if (!check_pow && !m_config.paranoid_checks) + return; + Hash long_hash = pb.long_block_hash; + if (long_hash == Hash{}) { // We did not calculate this long hash in parallel + auto ba = m_currency.get_block_long_hashing_data(block.header, body_proxy); + long_hash = m_hash_crypto_context.cn_slow_hash(ba.data(), ba.size()); } - return std::string(); + if (!check_hash(long_hash, info->difficulty)) + throw ConsensusError("Proof of work too weak"); } -void BlockChainState::fill_statistics(api::bytecoind::GetStatistics::Response &res) const { +void BlockChainState::fill_statistics(api::cnd::GetStatistics::Response &res) const { BlockChain::fill_statistics(res); - res.transaction_pool_size = m_memory_state_total_size; - res.transaction_pool_max_size = MAX_POOL_SIZE; - Hash minimal_tid; - res.transaction_pool_lowest_fee_per_byte = minimum_pool_fee_per_byte(&minimal_tid); + res.transaction_pool_count = m_memory_state_tx.size(); + res.transaction_pool_size = m_memory_state_total_size; + res.transaction_pool_max_size = m_max_pool_size; + res.transaction_pool_lowest_fee_per_byte = minimum_pool_fee_per_byte(false); } -void BlockChainState::calculate_consensus_values( - const api::BlockHeader &prev_info, uint32_t *next_median_size, Timestamp *next_median_timestamp) const { - std::vector last_blocks_sizes; - auto window = get_tip_segment(prev_info, m_currency.reward_blocks_window, true); - last_blocks_sizes.reserve(m_currency.reward_blocks_window); - for (auto it = window.begin(); it != window.end(); ++it) - last_blocks_sizes.push_back(it->block_size); - *next_median_size = common::median_value(&last_blocks_sizes); - - window = get_tip_segment(prev_info, m_currency.timestamp_check_window, false); - if (window.size() >= m_currency.timestamp_check_window) { - std::vector timestamps; - timestamps.reserve(m_currency.timestamp_check_window); - for (auto it = window.begin(); it != window.end(); ++it) - timestamps.push_back(it->timestamp); - *next_median_timestamp = common::median_value(×tamps); // sorts timestamps - //*next_unlock_timestamp = timestamps[timestamps.size() / 2]; - // unlike median_value, here we select lesser of 2 middle values for - // even-sized array, so - // that m_next_unlock_timestamp will never decrease with block number - // if (*next_unlock_timestamp < m_currency.block_future_time_limit) - // *next_unlock_timestamp = 0; - // else - // *next_unlock_timestamp -= m_currency.block_future_time_limit; - } else { - *next_median_timestamp = 0; - //*next_unlock_timestamp = 0; - } +Timestamp BlockChainState::calculate_next_median_timestamp(const api::BlockHeader &prev_info) const { + std::vector timestamps; + auto timestamp_check_window = m_currency.timestamp_check_window(prev_info.major_version); + timestamps.reserve(timestamp_check_window); + for_each_reversed_tip_segment(prev_info, timestamp_check_window, false, + [&](const api::BlockHeader &header) { timestamps.push_back(header.timestamp); }); + if (timestamps.size() >= timestamp_check_window) + return common::median_value(×tamps); // sorts timestamps + return 0; } -void BlockChainState::tip_changed() { - calculate_consensus_values(get_tip(), &m_next_median_size, &m_next_median_timestamp); +size_t BlockChainState::calculate_next_median_size(const api::BlockHeader &prev_info) const { + std::vector last_transactions_sizes; + last_transactions_sizes.reserve(m_currency.median_block_size_window); + for_each_reversed_tip_segment(prev_info, m_currency.median_block_size_window, true, + [&](const api::BlockHeader &header) { last_transactions_sizes.push_back(header.transactions_size); }); + return common::median_value(&last_transactions_sizes); } -void BlockChainState::create_mining_block_template(const AccountPublicAddress &adr, const BinaryArray &extra_nonce, - BlockTemplate *b, Difficulty *difficulty, Height *height) const { - create_mining_block_template(adr, extra_nonce, b, difficulty, height, get_tip_bid()); +size_t BlockChainState::calculate_next_median_block_capacity_vote(const api::BlockHeader &prev_info) const { + std::vector last_blocks_sizes; + last_blocks_sizes.reserve(m_currency.block_capacity_vote_window); + for_each_reversed_tip_segment( + prev_info, m_currency.block_capacity_vote_window, true, [&](const api::BlockHeader &header) { + if (header.major_version >= m_currency.amethyst_block_version) + last_blocks_sizes.push_back(header.block_capacity_vote); + }); + if (last_blocks_sizes.empty()) + return m_currency.block_capacity_vote_min; + return common::median_value(&last_blocks_sizes); } -void BlockChainState::create_mining_block_template(const AccountPublicAddress &adr, const BinaryArray &extra_nonce, - BlockTemplate *b, Difficulty *difficulty, Height *height, Hash parent_bid) const { +void BlockChainState::tip_changed() { + m_next_median_timestamp = calculate_next_median_timestamp(get_tip()); + m_next_median_size = calculate_next_median_size(get_tip()); + m_next_median_block_capacity_vote = calculate_next_median_block_capacity_vote(get_tip()); +} + +void BlockChainState::create_mining_block_template(const Hash &parent_bid, const AccountAddress &adr, + const BinaryArray &extra_nonce, BlockTemplate *b, Difficulty *difficulty, Height *height, + size_t *reserved_back_offset) const { api::BlockHeader parent_info; - if (!read_header(parent_bid, &parent_info)) + if (!get_header(parent_bid, &parent_info)) throw std::runtime_error("Attempt to mine from block we do not have"); - *height = parent_info.height + 1; - *b = BlockTemplate{}; - if (!fill_next_block_versions(parent_info, false, &b->major_version, &b->minor_version)) + *height = parent_info.height + 1; + *b = BlockTemplate{}; + uint8_t major_version_cm = 0; + if (!fill_next_block_versions(parent_info, false, &b->major_version, &major_version_cm, &b->minor_version)) throw std::runtime_error( - "Mining of block in chain not passing through last SW checkpoint is not possible (will not be accepted by network anyway)"); + "Mining of block in chain not passing through last hard checkpoint is not possible (will not be accepted by network anyway)"); + const bool is_amethyst = b->major_version >= m_currency.amethyst_block_version; - uint32_t next_median_size = 0; - Timestamp next_median_timestamp = 0; - calculate_consensus_values(parent_info, &next_median_size, &next_median_timestamp); - clear_mining_transactions(); // ???? + clear_mining_transactions(); // We periodically forget transactions for old blocks we gave as templates { std::vector timestamps; std::vector difficulties; - Height blocks_count = std::min(parent_info.height, m_currency.difficulty_blocks_count()); + const Height blocks_count = m_currency.difficulty_windows_plus_lag(); timestamps.reserve(blocks_count); difficulties.reserve(blocks_count); - auto timestamps_window = get_tip_segment(parent_info, blocks_count, false); - for (auto it = timestamps_window.begin(); it != timestamps_window.end(); ++it) { - timestamps.push_back(it->timestamp); - difficulties.push_back(it->cumulative_difficulty); - } + for_each_reversed_tip_segment(parent_info, blocks_count, false, [&](const api::BlockHeader &header) { + timestamps.push_back(header.timestamp); + difficulties.push_back(header.cumulative_difficulty); + }); + std::reverse(timestamps.begin(), timestamps.end()); + std::reverse(difficulties.begin(), difficulties.end()); + // timestamps.reserve(blocks_count); + // difficulties.reserve(blocks_count); + // auto timestamps_window = get_tip_segment(parent_info, blocks_count, false); + // for (auto it = timestamps_window.begin(); it != timestamps_window.end(); ++it) { + // timestamps.push_back(it->timestamp); + // difficulties.push_back(it->cumulative_difficulty); + // } *difficulty = m_currency.next_effective_difficulty(b->major_version, timestamps, difficulties); } + b->nonce.resize(4); if (b->is_merge_mined()) { - b->parent_block.major_version = 1; - b->parent_block.minor_version = 0; - b->parent_block.transaction_count = 1; + b->root_block.major_version = 1; + b->root_block.minor_version = 0; + b->root_block.transaction_count = 1; - extra_add_merge_mining_tag(b->parent_block.base_transaction.extra, TransactionExtraMergeMiningTag{}); + extra_add_merge_mining_tag(b->root_block.base_transaction.extra, TransactionExtraMergeMiningTag{}); } - b->previous_block_hash = parent_bid; - b->parent_block.timestamp = std::max(platform::now_unix_timestamp(), next_median_timestamp); - b->timestamp = b->parent_block.timestamp; + b->previous_block_hash = parent_bid; + const Timestamp next_median_timestamp = calculate_next_median_timestamp(parent_info); + b->root_block.timestamp = std::max(platform::now_unix_timestamp(), next_median_timestamp); + b->timestamp = b->root_block.timestamp; - auto next_minimum_size_median = m_currency.get_minimum_size_median(b->major_version); - auto effective_size_median = std::max(next_median_size, next_minimum_size_median); - Amount already_generated_coins = parent_info.already_generated_coins; + size_t max_txs_size = 0; + size_t effective_size_median = 0; - // auto max_total_size = (150 * effective_size_median) / 100; - const auto max_consensus_block_size = - std::min(m_currency.max_block_cumulative_size(*height), 2 * effective_size_median); - const auto max_txs_size = max_consensus_block_size - m_currency.miner_tx_blob_reserved_size; + if (is_amethyst) { + const size_t next_median_block_capacity_vote = calculate_next_median_block_capacity_vote(parent_info); + max_txs_size = next_median_block_capacity_vote - m_currency.miner_tx_blob_reserved_size - extra_nonce.size(); + } else { + const size_t next_median_size = calculate_next_median_size(parent_info); + const size_t next_minimum_size_median = m_currency.get_minimum_size_median(b->major_version); + effective_size_median = std::max(next_median_size, next_minimum_size_median); + const auto max_consensus_transactions_size = + std::min(m_currency.max_block_transactions_cumulative_size(*height), 2 * effective_size_median); + max_txs_size = max_consensus_transactions_size - m_currency.miner_tx_blob_reserved_size - extra_nonce.size(); + } std::vector pool_hashes; for (auto &&msf : m_memory_state_fee_tx) - for (auto &&ha : msf.second) - pool_hashes.push_back(ha); + pool_hashes.push_back(msf.second); size_t txs_size = 0; Amount txs_fee = 0; - DeltaState memory_state(*height, b->timestamp, this); // will be get_tip().timestamp_unlock after fork - // TODO - technically we should give unlock timestamp of next block, but more - // conservative also works - Amount base_reward = 0; - SignedAmount emission_change = 0; - m_currency.get_block_reward( - b->major_version, effective_size_median, 0, already_generated_coins, 0, &base_reward, &emission_change); + DeltaState memory_state(*height, b->timestamp, next_median_timestamp, this); + // Amount base_reward = m_currency.get_block_reward( + // b->major_version, *height, effective_size_median, 0, parent_info.already_generated_coins, 0); for (; !pool_hashes.empty(); pool_hashes.pop_back()) { auto tit = m_memory_state_tx.find(pool_hashes.back()); if (tit == m_memory_state_tx.end()) { m_log(logging::ERROR) << "Transaction " << pool_hashes.back() << " is in pool index, but not in pool"; - assert(false); + // assert(false); continue; } const size_t tx_size = tit->second.binary_tx.size(); @@ -538,21 +557,16 @@ void BlockChainState::create_mining_block_template(const AccountPublicAddress &a continue; BlockGlobalIndices global_indices; Height conflict_height = 0; - const std::string result = - redo_transaction_get_error(false, tit->second.tx, &memory_state, &global_indices, &conflict_height, true); - if (!result.empty()) { + try { // double-check that transcations can be added to block + redo_transaction(b->major_version, false, tit->second.tx, &memory_state, &global_indices, nullptr, true); + } catch (const ConsensusError &ex) { m_log(logging::ERROR) << "Transaction " << tit->first - << " is in pool, but could not be redone result=" << result + << " is in pool, but could not be redone what=" << common::what(ex) << " Conflict height=" << conflict_height << std::endl; continue; } - if (txs_size + tx_size > effective_size_median) { - Amount reward; - m_currency.get_block_reward(b->major_version, effective_size_median, txs_size + tx_size, - already_generated_coins, txs_fee + tx_fee, &reward, &emission_change); - if (reward < base_reward) - continue; - } + if (!is_amethyst && txs_size + tx_size > effective_size_median) + continue; // Effective median size will not grow anyway txs_size += tx_size; txs_fee += tx_fee; b->transaction_hashes.emplace_back(tit->first); @@ -561,29 +575,45 @@ void BlockChainState::create_mining_block_template(const AccountPublicAddress &a m_log(logging::TRACE) << "Transaction " << tit->first << " included to block template"; } + if (is_amethyst) { + // Vote for larger blocks if pool is full of expensive transactions + Amount desired_fee_per_byte = 100; + size_t block_capacity_vote = 0; + for (auto fit = m_memory_state_fee_tx.rbegin(); fit != m_memory_state_fee_tx.rend(); ++fit) { + if (fit->first < desired_fee_per_byte) + break; + auto tit = m_memory_state_tx.find(fit->second); + invariant(tit != m_memory_state_tx.end(), "Memory pool corrupted"); + block_capacity_vote += tit->second.binary_tx.size(); + } + block_capacity_vote += m_currency.block_capacity_vote_min / 2; // A bit of space for cheaper transactions + block_capacity_vote = std::max(block_capacity_vote, m_currency.block_capacity_vote_min); + block_capacity_vote = std::min(block_capacity_vote, m_currency.block_capacity_vote_max); + Amount block_reward = + txs_fee + m_currency.get_base_block_reward(b->major_version, *height, parent_info.already_generated_coins); + b->base_transaction = m_currency.construct_miner_tx(b->major_version, *height, block_reward, adr); + extra_add_block_capacity_vote(b->base_transaction.extra, block_capacity_vote); + if (!extra_nonce.empty()) + extra_add_nonce(b->base_transaction.extra, extra_nonce); + *reserved_back_offset = + common::get_varint_data(b->transaction_hashes.size()).size() + sizeof(Hash) * b->transaction_hashes.size(); + return; + } // two-phase miner transaction generation: we don't know exact block size - // until we prepare block, but we don't know - // reward until we know + // until we prepare block, but we don't know reward until we know // block size, so first miner transaction generated with fake amount of money, - // and with phase we know think we know - // expected block size - // make blocks coin-base tx looks close to real coinbase tx to get truthful - // blob size - - // bool r = m_currency.construct_miner_tx(b->major_version, *height, effective_size_median, - // already_generated_coins, - // txs_size, txs_fee, m_config.mineproof_secret, adr, &b->base_transaction, extra_nonce, 11); - // if (!r) { - // m_log(logging::ERROR) << logging::BrightRed << "Failed to construct miner tx, first chance"; - // return false; - // } - + // and with phase we know think we know expected block size + // make blocks coin-base tx looks close to real coinbase tx to get truthful blob size size_t cumulative_size = txs_size; const size_t TRIES_COUNT = 11; for (size_t try_count = 0; try_count < TRIES_COUNT; ++try_count) { - m_currency.construct_miner_tx(b->major_version, *height, effective_size_median, already_generated_coins, - cumulative_size, txs_fee, m_config.mineproof_secret, adr, &b->base_transaction, extra_nonce); - size_t coinbase_blob_size = seria::binary_size(b->base_transaction); + Amount block_reward = m_currency.get_block_reward(b->major_version, *height, effective_size_median, + cumulative_size, parent_info.already_generated_coins, txs_fee); + b->base_transaction = m_currency.construct_miner_tx(b->major_version, *height, block_reward, adr); + if (!extra_nonce.empty()) + extra_add_nonce(b->base_transaction.extra, extra_nonce); + size_t extra_size_without_delta = b->base_transaction.extra.size(); + size_t coinbase_blob_size = seria::binary_size(b->base_transaction); if (coinbase_blob_size + txs_size > cumulative_size) { cumulative_size = txs_size + coinbase_blob_size; continue; @@ -611,20 +641,16 @@ void BlockChainState::create_mining_block_template(const AccountPublicAddress &a << ", try_count=" << try_count; } } + *reserved_back_offset = + common::get_varint_data(b->transaction_hashes.size()).size() + sizeof(Hash) * b->transaction_hashes.size(); + *reserved_back_offset += b->base_transaction.extra.size() - extra_size_without_delta; invariant(cumulative_size == txs_size + seria::binary_size(b->base_transaction), "miner_tx case 2"); return; } throw std::runtime_error("Failed to create_block_template with " + common::to_string(TRIES_COUNT) + " attempts"); } -uint32_t BlockChainState::get_next_effective_median_size() const { - uint8_t should_be_major = 0, might_be_minor = 0; - fill_next_block_versions(get_tip(), false, &should_be_major, &might_be_minor); - auto next_minimum_size_median = m_currency.get_minimum_size_median(should_be_major); - return std::max(m_next_median_size, next_minimum_size_median); -} - -BroadcastAction BlockChainState::add_mined_block( +bool BlockChainState::add_mined_block( const BinaryArray &raw_block_template, RawBlock *raw_block, api::BlockHeader *info) { BlockTemplate block_template; seria::from_binary(block_template, raw_block_template); @@ -642,7 +668,7 @@ BroadcastAction BlockChainState::add_mined_block( if (tit2 == m_mining_transactions.end()) { m_log(logging::WARNING) << "The transaction " << tx_hash << " is absent in transaction pool on submit mined block"; - return BroadcastAction::NOTHING; + return false; } binary_tx = &(tit2->second.first); } @@ -661,21 +687,26 @@ void BlockChainState::clear_mining_transactions() const { ++tit; } -Amount BlockChainState::minimum_pool_fee_per_byte(Hash *minimal_tid) const { +Amount BlockChainState::minimum_pool_fee_per_byte(bool zero_if_not_full, Hash *minimal_tid) const { if (m_memory_state_fee_tx.empty()) { - *minimal_tid = Hash(); + if (minimal_tid) + *minimal_tid = Hash{}; + return 0; + } + if (zero_if_not_full && m_memory_state_total_size < m_max_pool_size) { + if (minimal_tid) + *minimal_tid = Hash{}; return 0; } auto be = m_memory_state_fee_tx.begin(); - invariant(!be->second.empty(), "Invariant dead, memory_state_fee_tx empty set"); - *minimal_tid = *(be->second.begin()); + if (minimal_tid) + *minimal_tid = be->second; return be->first; } void BlockChainState::on_reorganization( const std::map> &undone_transactions, bool undone_blocks) { // TODO - remove/add only those transactions that could have their referenced output keys changed - Height conflict_height = 0; if (undone_blocks) { PoolTransMap old_memory_state_tx; std::swap(old_memory_state_tx, m_memory_state_tx); @@ -683,89 +714,104 @@ void BlockChainState::on_reorganization( m_memory_state_fee_tx.clear(); m_memory_state_total_size = 0; for (auto &&msf : old_memory_state_tx) { - add_transaction(msf.first, msf.second.tx, msf.second.binary_tx, get_tip_height() + 1, get_tip().timestamp, - &conflict_height, true, std::string()); + try { + add_transaction(msf.first, msf.second.tx, msf.second.binary_tx, true, std::string()); + } catch (const std::exception &) { // Just skip now invalid transactions + } } } for (auto ud : undone_transactions) { - add_transaction(ud.first, ud.second.first, ud.second.second, get_tip_height() + 1, get_tip().timestamp, - &conflict_height, true, std::string()); + try { + add_transaction(ud.first, ud.second.first, ud.second.second, true, std::string()); + } catch (const std::exception &) { // Just skip now invalid transactions + } } m_tx_pool_version = 2; // add_transaction will erroneously increase } -AddTransactionResult BlockChainState::add_transaction(const Hash &tid, const Transaction &tx, - const BinaryArray &binary_tx, Timestamp now, Height *conflict_height, const std::string &source_address) { - // Timestamp g_timestamp = read_first_seen_timestamp(tid); - // if (g_timestamp != 0 && now > g_timestamp + m_config.mempool_tx_live_time) - // return AddTransactionResult::TOO_OLD; - return add_transaction( - tid, tx, binary_tx, get_tip_height() + 1, get_tip().timestamp, conflict_height, true, source_address); +std::vector BlockChainState::sync_pool( + const std::pair &from, const std::pair &to, size_t max_count) const { + std::vector result; + auto sit = m_memory_state_fee_tx.lower_bound(from); + if (sit != m_memory_state_fee_tx.end()) { + if (*sit != from) + ++sit; + } + while (sit != m_memory_state_fee_tx.begin()) { + --sit; + if (result.size() > max_count || *sit <= to) + break; + TransactionDesc desc; + desc.hash = sit->second; + auto tit = m_memory_state_tx.find(desc.hash); + invariant(tit != m_memory_state_tx.end(), ""); + desc.fee = tit->second.fee; + desc.size = tit->second.binary_tx.size(); + desc.newest_referenced_block = tit->second.newest_referenced_block; + result.push_back(desc); + } + return result; } -AddTransactionResult BlockChainState::add_transaction(const Hash &tid, const Transaction &tx, - const BinaryArray &binary_tx, Height unlock_height, Timestamp unlock_timestamp, Height *conflict_height, +bool BlockChainState::add_transaction(const Hash &tid, const Transaction &tx, const BinaryArray &binary_tx, bool check_sigs, const std::string &source_address) { if (m_memory_state_tx.count(tid) != 0) { m_archive.add(Archive::TRANSACTION, binary_tx, tid, source_address); - return AddTransactionResult::ALREADY_IN_POOL; + return false; // AddTransactionResult::ALREADY_IN_POOL; } // std::cout << "add_transaction " << tid << std::endl; const size_t my_size = binary_tx.size(); - const Amount my_fee = bytecoin::get_tx_fee(tx); + const Amount my_fee = cn::get_tx_fee(tx); const Amount my_fee_per_byte = my_fee / my_size; Hash minimal_tid; - Amount minimal_fee = minimum_pool_fee_per_byte(&minimal_tid); + Amount minimal_fee = minimum_pool_fee_per_byte(false, &minimal_tid); // Invariant is if 1 byte of cheapest transaction fits, then all transaction fits - if (m_memory_state_total_size >= MAX_POOL_SIZE && my_fee_per_byte < minimal_fee) - return AddTransactionResult::INCREASE_FEE; + if (m_memory_state_total_size >= m_max_pool_size && my_fee_per_byte < minimal_fee) + return false; // AddTransactionResult::INCREASE_FEE; // Deterministic behaviour here and below so tx pools have tendency to stay the same - if (m_memory_state_total_size >= MAX_POOL_SIZE && my_fee_per_byte == minimal_fee && tid < minimal_tid) - return AddTransactionResult::INCREASE_FEE; + if (m_memory_state_total_size >= m_max_pool_size && my_fee_per_byte == minimal_fee && tid < minimal_tid) + return false; // AddTransactionResult::INCREASE_FEE; for (const auto &input : tx.inputs) { - if (input.type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(input); + if (input.type() == typeid(InputKey)) { + const InputKey &in = boost::get(input); auto tit = m_memory_state_ki_tx.find(in.key_image); if (tit == m_memory_state_ki_tx.end()) continue; const PoolTransaction &other_tx = m_memory_state_tx.at(tit->second); const Amount other_fee_per_byte = other_tx.fee_per_byte(); if (my_fee_per_byte < other_fee_per_byte) - return AddTransactionResult::INCREASE_FEE; + return false; // AddTransactionResult::INCREASE_FEE; if (my_fee_per_byte == other_fee_per_byte && tid < tit->second) - return AddTransactionResult::INCREASE_FEE; + return false; // AddTransactionResult::INCREASE_FEE; break; // Can displace another transaction from the pool, Will have to make heavy-lifting for this tx } } for (const auto &input : tx.inputs) { - if (input.type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(input); - if (read_keyimage(in.key_image, conflict_height)) { - // std::cout << tid << " " << in.key_image << - // std::endl; - // m_log(logging::WARNING) << "OUTPUT_ALREADY_SPENT in transaction " << tid - // << std::endl; // TODO - remove - return AddTransactionResult::OUTPUT_ALREADY_SPENT; // Already spent in main chain + if (input.type() == typeid(InputKey)) { + const InputKey &in = boost::get(input); + Height conflict_height = 0; + if (read_keyimage(in.key_image, &conflict_height)) { + throw ConsensusErrorOutputSpent("Output already spent", in.key_image, conflict_height); } } } - Amount my_fee3 = 0; - const std::string validate_result = validate_semantic(false, tx, &my_fee3, m_config.paranoid_checks || check_sigs); - if (!validate_result.empty()) { - m_log(logging::WARNING) << "add_transaction validation failed " << validate_result << " in transaction " << tid - << std::endl; - return AddTransactionResult::BAN; - } - DeltaState memory_state(unlock_height, unlock_timestamp, this); + const Amount my_fee3 = + validate_semantic(m_currency, get_tip().major_version, false, tx, m_config.paranoid_checks || check_sigs); + // if (!validate_result.empty()) { + // m_log(logging::WARNING) << "add_transaction validation failed " << validate_result << " in transaction " << + // tid << std::endl; + // return AddTransactionResult::BAN; + // } + DeltaState memory_state(get_tip_height() + 1, get_tip().timestamp, get_tip().timestamp_median, this); BlockGlobalIndices global_indices; - const std::string redo_result = - redo_transaction_get_error(false, tx, &memory_state, &global_indices, conflict_height, check_sigs); - if (!redo_result.empty()) { - // std::cout << "Addding anyway for test " << std::endl; - m_log(logging::TRACE) << "add_transaction redo failed " << redo_result << " in transaction " << tid - << std::endl; - return AddTransactionResult::FAILED_TO_REDO; // Not a ban because reorg can change indices - } + Hash newest_referenced_bid; + redo_transaction( + get_tip().major_version, false, tx, &memory_state, &global_indices, &newest_referenced_bid, check_sigs); + // if (!redo_result.empty()) { + // m_log(logging::TRACE) << "add_transaction redo failed " << redo_result << " in transaction " << tid + // << std::endl; + // return AddTransactionResult::FAILED_TO_REDO; // Not a ban because reorg can change indices + // } if (my_fee != my_fee3) m_log(logging::ERROR) << "Inconsistent fees " << my_fee << ", " << my_fee3 << " in transaction " << tid << std::endl; @@ -779,9 +825,9 @@ AddTransactionResult BlockChainState::add_transaction(const Hash &tid, const Tra const PoolTransaction &other_tx = m_memory_state_tx.at(tit->second); const Amount other_fee_per_byte = other_tx.fee_per_byte(); if (my_fee_per_byte < other_fee_per_byte) - return AddTransactionResult::INCREASE_FEE; // Never because checked above + return false; // AddTransactionResult::INCREASE_FEE; // Never because checked above if (my_fee_per_byte == other_fee_per_byte && tid < tit->second) - return AddTransactionResult::INCREASE_FEE; // Never because checked above + return false; // AddTransactionResult::INCREASE_FEE; // Never because checked above remove_from_pool(tit->second); } bool all_inserted = true; @@ -789,31 +835,31 @@ AddTransactionResult BlockChainState::add_transaction(const Hash &tid, const Tra if (!m_memory_state_ki_tx.insert(std::make_pair(ki.first, tid)).second) all_inserted = false; } - if (!m_memory_state_tx.insert(std::make_pair(tid, PoolTransaction(tx, binary_tx, my_fee, 0))) - .second) // TODO set timestamp + const auto now = platform::now_unix_timestamp(); + if (!m_memory_state_tx + .insert(std::make_pair(tid, PoolTransaction(tx, binary_tx, my_fee, now, newest_referenced_bid))) + .second) all_inserted = false; - if (!m_memory_state_fee_tx[my_fee_per_byte].insert(tid).second) + if (!m_memory_state_fee_tx.insert(std::make_pair(my_fee_per_byte, tid)).second) all_inserted = false; // insert all before throw invariant(all_inserted, "memory_state_fee_tx empty"); m_memory_state_total_size += my_size; - while (m_memory_state_total_size > MAX_POOL_SIZE) { + while (m_memory_state_total_size > m_max_pool_size) { invariant(!m_memory_state_fee_tx.empty(), "memory_state_fee_tx empty"); - auto &be = m_memory_state_fee_tx.begin()->second; - invariant(!be.empty(), "memory_state_fee_tx empty set"); - Hash rhash = *(be.begin()); + // auto &be = m_memory_state_fee_tx.begin()->second; + // invariant(!be.empty(), "memory_state_fee_tx empty set"); + Hash rhash = m_memory_state_fee_tx.begin()->second; const PoolTransaction &minimal_tx = m_memory_state_tx.at(rhash); - if (m_memory_state_total_size < MAX_POOL_SIZE + minimal_tx.binary_tx.size()) + if (m_memory_state_total_size < m_max_pool_size + minimal_tx.binary_tx.size()) break; // Removing would diminish pool below max size remove_from_pool(rhash); } - auto min_size = m_memory_state_fee_tx.empty() || m_memory_state_fee_tx.begin()->second.empty() + auto min_size = m_memory_state_fee_tx.empty() ? 0 - : m_memory_state_tx.at(*(m_memory_state_fee_tx.begin()->second.begin())).binary_tx.size(); - auto min_fee_per_byte = m_memory_state_fee_tx.empty() || m_memory_state_fee_tx.begin()->second.empty() - ? 0 - : m_memory_state_fee_tx.begin()->first; - // if( m_memory_state_total_size-min_size >= MAX_POOL_SIZE) + : m_memory_state_tx.at(m_memory_state_fee_tx.begin()->second).binary_tx.size(); + auto min_fee_per_byte = m_memory_state_fee_tx.empty() ? 0 : m_memory_state_fee_tx.begin()->first; + // if( m_memory_state_total_size-min_size >= m_max_pool_size) // std::cout << "Aha" << std::endl; m_log(logging::INFO) << "Added transaction with hash=" << tid << " size=" << my_size << " fee=" << my_fee << " fee/byte=" << my_fee_per_byte << " current_pool_size=(" @@ -827,18 +873,18 @@ AddTransactionResult BlockChainState::add_transaction(const Hash &tid, const Tra // common::pod_to_hex(ff) << std::endl; // } m_tx_pool_version += 1; - return AddTransactionResult::BROADCAST_ALL; + return true; } bool BlockChainState::get_largest_referenced_height(const TransactionPrefix &transaction, Height *block_height) const { - std::map largest_indices; // Do not check same used amount twice + std::map largest_indices; // Do not check same used amount twice size_t input_index = 0; for (const auto &input : transaction.inputs) { - if (input.type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(input); + if (input.type() == typeid(InputKey)) { + const InputKey &in = boost::get(input); if (in.output_indexes.empty()) return false; // semantics invalid - uint32_t largest_index = in.output_indexes[0]; + size_t largest_index = in.output_indexes[0]; for (size_t i = 1; i < in.output_indexes.size(); ++i) { largest_index = largest_index + in.output_indexes[i]; } @@ -866,29 +912,27 @@ void BlockChainState::remove_from_pool(Hash tid) { bool all_erased = true; const Transaction &tx = tit->second.tx; for (const auto &input : tx.inputs) { - if (input.type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(input); + if (input.type() == typeid(InputKey)) { + const InputKey &in = boost::get(input); if (m_memory_state_ki_tx.erase(in.key_image) != 1) all_erased = false; } } const size_t my_size = tit->second.binary_tx.size(); const Amount my_fee_per_byte = tit->second.fee_per_byte(); - if (m_memory_state_fee_tx[my_fee_per_byte].erase(tid) != 1) + if (m_memory_state_fee_tx.erase(std::make_pair(my_fee_per_byte, tid)) != 1) all_erased = false; - if (m_memory_state_fee_tx[my_fee_per_byte].empty()) - m_memory_state_fee_tx.erase(my_fee_per_byte); + // if (m_memory_state_fee_tx[my_fee_per_byte].empty()) + // m_memory_state_fee_tx.erase(my_fee_per_byte); m_memory_state_total_size -= my_size; m_memory_state_tx.erase(tit); invariant(all_erased, "remove_memory_pool failed to erase everything"); // We do not increment m_tx_pool_version, because removing tx from pool is // always followed by reset or increment - auto min_size = m_memory_state_fee_tx.empty() || m_memory_state_fee_tx.begin()->second.empty() + auto min_size = m_memory_state_fee_tx.empty() ? 0 - : m_memory_state_tx.at(*(m_memory_state_fee_tx.begin()->second.begin())).binary_tx.size(); - auto min_fee_per_byte = m_memory_state_fee_tx.empty() || m_memory_state_fee_tx.begin()->second.empty() - ? 0 - : m_memory_state_fee_tx.begin()->first; + : m_memory_state_tx.at(m_memory_state_fee_tx.begin()->second).binary_tx.size(); + auto min_fee_per_byte = m_memory_state_fee_tx.empty() ? 0 : m_memory_state_fee_tx.begin()->first; m_log(logging::INFO) << "Removed transaction with hash=" << tid << " size=" << my_size << " current_pool_size=(" << m_memory_state_total_size - min_size << "+" << min_size << ")=" << m_memory_state_total_size << " count=" << m_memory_state_tx.size() << " min fee/byte=" << min_fee_per_byte << std::endl; @@ -900,134 +944,149 @@ void BlockChainState::remove_from_pool(Hash tid) { // if output not found, conflict height is set to currency max_block_height // if no error, conflict_height is set to newest referenced height, (for coinbase transaction to 0) -std::string BlockChainState::redo_transaction_get_error(bool generating, const Transaction &transaction, - DeltaState *delta_state, BlockGlobalIndices *global_indices, Height *conflict_height, bool check_sigs) const { +void BlockChainState::redo_transaction(uint8_t major_block_version, bool generating, const Transaction &transaction, + DeltaState *delta_state, BlockGlobalIndices *global_indices, Hash *newest_referenced_bid, bool check_sigs) const { const bool check_outputs = check_sigs; Hash tx_prefix_hash; if (m_config.paranoid_checks || check_sigs) tx_prefix_hash = get_transaction_prefix_hash(transaction); - DeltaState tx_delta(delta_state->get_block_height(), delta_state->get_unlock_timestamp(), delta_state); + DeltaState tx_delta(delta_state->get_block_height(), delta_state->get_block_timestamp(), + delta_state->get_block_median_timestamp(), delta_state); global_indices->resize(global_indices->size() + 1); auto &my_indices = global_indices->back(); my_indices.reserve(transaction.outputs.size()); - *conflict_height = 0; - size_t input_index = 0; - for (const auto &input : transaction.inputs) { - if (input.type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(input); + Height newest_referenced_height = 0; + std::vector> all_output_keys; // For half-size sigs + std::vector all_keyimages; // For half-size sigs + for (size_t input_index = 0; input_index != transaction.inputs.size(); ++input_index) { + const auto &input = transaction.inputs.at(input_index); + if (input.type() == typeid(InputKey)) { + const InputKey &in = boost::get(input); - if (m_config.paranoid_checks || check_sigs || check_outputs) { + if (m_config.paranoid_checks || check_sigs || check_outputs || newest_referenced_bid) { Height height = 0; - if (tx_delta.read_keyimage(in.key_image, &height)) { - *conflict_height = height; - return "INPUT_KEYIMAGE_ALREADY_SPENT"; - } - if (in.output_indexes.empty()) - return "INPUT_UNKNOWN_TYPE"; // Never - checked in validate_semantic - std::vector global_indexes(in.output_indexes.size()); - global_indexes[0] = in.output_indexes[0]; - for (size_t i = 1; i < in.output_indexes.size(); ++i) { - global_indexes[i] = global_indexes[i - 1] + in.output_indexes[i]; - } + if (tx_delta.read_keyimage(in.key_image, &height)) + throw ConsensusErrorOutputSpent("Output already spent", in.key_image, height); + // if (in.output_indexes.size() < m_currency.minimum_anonymity(major_block_version) + 1 && + // !m_currency.is_dust(in.amount)) { + // if (m_currency.net == "main") + // throw ConsensusError("Anonymity too low"); + // In test/stage net we lack enough coins of each non-dust denomination + // } + std::vector global_indexes; + if (!relative_output_offsets_to_absolute(&global_indexes, in.output_indexes)) + throw ConsensusError("Output indexes invalid in input"); std::vector output_keys(global_indexes.size()); for (size_t i = 0; i != global_indexes.size(); ++i) { UnlockTimePublickKeyHeightSpent unp; - if (!tx_delta.read_amount_output(in.amount, global_indexes[i], &unp)) { - *conflict_height = m_currency.max_block_height; - return "INPUT_INVALID_GLOBAL_INDEX"; - } - *conflict_height = std::max(*conflict_height, unp.height); - if (!m_currency.is_transaction_spend_time_unlocked(unp.unlock_block_or_timestamp, - delta_state->get_block_height(), delta_state->get_unlock_timestamp())) - return "INPUT_SPEND_LOCKED_OUT"; - output_keys[i] = unp.public_key; + if (!tx_delta.read_amount_output(in.amount, global_indexes[i], &unp)) + throw ConsensusErrorOutputDoesNotExist("Output does not exist", input_index, global_indexes[i]); + if (unp.auditable && global_indexes.size() != 1) + throw ConsensusErrorBadOutputOrSignature("Auditable output mixed", unp.height); + if (!m_currency.is_transaction_unlocked(major_block_version, unp.unlock_block_or_timestamp, + delta_state->get_block_height(), delta_state->get_block_timestamp(), + delta_state->get_block_median_timestamp())) + throw ConsensusErrorBadOutputOrSignature("Output locked", unp.height); + output_keys[i] = unp.public_key; + newest_referenced_height = std::max(newest_referenced_height, unp.height); } - std::vector output_key_pointers; - output_key_pointers.reserve(output_keys.size()); - std::for_each(output_keys.begin(), output_keys.end(), - [&output_key_pointers](const PublicKey &key) { output_key_pointers.push_back(&key); }); - bool key_corrupted = false; - if ((m_config.paranoid_checks || check_sigs) && - !check_ring_signature(tx_prefix_hash, in.key_image, output_key_pointers.data(), - output_key_pointers.size(), transaction.signatures[input_index].data(), - delta_state->get_block_height() >= m_currency.key_image_subgroup_checking_height, - &key_corrupted)) { - if (key_corrupted) // TODO - db corrupted - return "INPUT_CORRUPTED_SIGNATURES"; - return "INPUT_INVALID_SIGNATURES"; + if (m_config.paranoid_checks || check_sigs) { + if (transaction.signatures.type() == typeid(RingSignatures)) { + auto &signatures = boost::get(transaction.signatures); + // std::vector output_key_pointers; + // output_key_pointers.reserve(output_keys.size()); + // std::for_each(output_keys.begin(), output_keys.end(), + // [&output_key_pointers](const PublicKey &key) { + // output_key_pointers.push_back(&key); }); + if (!check_ring_signature(tx_prefix_hash, in.key_image, output_keys.data(), output_keys.size(), + signatures.signatures.at(input_index), + delta_state->get_block_height() >= m_currency.key_image_subgroup_checking_height)) { + throw ConsensusErrorBadOutputOrSignature{ + "Bad signature or output reference changed", newest_referenced_height}; + } + } else if (transaction.signatures.type() == typeid(RingSignature3)) { + all_output_keys.push_back(std::move(output_keys)); + all_keyimages.push_back(in.key_image); + } else + throw ConsensusError("Unknown signatures type"); } } - if (in.output_indexes.size() == 1) - tx_delta.spend_output(in.amount, in.output_indexes[0]); tx_delta.store_keyimage(in.key_image, delta_state->get_block_height()); } - input_index++; + } + if (!all_output_keys.empty()) { + invariant(transaction.signatures.type() == typeid(RingSignature3), ""); + auto &signatures = boost::get(transaction.signatures); + if (!crypto::check_ring_signature3(tx_prefix_hash, all_keyimages, all_output_keys, signatures)) + throw ConsensusErrorBadOutputOrSignature{ + "Bad signature or output reference changed", newest_referenced_height}; + } + if (newest_referenced_bid) { + // get_chain cannot fail if got all corresponding output keys successfully + invariant(get_chain(newest_referenced_height, newest_referenced_bid), ""); } for (const auto &output : transaction.outputs) { - if (output.target.type() == typeid(KeyOutput)) { - const KeyOutput &key_output = boost::get(output.target); - auto global_index = tx_delta.push_amount_output(output.amount, transaction.unlock_block_or_timestamp, 0, - key_output.public_key); // DeltaState ignores unlock point + if (output.type() == typeid(OutputKey)) { + const auto &key_output = boost::get(output); + auto global_index = tx_delta.push_amount_output(key_output.amount, transaction.unlock_block_or_timestamp, 0, + key_output.public_key, key_output.is_auditable); // DeltaState ignores unlock point my_indices.push_back(global_index); } } tx_delta.apply(delta_state); // delta_state might be memory pool, we protect it from half-modification - return std::string(); } void BlockChainState::undo_transaction(IBlockChainState *delta_state, Height, const Transaction &tx) { for (auto oit = tx.outputs.rbegin(); oit != tx.outputs.rend(); ++oit) { - if (oit->target.type() == typeid(KeyOutput)) { + if (oit->type() == typeid(OutputKey)) { + const auto &key_output = boost::get(*oit); delta_state->pop_amount_output( - oit->amount, tx.unlock_block_or_timestamp, boost::get(oit->target).public_key); + key_output.amount, tx.unlock_block_or_timestamp, key_output.public_key, key_output.is_auditable); } } for (auto iit = tx.inputs.rbegin(); iit != tx.inputs.rend(); ++iit) { - if (iit->type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(*iit); + if (iit->type() == typeid(InputKey)) { + const InputKey &in = boost::get(*iit); + unprocess_input(in); delta_state->delete_keyimage(in.key_image); - if (in.output_indexes.size() == 1) - spend_output(in.amount, in.output_indexes[0], false); } } } -bool BlockChainState::redo_block(const Block &block, +void BlockChainState::redo_block(const Block &block, const api::BlockHeader &info, DeltaState *delta_state, BlockGlobalIndices *global_indices) const { - Height conflict_height; - std::string result = redo_transaction_get_error( - true, block.header.base_transaction, delta_state, global_indices, &conflict_height, false); - if (!result.empty()) - return false; + redo_transaction( + block.header.major_version, true, block.header.base_transaction, delta_state, global_indices, nullptr, false); for (auto tit = block.transactions.begin(); tit != block.transactions.end(); ++tit) { - std::string result = - redo_transaction_get_error(false, *tit, delta_state, global_indices, &conflict_height, false); - if (!result.empty()) - return false; + redo_transaction(block.header.major_version, false, *tit, delta_state, global_indices, nullptr, false); } - return true; } -bool BlockChainState::redo_block(const Hash &bhash, const Block &block, const api::BlockHeader &info) { - DeltaState delta(info.height, info.timestamp, this); +void BlockChainState::redo_block(const Hash &bhash, const Block &block, const api::BlockHeader &info) { + DeltaState delta(info.height, info.timestamp, info.timestamp_median, this); BlockGlobalIndices global_indices; global_indices.reserve(block.transactions.size() + 1); - const bool check_sigs = m_config.paranoid_checks || !m_currency.is_in_sw_checkpoint_zone(info.height + 1); - if (check_sigs && - !ring_checker - .start_work_get_error(this, m_currency, block, info.height, info.timestamp, - info.height >= m_currency.key_image_subgroup_checking_height) - .empty()) - return false; - if (!redo_block(block, info, &delta, &global_indices)) - return false; - if (check_sigs && !ring_checker.signatures_valid()) - return false; + const bool check_sigs = m_config.paranoid_checks || !m_currency.is_in_hard_checkpoint_zone(info.height + 1); + if (check_sigs) + m_ring_checker.start_work(this, m_currency, block, info.height, info.timestamp, info.timestamp_median, + info.height >= m_currency.key_image_subgroup_checking_height); + redo_block(block, info, &delta, &global_indices); + if (check_sigs) { + auto errors = m_ring_checker.move_errors(); + if (!errors.empty()) + throw errors.front(); // We report first error only + } delta.apply(this); // Will remove from pool by key_image + for (auto tit = block.transactions.begin(); tit != block.transactions.end(); ++tit) { + for (size_t input_index = 0; input_index != tit->inputs.size(); ++input_index) + if (tit->inputs.at(input_index).type() == typeid(InputKey)) + process_input(block.header.transaction_hashes.at(tit - block.transactions.begin()), input_index, + boost::get(tit->inputs.at(input_index))); + } m_tx_pool_version = 2; auto key = @@ -1035,13 +1094,10 @@ bool BlockChainState::redo_block(const Hash &bhash, const Block &block, const ap BinaryArray ba = seria::to_binary(global_indices); m_db.put(key, ba, true); - // for (auto th : block.header.transaction_hashes) { - // update_first_seen_timestamp(th, 0); - // } auto now = std::chrono::steady_clock::now(); if (m_config.net != "main" || - std::chrono::duration_cast(now - log_redo_block_timestamp).count() > 1000) { - log_redo_block_timestamp = now; + std::chrono::duration_cast(now - m_log_redo_block_timestamp).count() > 1000) { + m_log_redo_block_timestamp = now; m_log(logging::INFO) << "redo_block height=" << info.height << " bid=" << bhash << " #tx=" << block.transactions.size() << std::endl; } else { @@ -1049,15 +1105,20 @@ bool BlockChainState::redo_block(const Hash &bhash, const Block &block, const ap m_log(logging::TRACE) << "redo_block height=" << info.height << " bid=" << bhash << " #tx=" << block.transactions.size() << std::endl; } - return true; } void BlockChainState::undo_block(const Hash &bhash, const Block &block, Height height) { - // if (height % 100 == 0) - // std::cout << "undo_block height=" << height << " bid=" << bhash - // << " new tip_bid=" << block.header.previous_block_hash << std::endl; - m_log(logging::INFO) << "undo_block height=" << height << " bid=" << bhash - << " new tip_bid=" << block.header.previous_block_hash << std::endl; + auto now = std::chrono::steady_clock::now(); + if (m_config.net != "main" || + std::chrono::duration_cast(now - m_log_redo_block_timestamp).count() > 1000) { + m_log_redo_block_timestamp = now; + m_log(logging::INFO) << "undo_block height=" << height << " bid=" << bhash + << " new tip_bid=" << block.header.previous_block_hash << std::endl; + } else { + if (m_config.paranoid_checks) + m_log(logging::TRACE) << "undo_block height=" << height << " bid=" << bhash + << " new tip_bid=" << block.header.previous_block_hash << std::endl; + } for (auto tit = block.transactions.rbegin(); tit != block.transactions.rend(); ++tit) { undo_transaction(this, height, *tit); } @@ -1078,29 +1139,30 @@ bool BlockChainState::read_block_output_global_indices(const Hash &bid, BlockGlo return true; } -std::vector BlockChainState::get_random_outputs( - Amount amount, size_t output_count, Height confirmed_height, Timestamp time) const { +std::vector BlockChainState::get_random_outputs(uint8_t block_major_version, Amount amount, + size_t output_count, Height confirmed_height, Timestamp block_timestamp, Timestamp block_median_timestamp) const { std::vector result; - uint32_t total_count = next_global_index_for_amount(amount); + std::vector spent_result; + size_t total_count = next_global_index_for_amount(amount); // We might need better algorithm if we have lots of locked amounts - std::set tried_or_added; + std::set tried_or_added; crypto::random_engine generator; std::lognormal_distribution distribution(1.9, 1.0); // Magic params here const uint32_t linear_part = 150; // Magic params here size_t attempts = 0; for (; result.size() < output_count && attempts < output_count * 20; ++attempts) { // TODO - 20 - uint32_t num = 0; + size_t num = 0; if (result.size() % 2 == 0) { // Half of outputs linear if (total_count <= linear_part) - num = crypto::rand() % total_count; // 0 handled above + num = crypto::rand() % total_count; // 0 handled above else - num = total_count - 1 - crypto::rand() % linear_part; + num = total_count - 1 - crypto::rand() % linear_part; } else { double sample = distribution(generator); int d_num = static_cast(std::floor(total_count * (1 - std::pow(10, -sample / 10)))); if (d_num < 0 || d_num >= int(total_count)) continue; - num = static_cast(d_num); + num = static_cast(d_num); } if (!tried_or_added.insert(num).second) continue; @@ -1114,22 +1176,26 @@ std::vector BlockChainState::get_random_outputs( // to get descent results after small number of attempts continue; } - if (unp.spent) + if (unp.auditable) continue; - if (!m_currency.is_transaction_spend_time_unlocked(unp.unlock_block_or_timestamp, confirmed_height, time)) + if (!m_currency.is_transaction_unlocked(block_major_version, unp.unlock_block_or_timestamp, confirmed_height, + block_timestamp, block_median_timestamp)) continue; + if (unp.spent && spent_result.size() >= output_count) + continue; // We need only so much spent api::Output item; - item.amount = amount; - item.index = num; + item.amount = amount; + item.index = num; item.unlock_block_or_timestamp = unp.unlock_block_or_timestamp; item.public_key = unp.public_key; item.height = unp.height; - result.push_back(item); + (unp.spent ? spent_result : result).push_back(item); } - if(result.size() < output_count){ + if (result.size() < output_count) { // Read the whole index. - size_t attempts = 0; - for (DB::Cursor cur = m_db.rbegin(AMOUNT_OUTPUT_PREFIX + common::write_varint_sqlite4(amount)); result.size() < output_count && attempts < 10000 && !cur.end(); cur.next(), ++attempts) { // TODO - 10000 + size_t attempts = 0; + for (DB::Cursor cur = m_db.rbegin(AMOUNT_OUTPUT_PREFIX + common::write_varint_sqlite4(amount)); + result.size() < output_count && attempts < 10000 && !cur.end(); cur.next(), ++attempts) { // TODO - 10000 const char *be = cur.get_suffix().data(); const char *en = be + cur.get_suffix().size(); uint32_t global_index = common::integer_cast(common::read_varint_sqlite4(be, en)); @@ -1137,17 +1203,25 @@ std::vector BlockChainState::get_random_outputs( continue; UnlockTimePublickKeyHeightSpent unp; seria::from_binary(unp, cur.get_value_array()); - if (unp.spent || unp.height > confirmed_height) + if (unp.auditable || unp.height > confirmed_height) continue; - if (!m_currency.is_transaction_spend_time_unlocked(unp.unlock_block_or_timestamp, confirmed_height, time)) + if (!m_currency.is_transaction_unlocked(block_major_version, unp.unlock_block_or_timestamp, + confirmed_height, block_timestamp, block_median_timestamp)) continue; + if (unp.spent && spent_result.size() >= output_count) + continue; // We need only so much spent api::Output item; - item.amount = amount; - item.index = global_index; + item.amount = amount; + item.index = global_index; item.unlock_block_or_timestamp = unp.unlock_block_or_timestamp; item.public_key = unp.public_key; item.height = unp.height; - result.push_back(item); + (unp.spent ? spent_result : result).push_back(item); + } + // To satisfy minimum anonymity requirement for very rare coins, we add spent as a last resort + while (result.size() < output_count && !spent_result.empty()) { + result.push_back(spent_result.back()); + spent_result.pop_back(); } } return result; @@ -1176,18 +1250,20 @@ bool BlockChainState::read_keyimage(const KeyImage &key_image, Height *height) c return true; } -uint32_t BlockChainState::push_amount_output( - Amount amount, BlockOrTimestamp unlock_time, Height block_height, const PublicKey &pk) { - uint32_t my_gi = next_global_index_for_amount(amount); - auto key = AMOUNT_OUTPUT_PREFIX + common::write_varint_sqlite4(amount) + common::write_varint_sqlite4(my_gi); - BinaryArray ba = seria::to_binary(UnlockTimePublickKeyHeightSpent{unlock_time, pk, block_height}); +size_t BlockChainState::push_amount_output( + Amount amount, BlockOrTimestamp unlock_time, Height block_height, const PublicKey &pk, bool is_auditable) { + auto my_gi = next_global_index_for_amount(amount); + auto key = AMOUNT_OUTPUT_PREFIX + common::write_varint_sqlite4(amount) + common::write_varint_sqlite4(my_gi); + BinaryArray ba = + seria::to_binary(UnlockTimePublickKeyHeightSpent{unlock_time, pk, block_height, is_auditable, false, {}}); m_db.put(key, ba, true); m_next_gi_for_amount[amount] += 1; return my_gi; } -void BlockChainState::pop_amount_output(Amount amount, BlockOrTimestamp unlock_time, const PublicKey &pk) { - uint32_t next_gi = next_global_index_for_amount(amount); +void BlockChainState::pop_amount_output( + Amount amount, BlockOrTimestamp unlock_time, const PublicKey &pk, bool is_auditable) { + auto next_gi = next_global_index_for_amount(amount); invariant(next_gi != 0, "BlockChainState::pop_amount_output underflow"); next_gi -= 1; m_next_gi_for_amount[amount] -= 1; @@ -1195,25 +1271,25 @@ void BlockChainState::pop_amount_output(Amount amount, BlockOrTimestamp unlock_t UnlockTimePublickKeyHeightSpent unp; invariant(read_amount_output(amount, next_gi, &unp), "BlockChainState::pop_amount_output element does not exist"); - // TODO - check also was_height after upgrade to version 4 ? - invariant(!unp.spent && unp.unlock_block_or_timestamp == unlock_time && unp.public_key == pk, + invariant(!unp.spent && unp.unlock_block_or_timestamp == unlock_time && unp.public_key == pk && + unp.auditable == is_auditable, "BlockChainState::pop_amount_output popping wrong element"); m_db.del(key, true); } -uint32_t BlockChainState::next_global_index_for_amount(Amount amount) const { +size_t BlockChainState::next_global_index_for_amount(Amount amount) const { auto it = m_next_gi_for_amount.find(amount); if (it != m_next_gi_for_amount.end()) return it->second; std::string prefix = AMOUNT_OUTPUT_PREFIX + common::write_varint_sqlite4(amount); DB::Cursor cur2 = m_db.rbegin(prefix); - uint32_t alt_in = cur2.end() ? 0 : common::integer_cast(common::read_varint_sqlite4(cur2.get_suffix())) + 1; + size_t alt_in = cur2.end() ? 0 : common::integer_cast(common::read_varint_sqlite4(cur2.get_suffix())) + 1; m_next_gi_for_amount[amount] = alt_in; return alt_in; } bool BlockChainState::read_amount_output( - Amount amount, uint32_t global_index, UnlockTimePublickKeyHeightSpent *unp) const { + Amount amount, size_t global_index, UnlockTimePublickKeyHeightSpent *unp) const { auto key = AMOUNT_OUTPUT_PREFIX + common::write_varint_sqlite4(amount) + common::write_varint_sqlite4(global_index); BinaryArray rb; if (!m_db.get(key, rb)) @@ -1222,28 +1298,154 @@ bool BlockChainState::read_amount_output( return true; } -void BlockChainState::spend_output(Amount amount, uint32_t global_index) { spend_output(amount, global_index, true); } -void BlockChainState::spend_output(Amount amount, uint32_t global_index, bool spent) { +void BlockChainState::process_input(const Hash &tid, size_t iid, const InputKey &input) { + if (chain_reaction == 0) + return; + if (input.output_indexes.size() == 1) { + UnlockTimePublickKeyHeightSpent unp; + invariant(read_amount_output(input.amount, input.output_indexes.at(0), &unp), ""); + spend_output( + std::move(unp), input.amount, input.output_indexes.at(0), std::numeric_limits::max(), 0, true); + return; + } + if (chain_reaction == 1) + return; + const auto input_index = m_next_nz_input_index; + auto din_key = DIN_PREFIX + common::write_varint_sqlite4(m_next_nz_input_index); + m_next_nz_input_index += 1; + std::vector global_indexes; + invariant(relative_output_offsets_to_absolute(&global_indexes, input.output_indexes), ""); + InputDesc din; + std::vector unspents; + for (size_t i = 0; i != global_indexes.size(); ++i) { + const auto global_index = global_indexes.at(i); + UnlockTimePublickKeyHeightSpent unp; + invariant(read_amount_output(input.amount, global_index, &unp), ""); + if (unp.spent) + continue; + din.first.push_back(global_index); + unspents.push_back(std::move(unp)); + } + if (din.first.size() > 1) + for (size_t i = 0; i != din.first.size(); ++i) { + unspents[i].dins.push_back(input_index); + auto key = AMOUNT_OUTPUT_PREFIX + common::write_varint_sqlite4(input.amount) + + common::write_varint_sqlite4(din.first[i]); + m_db.put(key, seria::to_binary(unspents[i]), false); + } + m_db.put(din_key, seria::to_binary(din), true); + if (din.first.size() == 1) { + spend_output(std::move(unspents[0]), input.amount, din.first[0], input_index, 0, true); + } +} + +void BlockChainState::unprocess_input(const InputKey &input) { + if (chain_reaction == 0) + return; + if (input.output_indexes.size() == 1) { + UnlockTimePublickKeyHeightSpent unp; + invariant(read_amount_output(input.amount, input.output_indexes.at(0), &unp), ""); + spend_output( + std::move(unp), input.amount, input.output_indexes.at(0), std::numeric_limits::max(), 0, false); + return; + } + if (chain_reaction == 1) + return; + m_next_nz_input_index -= 1; + const auto input_index = m_next_nz_input_index; + auto din_key = DIN_PREFIX + common::write_varint_sqlite4(input_index); + BinaryArray din_ba; + invariant(m_db.get(din_key, din_ba), ""); + InputDesc din; + seria::from_binary(din, din_ba); + invariant(din.second.empty(), ""); + if (din.first.size() > 1) + for (auto global_index : din.first) { + UnlockTimePublickKeyHeightSpent unp; + invariant(read_amount_output(input.amount, global_index, &unp), ""); + invariant(!unp.dins.empty() && unp.dins.back() == input_index, ""); + unp.dins.pop_back(); + auto key = AMOUNT_OUTPUT_PREFIX + common::write_varint_sqlite4(input.amount) + + common::write_varint_sqlite4(global_index); + m_db.put(key, seria::to_binary(unp), false); + } + m_db.del(din_key, true); + if (din.first.size() == 1) { + UnlockTimePublickKeyHeightSpent unp; + invariant(read_amount_output(input.amount, din.first[0], &unp), ""); + spend_output(std::move(unp), input.amount, din.first[0], input_index, 0, false); + } +} + +void BlockChainState::spend_output(UnlockTimePublickKeyHeightSpent &&output, Amount amount, size_t global_index, + size_t trigger_input_index, size_t level, bool spent) { + if (level > 2) + std::cout << "Sure spent level=" << level << " am:gi=" << amount << ":" << global_index << std::endl; auto key = AMOUNT_OUTPUT_PREFIX + common::write_varint_sqlite4(amount) + common::write_varint_sqlite4(global_index); - BinaryArray rb; - if (!m_db.get(key, rb)) + bool no_subgroup_check_aftermath = + (amount == 6299999999000000 && global_index == 0) || (amount == 18899999999000000 && global_index == 0); + if (spent) { + invariant(no_subgroup_check_aftermath || output.spent == 0, ""); + output.spent += 1; + } else { + invariant(no_subgroup_check_aftermath || output.spent == 1, ""); + output.spent -= 1; + } + m_db.put(key, seria::to_binary(output), false); + if (spent && output.spent > 1) return; - UnlockTimePublickKeyHeightSpent was; - seria::from_binary(was, rb); - was.spent = spent; - m_db.put(key, seria::to_binary(was), false); + if (!spent && output.spent > 0) + return; + if (!spent) // Not the fastest code, but undo is rare + std::reverse(output.dins.begin(), output.dins.end()); + for (auto input_index : output.dins) { + if (input_index == trigger_input_index) + continue; + auto din_key = DIN_PREFIX + common::write_varint_sqlite4(input_index); + BinaryArray din_ba; + invariant(m_db.get(din_key, din_ba), ""); + InputDesc din; + seria::from_binary(din, din_ba); + size_t only_index = std::numeric_limits::max(); + if (spent) { + size_t found_index = std::lower_bound(din.first.begin(), din.first.end(), global_index) - din.first.begin(); + invariant(found_index != din.first.size() && din.first.at(found_index) == global_index, ""); + din.first.erase(din.first.begin() + found_index); + din.second.push_back(global_index); + // std::cout << "Removing spent " << amount << ":" << global_index << " from " << din.tid << ":" << + // din.index << std::endl; + if (din.first.size() == 1) { + only_index = din.first.back(); + } + } else { + if (din.first.size() == 1) { + only_index = din.first.back(); + } + invariant(!din.second.empty() && din.second.back() == global_index, ""); + din.second.pop_back(); + size_t insert_index = + std::lower_bound(din.first.begin(), din.first.end(), global_index) - din.first.begin(); + din.first.insert(din.first.begin() + insert_index, global_index); + } + m_db.put(din_key, seria::to_binary(din), false); + if (only_index != std::numeric_limits::max()) { + UnlockTimePublickKeyHeightSpent unp; + invariant(read_amount_output(amount, only_index, &unp), ""); + spend_output(std::move(unp), amount, only_index, input_index, level + 1, spent); + } + } } void BlockChainState::test_print_outputs() { - Amount previous_amount = (Amount)-1; - uint32_t next_global_index = 0; - int total_counter = 0; - std::map coins; + Amount previous_amount = (Amount)-1; + size_t next_global_index = 0; + int total_counter = 0; + std::map coins; for (DB::Cursor cur = m_db.begin(AMOUNT_OUTPUT_PREFIX); !cur.end(); cur.next()) { - const char *be = cur.get_suffix().data(); - const char *en = be + cur.get_suffix().size(); - auto amount = common::read_varint_sqlite4(be, en); - uint32_t global_index = common::integer_cast(common::read_varint_sqlite4(be, en)); + const char *be = cur.get_suffix().data(); + const char *en = be + cur.get_suffix().size(); + auto amount = common::read_varint_sqlite4(be, en); + size_t global_index = common::integer_cast(common::read_varint_sqlite4(be, en)); if (be != en) std::cout << "Excess value bytes for amount=" << amount << " index=" << global_index << std::endl; if (amount != previous_amount) { @@ -1270,7 +1472,7 @@ void BlockChainState::test_print_outputs() { if (total_count != co.second) std::cout << "Wrong next_global_index_for_amount amount=" << co.first << " total_count=" << total_count << " should be " << co.second << std::endl; - for (uint32_t i = 0; i != total_count; ++i) { + for (size_t i = 0; i != total_count; ++i) { UnlockTimePublickKeyHeightSpent unp; if (!read_amount_output(co.first, i, &unp)) std::cout << "Failed to read amount=" << co.first << " index=" << i << std::endl; diff --git a/src/Core/BlockChainState.hpp b/src/Core/BlockChainState.hpp index 644d8382..3c0fc202 100644 --- a/src/Core/BlockChainState.hpp +++ b/src/Core/BlockChainState.hpp @@ -5,12 +5,11 @@ #include #include -#include #include "BlockChain.hpp" #include "Multicore.hpp" #include "crypto/hash.hpp" -namespace bytecoin { +namespace cn { class Config; @@ -19,80 +18,89 @@ class IBlockChainState { struct UnlockTimePublickKeyHeightSpent { BlockOrTimestamp unlock_block_or_timestamp = 0; PublicKey public_key; - Height height = 0; - bool spent = false; + Height height = 0; + bool auditable = false; + uint8_t spent = 0; // Aftermath of "keyimage out of subgroup" attack + std::vector dins; }; - virtual ~IBlockChainState() {} - virtual void store_keyimage(const KeyImage &, Height) = 0; - virtual void delete_keyimage(const KeyImage &) = 0; + virtual ~IBlockChainState() = default; + virtual void store_keyimage(const KeyImage &, Height) = 0; + virtual void delete_keyimage(const KeyImage &) = 0; virtual bool read_keyimage(const KeyImage &, Height *) const = 0; - virtual uint32_t push_amount_output(Amount, BlockOrTimestamp, Height, const PublicKey &) = 0; - virtual void pop_amount_output(Amount, BlockOrTimestamp, const PublicKey &) = 0; - virtual uint32_t next_global_index_for_amount(Amount) const = 0; - virtual bool read_amount_output(Amount, uint32_t global_index, UnlockTimePublickKeyHeightSpent *) const = 0; - virtual void spend_output(Amount, uint32_t global_index) = 0; + virtual size_t push_amount_output(Amount, BlockOrTimestamp, Height, const PublicKey &, bool is_auditable) = 0; + virtual void pop_amount_output(Amount, BlockOrTimestamp, const PublicKey &, bool is_auditable) = 0; + virtual size_t next_global_index_for_amount(Amount) const = 0; + virtual bool read_amount_output(Amount, size_t global_index, UnlockTimePublickKeyHeightSpent *) const = 0; }; class BlockChainState : public BlockChain, private IBlockChainState { public: BlockChainState(logging::ILogger &, const Config &, const Currency &, bool read_only); - uint32_t get_next_effective_median_size() const; - - std::vector get_random_outputs(Amount, size_t output_count, Height, Timestamp) const; - typedef std::vector> BlockGlobalIndices; + std::vector get_random_outputs(uint8_t block_major_version, Amount, size_t output_count, Height, + Timestamp block_timestamp, Timestamp block_median_timestamp) const; + typedef std::vector> BlockGlobalIndices; bool read_block_output_global_indices(const Hash &bid, BlockGlobalIndices *) const; - Amount minimum_pool_fee_per_byte(Hash *minimal_tid) const; - AddTransactionResult add_transaction(const Hash &tid, const Transaction &, const BinaryArray &binary_tx, - Timestamp now, Height *conflict_height, const std::string &source_address); + Amount minimum_pool_fee_per_byte(bool zero_if_not_full, Hash *minimal_tid = nullptr) const; + bool add_transaction(const Hash &tid, const Transaction &, const BinaryArray &binary_tx, bool check_sigs, + const std::string &source_address); bool get_largest_referenced_height(const TransactionPrefix &tx, Height *block_height) const; - uint32_t get_tx_pool_version() const { return m_tx_pool_version; } + size_t get_tx_pool_version() const { return m_tx_pool_version; } struct PoolTransaction { Transaction tx; BinaryArray binary_tx; + Amount amount; Amount fee; Timestamp timestamp; + Hash newest_referenced_block; - PoolTransaction(const Transaction &tx, const BinaryArray &binary_tx, Amount fee, Timestamp timestamp); + PoolTransaction(const Transaction &tx, const BinaryArray &binary_tx, Amount fee, Timestamp timestamp, + const Hash &newest_referenced_block); Amount fee_per_byte() const { return fee / binary_tx.size(); } }; typedef std::map PoolTransMap; const PoolTransMap &get_memory_state_transactions() const { return m_memory_state_tx; } + std::vector sync_pool( + const std::pair &from, const std::pair &to, size_t max_count) const; - void create_mining_block_template( - const AccountPublicAddress &, const BinaryArray &extra_nonce, BlockTemplate *, Difficulty *, Height *) const; - void create_mining_block_template(const AccountPublicAddress &, const BinaryArray &extra_nonce, BlockTemplate *, - Difficulty *, Height *, Hash) const; - BroadcastAction add_mined_block(const BinaryArray &raw_block_template, RawBlock *, api::BlockHeader *); + void create_mining_block_template(const Hash &, const AccountAddress &, const BinaryArray &extra_nonce, + BlockTemplate *, Difficulty *, Height *, size_t *) const; + bool add_mined_block(const BinaryArray &raw_block_template, RawBlock *, api::BlockHeader *); static api::BlockHeader fill_genesis(Hash genesis_bid, const BlockTemplate &); void test_print_outputs(); - void fill_statistics(api::bytecoind::GetStatistics::Response &res) const override; + void fill_statistics(api::cnd::GetStatistics::Response &res) const override; protected: - virtual std::string check_standalone_consensus(const PreparedBlock &pb, api::BlockHeader *info, - const api::BlockHeader &prev_info, bool check_pow) const override; - virtual bool redo_block(const Hash &bhash, const Block &, const api::BlockHeader &) override; - virtual void undo_block(const Hash &bhash, const Block &, Height) override; + void check_standalone_consensus(const PreparedBlock &pb, api::BlockHeader *info, const api::BlockHeader &prev_info, + bool check_pow) const override; + void redo_block(const Hash &bhash, const Block &, const api::BlockHeader &) override; // throws ConsensusError + void undo_block(const Hash &bhash, const Block &, Height) override; private: class DeltaState : public IBlockChainState { std::map m_keyimages; // sorted to speed up bulk saving to DB - std::map>> m_global_amounts; - std::vector> m_spent_outputs; + std::map>> m_global_amounts; + // std::vector> m_spent_outputs; Height m_block_height; // Every delta state corresponds to some height - Timestamp m_unlock_timestamp; + Timestamp m_block_timestamp; + Timestamp m_block_median_timestamp; const IBlockChainState *m_parent_state; // const parent to prevent accidental parent modification public: - explicit DeltaState(Height block_height, Timestamp unlock_timestamp, const IBlockChainState *parent_state) - : m_block_height(block_height), m_unlock_timestamp(unlock_timestamp), m_parent_state(parent_state) {} + explicit DeltaState(Height block_height, Timestamp block_timestamp, Timestamp block_median_timestamp, + const IBlockChainState *parent_state) + : m_block_height(block_height) + , m_block_timestamp(block_timestamp) + , m_block_median_timestamp(block_median_timestamp) + , m_parent_state(parent_state) {} Height get_block_height() const { return m_block_height; } - Height get_unlock_timestamp() const { return m_unlock_timestamp; } + Height get_block_timestamp() const { return m_block_timestamp; } + Height get_block_median_timestamp() const { return m_block_median_timestamp; } void apply(IBlockChainState *parent_state) const; // Apply modifications to (non-const) parent void clear(Height new_block_height); // We use it for memory_state const std::map &get_keyimages() const { return m_keyimages; } @@ -101,60 +109,64 @@ class BlockChainState : public BlockChain, private IBlockChainState { void delete_keyimage(const KeyImage &) override; bool read_keyimage(const KeyImage &, Height *) const override; - uint32_t push_amount_output(Amount, BlockOrTimestamp, Height, const PublicKey &) override; - void pop_amount_output(Amount, BlockOrTimestamp, const PublicKey &) override; - uint32_t next_global_index_for_amount(Amount) const override; - bool read_amount_output(Amount, uint32_t global_index, UnlockTimePublickKeyHeightSpent *) const override; - void spend_output(Amount, uint32_t global_index) override; + size_t push_amount_output(Amount, BlockOrTimestamp, Height, const PublicKey &, bool is_auditable) override; + void pop_amount_output(Amount, BlockOrTimestamp, const PublicKey &, bool is_auditable) override; + size_t next_global_index_for_amount(Amount) const override; + bool read_amount_output(Amount, size_t global_index, UnlockTimePublickKeyHeightSpent *) const override; }; void store_keyimage(const KeyImage &, Height) override; void delete_keyimage(const KeyImage &) override; bool read_keyimage(const KeyImage &, Height *) const override; - uint32_t push_amount_output(Amount, BlockOrTimestamp, Height, const PublicKey &) override; - void pop_amount_output(Amount, BlockOrTimestamp, const PublicKey &) override; - uint32_t next_global_index_for_amount(Amount) const override; - bool read_amount_output(Amount, uint32_t global_index, UnlockTimePublickKeyHeightSpent *) const override; - void spend_output(Amount, uint32_t global_index) override; - void spend_output(Amount, uint32_t global_index, bool spent); + size_t push_amount_output(Amount, BlockOrTimestamp, Height, const PublicKey &, bool is_auditable) override; + void pop_amount_output(Amount, BlockOrTimestamp, const PublicKey &, bool is_auditable) override; + size_t next_global_index_for_amount(Amount) const override; + bool read_amount_output(Amount, size_t global_index, UnlockTimePublickKeyHeightSpent *) const override; + void spend_output(UnlockTimePublickKeyHeightSpent &&, Amount, size_t global_index, size_t trigger_input_index, + size_t level, bool spent); - std::string redo_transaction_get_error(bool generating, const Transaction &, DeltaState *, BlockGlobalIndices *, - Height *conflict_height, bool check_sigs) const; - bool redo_block(const Block &, const api::BlockHeader &, DeltaState *, BlockGlobalIndices *) const; + void redo_transaction(uint8_t major_block_version, bool generating, const Transaction &, DeltaState *, + BlockGlobalIndices *, Hash *newest_referenced_bid, + bool check_sigs) const; // throws ConsensusError + void redo_block( + const Block &, const api::BlockHeader &, DeltaState *, BlockGlobalIndices *) const; // throws ConsensusError void undo_transaction(IBlockChainState *delta_state, Height, const Transaction &); + const size_t m_max_pool_size; mutable crypto::CryptoNightContext m_hash_crypto_context; - mutable std::unordered_map - m_next_gi_for_amount; // Read from db on first use, write on modification + mutable std::unordered_map m_next_gi_for_amount; + // Read from db on first use, write on modification - AddTransactionResult add_transaction(const Hash &tid, const Transaction &tx, const BinaryArray &binary_tx, - Height unlock_height, Timestamp unlock_timestamp, Height *conflict_height, bool check_sigs, - const std::string &source_address); void remove_from_pool(Hash tid); - uint32_t m_tx_pool_version = 2; // Incremented every time pool changes, reset to 2 on redo block. 2 is selected - // because wallet resets to 1, so after both reset pool versions do not equal + size_t m_tx_pool_version = 2; // Incremented every time pool changes, reset to 2 on redo block. 2 is selected + // because wallet resets to 1, so after both reset pool versions do not equal PoolTransMap m_memory_state_tx; std::map m_memory_state_ki_tx; - std::map> m_memory_state_fee_tx; + std::set> m_memory_state_fee_tx; size_t m_memory_state_total_size = 0; mutable std::map> m_mining_transactions; // We remember them for several blocks void clear_mining_transactions() const; - - Timestamp m_next_median_timestamp = 0; - uint32_t m_next_median_size = 0; - virtual void tip_changed() override; // Updates values above - virtual void on_reorganization( + size_t m_next_nz_input_index = 0; + void process_input(const Hash &tid, size_t iid, const InputKey &input); + void unprocess_input(const InputKey &input); + + Timestamp m_next_median_timestamp = 0; + size_t m_next_median_size = 0; + size_t m_next_median_block_capacity_vote = 0; + void tip_changed() override; // Updates values above + void on_reorganization( const std::map> &undone_transactions, bool undone_blocks) override; - void calculate_consensus_values( - const api::BlockHeader &prev_info, uint32_t *next_median_size, Timestamp *next_median_timestamp) const; + Timestamp calculate_next_median_timestamp(const api::BlockHeader &prev_info) const; + size_t calculate_next_median_size(const api::BlockHeader &prev_info) const; + size_t calculate_next_median_block_capacity_vote(const api::BlockHeader &prev_info) const; - RingCheckerMulticore ring_checker; - std::chrono::steady_clock::time_point log_redo_block_timestamp; + RingCheckerMulticore m_ring_checker; + std::chrono::steady_clock::time_point m_log_redo_block_timestamp; }; -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/Config.cpp b/src/Core/Config.cpp index e7cce891..d5e25c54 100644 --- a/src/Core/Config.cpp +++ b/src/Core/Config.cpp @@ -7,11 +7,15 @@ #include #include "CryptoNoteConfig.hpp" #include "common/Base64.hpp" +#include "common/CommandLine.hpp" +#include "p2p/P2pProtocolDefinitions.hpp" #include "platform/PathTools.hpp" #include "platform/Time.hpp" +#include "rpc_api.hpp" using namespace common; -using namespace bytecoin; +using namespace cn; +using namespace parameters; static void parse_peer_and_add_to_container(const std::string &str, std::vector &container) { NetworkAddress na{}; @@ -33,14 +37,11 @@ static std::string get_net(common::CommandLine &cmd) { return "main"; } -const static UUID BYTECOIN_NETWORK{{0x11, 0x10, 0x01, 0x11, 0x11, 0x00, 0x01, 0x01, 0x10, 0x11, 0x00, 0x12, 0x10, 0x11, - 0x01, 0x10}}; // Bender's nightmare - Config::Config(common::CommandLine &cmd) : net(get_net(cmd)) , is_archive(cmd.get_bool("--archive")) - , blocks_file_name(parameters::BLOCKS_FILENAME) - , block_indexes_file_name(parameters::BLOCKINDEXES_FILENAME) + , blocks_file_name(BLOCKS_FILENAME) + , block_indexes_file_name(BLOCKINDEXES_FILENAME) , crypto_note_name(CRYPTONOTE_NAME) , network_id(BYTECOIN_NETWORK) , p2p_bind_port(P2P_DEFAULT_PORT) @@ -48,26 +49,17 @@ Config::Config(common::CommandLine &cmd) , p2p_bind_ip("0.0.0.0") , multicast_address("239.195.17.131") , multicast_port(P2P_DEFAULT_PORT) - , multicast_period(net == "main" ? 0 : 60.0f) // No multicast in mainnet due to anonymity + , multicast_period(net == "main" ? 0 : 60.0f) // No multicast in main net due to anonymity + , secrets_via_api(cmd.get_bool("--secrets-via-api")) , bytecoind_bind_port(RPC_DEFAULT_PORT) , bytecoind_bind_ip("127.0.0.1") // Less attack vectors from outside for ordinary uses - , bytecoind_remote_port(0) , bytecoind_remote_ip("127.0.0.1") - , db_commit_period_wallet_cache(290) - , db_commit_period_blockchain(310) , walletd_bind_port(WALLET_RPC_DEFAULT_PORT) , walletd_bind_ip("127.0.0.1") // Connection to wallet allows spending - , p2p_local_white_list_limit(1000) - , p2p_local_gray_list_limit(5000) - , p2p_default_peers_in_handshake(P2P_DEFAULT_PEERS_IN_HANDSHAKE) - , p2p_max_outgoing_connections(8) - , p2p_max_incoming_connections(100) - , p2p_whitelist_connections_percent(70) - , p2p_block_ids_sync_default_count(BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT) - , p2p_blocks_sync_default_count(BLOCKS_SYNCHRONIZING_DEFAULT_COUNT) - , rpc_get_blocks_fast_max_count(COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT) + , rpc_sync_blocks_max_count(api::cnd::SyncBlocks::Request::MAX_COUNT) , paranoid_checks(cmd.get_bool("--paranoid-checks")) - , trusted_public_key(P2P_STAT_TRUSTED_PUBLIC_KEY) { + , trusted_public_key(P2P_STAT_TRUSTED_PUBLIC_KEY) + , payment_queue_confirmations(720) { if (net == "test") { network_id.data[0] += 1; p2p_bind_port += 1000; @@ -77,6 +69,7 @@ Config::Config(common::CommandLine &cmd) multicast_port += 1000; if (const char *pa = cmd.get("--time-multiplier")) platform::set_time_multiplier_for_tests(boost::lexical_cast(pa)); + payment_queue_confirmations = 30; } if (net == "stage") { network_id.data[0] += 2; @@ -111,24 +104,24 @@ Config::Config(common::CommandLine &cmd) "Setting --ssl_certificate_password impossible - this binary is built without OpenSSL"); #endif } - if (const char *pa = cmd.get("--bytecoind-authorization")) { + if (const char *pa = cmd.get("--" CRYPTONOTE_NAME "d-authorization")) { bytecoind_authorization = common::base64::encode(BinaryArray(pa, pa + strlen(pa))); bytecoind_authorization_private = bytecoind_authorization; } - if (const char *pa = cmd.get("--bytecoind-authorization-private")) { + if (const char *pa = cmd.get("--" CRYPTONOTE_NAME "d-authorization-private")) { bytecoind_authorization_private = common::base64::encode(BinaryArray(pa, pa + strlen(pa))); } - if (const char *pa = cmd.get("--bytecoind-bind-address")) { + if (const char *pa = cmd.get("--" CRYPTONOTE_NAME "d-bind-address")) { if (!common::parse_ip_address_and_port(pa, &bytecoind_bind_ip, &bytecoind_bind_port)) throw std::runtime_error("Wrong address format " + std::string(pa) + ", should be ip:port"); } - if (const char *pa = cmd.get("--bytecoind-remote-address")) { + if (const char *pa = cmd.get("--" CRYPTONOTE_NAME "d-remote-address")) { std::string addr = pa; const std::string prefix = "https://"; if (addr.find(prefix) == 0) { #if !platform_USE_SSL - throw std::runtime_error( - "Using https in --bytecoind-remote-address impossible - this binary is built without OpenSSL"); + throw std::runtime_error("Using https in --" CRYPTONOTE_NAME + "d-remote-address impossible - this binary is built without OpenSSL"); #endif std::string sip; std::string sport; @@ -175,16 +168,11 @@ Config::Config(common::CommandLine &cmd) std::sort(seed_nodes.begin(), seed_nodes.end()); std::sort(priority_nodes.begin(), priority_nodes.end()); - if (const char *pa = cmd.get("--mineproof-secret")) { - if (!common::pod_from_hex(pa, mineproof_secret)) - throw std::runtime_error("--mineproof-secret must be 64 hex characters"); - } - - data_folder = platform::get_app_data_folder(crypto_note_name); + data_folder = platform::get_app_data_folder(CRYPTONOTE_NAME); if (net != "main") data_folder += "_" + net + "net"; if (const char *pa = cmd.get("--data-folder")) { - data_folder = pa; + data_folder = platform::normalize_folder(pa); if (!platform::folder_exists(data_folder)) throw std::runtime_error("Data folder must exist " + data_folder); } else { @@ -193,6 +181,17 @@ Config::Config(common::CommandLine &cmd) } } +std::string Config::prepare_usage(const std::string &usage) { + std::string result = usage; + boost::replace_all(result, "bytecoin", CRYPTONOTE_NAME); + boost::replace_all(result, "blocks.bin", BLOCKS_FILENAME); + boost::replace_all(result, "blockindexes.bin", BLOCKINDEXES_FILENAME); + boost::replace_all(result, "8080", common::to_string(P2P_DEFAULT_PORT)); + boost::replace_all(result, "8081", common::to_string(RPC_DEFAULT_PORT)); + boost::replace_all(result, "8070", common::to_string(WALLET_RPC_DEFAULT_PORT)); + return result; +} + std::string Config::get_data_folder(const std::string &subdir) const { std::string folder = data_folder; // This code is called just several times at startup, so no caching diff --git a/src/Core/Config.hpp b/src/Core/Config.hpp index 316d3a4a..01bfd3f2 100644 --- a/src/Core/Config.hpp +++ b/src/Core/Config.hpp @@ -8,14 +8,18 @@ #include #include #include "CryptoNote.hpp" -#include "common/CommandLine.hpp" #include "p2p/P2pProtocolTypes.hpp" -namespace bytecoin { +namespace common { +class CommandLine; +} + +namespace cn { class Config { // Consensus does not depend on those parameters public: explicit Config(common::CommandLine &cmd); + static std::string prepare_usage(const std::string &usage); // replaces defaults std::string net; bool is_archive; @@ -25,6 +29,7 @@ class Config { // Consensus does not depend on those parameters std::string crypto_note_name; UUID network_id; + bool allow_empty_network_id = false; uint16_t p2p_bind_port; uint16_t p2p_external_port; @@ -32,6 +37,7 @@ class Config { // Consensus does not depend on those parameters std::string multicast_address; uint16_t multicast_port; float multicast_period; + bool secrets_via_api; std::string ssl_certificate_pem_file; boost::optional ssl_certificate_password; @@ -39,26 +45,64 @@ class Config { // Consensus does not depend on those parameters std::string bytecoind_authorization_private; uint16_t bytecoind_bind_port; std::string bytecoind_bind_ip; - uint16_t bytecoind_remote_port; + uint16_t bytecoind_remote_port = 0; std::string bytecoind_remote_ip; - Hash mineproof_secret; - float db_commit_period_wallet_cache; - float db_commit_period_blockchain; + + size_t max_pool_size = 4 * 1000 * 1000; + size_t max_undo_transactions_size = 200 * 1000 * 1000; + // During very large reorganization, only last transaction within limit will be redone + + // a bit different commit periods to make most commits not simultaneous + Timestamp db_commit_period_wallet_cache = 291; + Timestamp db_commit_period_blockchain = 311; + Timestamp db_commit_period_peers = 60; + size_t db_commit_every_n_blocks = 50000; + // This affects DB transaction size. TODO - sum size of blocks instead std::string walletd_authorization; uint16_t walletd_bind_port; std::string walletd_bind_ip; - size_t p2p_local_white_list_limit; - size_t p2p_local_gray_list_limit; - size_t p2p_default_peers_in_handshake; - size_t p2p_max_outgoing_connections; - size_t p2p_max_incoming_connections; - size_t p2p_whitelist_connections_percent; - - size_t p2p_block_ids_sync_default_count; - size_t p2p_blocks_sync_default_count; - size_t rpc_get_blocks_fast_max_count; + size_t p2p_local_white_list_limit = 1000; + size_t p2p_local_gray_list_limit = 5000; + size_t p2p_default_peers_in_handshake = 250; + size_t p2p_max_outgoing_connections = 8; + size_t p2p_max_incoming_connections = 100; + size_t p2p_whitelist_connections_percent = 70; + + Timestamp p2p_ban_period = 60 * 15; + Timestamp p2p_reconnect_period = 60 * 5; + Timestamp p2p_reconnect_period_seed = 86400; + Timestamp p2p_reconnect_period_priority = 30; + float p2p_no_internet_reconnect_delay = 0.5f; + // When we fail to connect to any peer after lots of attempts + + float p2p_network_unreachable_delay = 10.0f; + // When connect() fails immediately several times in a row + + Timestamp p2p_no_incoming_handshake_disconnect_timeout = 30; + Timestamp p2p_no_incoming_message_disconnect_timeout = 60 * 6; + Timestamp p2p_no_outgoing_message_ping_timeout = 60 * 4; + + size_t rpc_sync_blocks_max_count; + + Height p2p_outgoing_peer_max_lag = 5; + // if peer we are connected to is/starts lagging by 5 blocks or more, we will + // disconnect and delay connect it, in hope to find better peers + + size_t max_downloading_blocks_from_each_peer = 100; + size_t download_window = 2000; + float download_block_timeout = 30.0f; + float download_transaction_timeout = 30.0f; + float download_chain_timeout = 30.0f; + float sync_pool_timeout = 30.0f; + float max_on_idle_time = 0.1f; // seconds + size_t download_broadcast_every_n_blocks = 10000; + // During download, we send time sync commands periodically to inform other that + // they can now download more blocks from us + + Timestamp wallet_sync_timestamp_granularity = 86400 * 30; + // Sending exact timestamp of wallet to public node allows tracking std::vector seed_nodes; std::vector priority_nodes; @@ -70,6 +114,8 @@ class Config { // Consensus does not depend on those parameters std::string get_data_folder() const { return data_folder; } // suppress creation of dir itself std::string get_data_folder(const std::string &subdir) const; + + Height payment_queue_confirmations; }; -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/CryptoNoteTools.cpp b/src/Core/CryptoNoteTools.cpp index 4a8b77a0..4f4e6d34 100644 --- a/src/Core/CryptoNoteTools.cpp +++ b/src/Core/CryptoNoteTools.cpp @@ -7,11 +7,13 @@ #include "common/Varint.hpp" #include "seria/ISeria.hpp" -using namespace bytecoin; +using namespace cn; -Hash bytecoin::get_base_transaction_hash(const BaseTransaction &tx) { +Hash cn::get_root_block_base_transaction_hash(const BaseTransaction &tx) { if (tx.version < 2) return get_object_hash(tx); + // XMR(XMO) as popular MM root, see details in monero/src/cryptonote_basic/cryptonote_format_utils.cpp + // bc36789e7a1e281436464229828f817d6612f7b477d66591ff96a9e064bcc98a = hash(1 zero byte (RCTTypeNull)) BinaryArray data{{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x36, 0x78, 0x9e, 0x7a, 0x1e, 0x28, 0x14, 0x36, 0x46, 0x42, 0x29, 0x82, 0x8f, 0x81, 0x7d, 0x66, 0x12, 0xf7, 0xb4, @@ -22,13 +24,13 @@ Hash bytecoin::get_base_transaction_hash(const BaseTransaction &tx) { return crypto::cn_fast_hash(data.data(), data.size()); } -void bytecoin::set_solo_mining_tag(BlockTemplate &block) { +void cn::set_root_extra_to_solo_mining_tag(BlockTemplate &block) { if (block.is_merge_mined()) { TransactionExtraMergeMiningTag mmTag; mmTag.depth = 0; - block.parent_block.base_transaction.extra.clear(); + block.root_block.base_transaction.extra.clear(); mmTag.merkle_root = get_auxiliary_block_header_hash(block, get_body_proxy_from_template(block)); - extra_add_merge_mining_tag(block.parent_block.base_transaction.extra, mmTag); + extra_add_merge_mining_tag(block.root_block.base_transaction.extra, mmTag); } } @@ -68,7 +70,7 @@ void decompose_amount_into_digits( } } -void bytecoin::decompose_amount(Amount amount, Amount dust_threshold, std::vector *decomposed_amounts) { +void cn::decompose_amount(Amount amount, Amount dust_threshold, std::vector *decomposed_amounts) { decompose_amount_into_digits(amount, dust_threshold, [&](Amount amount) { decomposed_amounts->push_back(amount); }, [&](Amount dust) { // This code will work relatively well for any dust_threshold <= 6 @@ -78,16 +80,17 @@ void bytecoin::decompose_amount(Amount amount, Amount dust_threshold, std::vecto decomposed_amounts->push_back(du1); if (du0 != 0) decomposed_amounts->push_back(du0); - }); + }); } const size_t KEY_IMAGE_SIZE = sizeof(KeyImage); const size_t OUTPUT_KEY_SIZE = sizeof(PublicKey); +const size_t OUTPUT_SECRET_SIZE = sizeof(Hash); const size_t AMOUNT_SIZE = sizeof(uint64_t) + 2; // varint const size_t IO_COUNT_SIZE = 3; // varint const size_t GLOBAL_INDEXES_VECTOR_SIZE_SIZE = 1; // varint -const size_t GLOBAL_INDEXES_INITIAL_VALUE_SIZE = sizeof(uint32_t); // varint -const size_t GLOBAL_INDEXES_DIFFERENCE_SIZE = sizeof(uint32_t); // varint +const size_t GLOBAL_INDEXES_INITIAL_VALUE_SIZE = sizeof(size_t); // varint +const size_t GLOBAL_INDEXES_DIFFERENCE_SIZE = sizeof(size_t); // varint const size_t SIGNATURE_SIZE = sizeof(Signature); const size_t EXTRA_TAG_SIZE = 1; const size_t INPUT_TAG_SIZE = 1; @@ -98,48 +101,84 @@ const size_t TRANSACTION_UNLOCK_TIME_SIZE = sizeof(uint64_t); const size_t header_size = TRANSACTION_VERSION_SIZE + TRANSACTION_UNLOCK_TIME_SIZE + EXTRA_TAG_SIZE + PUBLIC_KEY_SIZE; -size_t bytecoin::get_maximum_tx_input_size(size_t anonymity) { +size_t cn::get_maximum_tx_input_size(size_t anonymity) { return INPUT_TAG_SIZE + AMOUNT_SIZE + KEY_IMAGE_SIZE + SIGNATURE_SIZE + GLOBAL_INDEXES_VECTOR_SIZE_SIZE + GLOBAL_INDEXES_INITIAL_VALUE_SIZE + anonymity * (GLOBAL_INDEXES_DIFFERENCE_SIZE + SIGNATURE_SIZE); } -size_t bytecoin::get_maximum_tx_size(size_t input_count, size_t output_count, size_t anonymity) { - const size_t outputs_size = IO_COUNT_SIZE + output_count * (OUTPUT_TAG_SIZE + OUTPUT_KEY_SIZE + AMOUNT_SIZE); - const size_t inputs_size = IO_COUNT_SIZE + input_count * get_maximum_tx_input_size(anonymity); +size_t cn::get_maximum_tx_size(size_t input_count, size_t output_count, size_t anonymity) { + const size_t outputs_size = + IO_COUNT_SIZE + output_count * (OUTPUT_TAG_SIZE + OUTPUT_KEY_SIZE + OUTPUT_SECRET_SIZE + AMOUNT_SIZE); + const size_t inputs_size = IO_COUNT_SIZE + input_count * get_maximum_tx_input_size(anonymity); return header_size + outputs_size + inputs_size; } -bool bytecoin::get_tx_fee(const TransactionPrefix &tx, uint64_t *fee) { - uint64_t amount_in = 0; +Amount cn::get_tx_sum_outputs(const TransactionPrefix &tx) { uint64_t amount_out = 0; - + for (const auto &o : tx.outputs) { + if (o.type() == typeid(OutputKey)) + amount_out += boost::get(o).amount; + } + return amount_out; +} +Amount cn::get_tx_sum_inputs(const TransactionPrefix &tx) { + uint64_t amount_in = 0; for (const auto &in : tx.inputs) { - if (in.type() == typeid(KeyInput)) { - amount_in += boost::get(in).amount; - } + if (in.type() == typeid(InputKey)) + amount_in += boost::get(in).amount; } - for (const auto &o : tx.outputs) - amount_out += o.amount; + return amount_in; +} + +bool cn::get_tx_fee(const TransactionPrefix &tx, uint64_t *fee) { + uint64_t amount_in = get_tx_sum_inputs(tx); + uint64_t amount_out = get_tx_sum_outputs(tx); + if (amount_in < amount_out) return false; *fee = amount_in - amount_out; return true; } -uint64_t bytecoin::get_tx_fee(const TransactionPrefix &tx) { +uint64_t cn::get_tx_fee(const TransactionPrefix &tx) { uint64_t r = 0; if (!get_tx_fee(tx, &r)) return 0; return r; } -BlockBodyProxy bytecoin::get_body_proxy_from_template(const BlockTemplate &bt) { +std::vector cn::absolute_output_offsets_to_relative(const std::vector &off) { + invariant(!off.empty(), "Output indexes cannot be empty"); + std::vector relative(off.size()); + relative[0] = off[0]; + for (size_t i = 1; i < off.size(); ++i) { + invariant(off[i] > off[i - 1], "Output indexes must be unique and sorted"); + relative[i] = off[i] - off[i - 1]; + } + return relative; +} + +bool cn::relative_output_offsets_to_absolute(std::vector *result, const std::vector &off) { + if (off.empty()) + return false; + std::vector absolute(off.size()); + absolute[0] = off[0]; + for (size_t i = 1; i < off.size(); ++i) { + if (off[i] == 0 || std::numeric_limits::max() - absolute[i - 1] < off[i]) + return false; + absolute[i] = absolute[i - 1] + off[i]; + } + *result = std::move(absolute); + return true; +} + +BlockBodyProxy cn::get_body_proxy_from_template(const BlockTemplate &bt) { BlockBodyProxy body_proxy; std::vector transaction_hashes; transaction_hashes.reserve(bt.transaction_hashes.size() + 1); transaction_hashes.push_back(get_object_hash(bt.base_transaction)); transaction_hashes.insert(transaction_hashes.end(), bt.transaction_hashes.begin(), bt.transaction_hashes.end()); body_proxy.transactions_merkle_root = crypto::tree_hash(transaction_hashes.data(), transaction_hashes.size()); - body_proxy.transaction_count = static_cast(transaction_hashes.size()); + body_proxy.transaction_count = transaction_hashes.size(); return body_proxy; } diff --git a/src/Core/CryptoNoteTools.hpp b/src/Core/CryptoNoteTools.hpp index 16b3275c..951200c9 100644 --- a/src/Core/CryptoNoteTools.hpp +++ b/src/Core/CryptoNoteTools.hpp @@ -8,7 +8,7 @@ #include "crypto/hash.hpp" #include "seria/BinaryOutputStream.hpp" -namespace bytecoin { +namespace cn { template Hash get_object_hash(const T &object, size_t *size = nullptr) { @@ -18,16 +18,29 @@ Hash get_object_hash(const T &object, size_t *size = nullptr) { return crypto::cn_fast_hash(ba.data(), ba.size()); } -Hash get_base_transaction_hash(const BaseTransaction &tx); +Hash get_root_block_base_transaction_hash(const BaseTransaction &tx); -void set_solo_mining_tag(BlockTemplate &block); // MM headers must still have valid mm_tag if solo mining +void set_root_extra_to_solo_mining_tag(BlockTemplate &block); // MM headers must still have valid mm_tag if solo mining void decompose_amount(Amount amount, Amount dust_threshold, std::vector *decomposed_amounts); size_t get_maximum_tx_size(size_t input_count, size_t output_count, size_t anonymity); size_t get_maximum_tx_input_size(size_t anonymity); -bool get_tx_fee(const TransactionPrefix &tx, uint64_t *fee); -uint64_t get_tx_fee(const TransactionPrefix &tx); +Amount get_tx_sum_outputs(const TransactionPrefix &tx); +Amount get_tx_sum_inputs(const TransactionPrefix &tx); -BlockBodyProxy get_body_proxy_from_template(const BlockTemplate &bt); +inline bool add_amount(Amount &sum, Amount amount) { + if (std::numeric_limits::max() - amount < sum) + return false; + sum += amount; + return true; } + +bool get_tx_fee(const TransactionPrefix &tx, Amount *fee); +Amount get_tx_fee(const TransactionPrefix &tx); + +std::vector absolute_output_offsets_to_relative(const std::vector &off); +bool relative_output_offsets_to_absolute(std::vector *result, const std::vector &off); + +BlockBodyProxy get_body_proxy_from_template(const BlockTemplate &bt); +} // namespace cn diff --git a/src/Core/Currency.cpp b/src/Core/Currency.cpp index eef3a4c6..fc0b9c47 100644 --- a/src/Core/Currency.cpp +++ b/src/Core/Currency.cpp @@ -4,7 +4,6 @@ #include "Currency.hpp" #include #include -#include #include #include "CryptoNote.hpp" #include "CryptoNoteConfig.hpp" @@ -24,7 +23,8 @@ #include "seria/BinaryOutputStream.hpp" using namespace common; -using namespace bytecoin; +using namespace cn; +using namespace parameters; const std::vector Currency::PRETTY_AMOUNTS = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000, 20000, @@ -53,258 +53,283 @@ const std::vector Currency::DECIMAL_PLACES = {1, 10, 100, 1000, 10000, 1 Currency::Currency(const std::string &net) : net(net) - , max_block_height(parameters::MAX_BLOCK_NUMBER) - , max_block_blob_size(parameters::MAX_BLOCK_BLOB_SIZE) - , max_tx_size(parameters::MAX_TX_SIZE) - , public_address_base58_prefix(parameters::PUBLIC_ADDRESS_BASE58_PREFIX) - , mined_money_unlock_window(parameters::MINED_MONEY_UNLOCK_WINDOW) - , timestamp_check_window(parameters::BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) - , block_future_time_limit(parameters::BLOCK_FUTURE_TIME_LIMIT) - , money_supply(parameters::MONEY_SUPPLY) - , emission_speed_factor(parameters::EMISSION_SPEED_FACTOR) - , reward_blocks_window(parameters::REWARD_BLOCKS_WINDOW) - , minimum_size_median(parameters::MINIMUM_SIZE_MEDIAN) - , miner_tx_blob_reserved_size(parameters::COINBASE_BLOB_RESERVED_SIZE) - , number_of_decimal_places(parameters::DISPLAY_DECIMAL_POINT) - , default_dust_threshold(parameters::DEFAULT_DUST_THRESHOLD) - , self_dust_threshold(parameters::SELF_DUST_THRESHOLD) + , max_block_height(MAX_BLOCK_NUMBER) + , mined_money_unlock_window(MINED_MONEY_UNLOCK_WINDOW) + , block_future_time_limit(BLOCK_FUTURE_TIME_LIMIT) + , money_supply(MONEY_SUPPLY) + , emission_speed_factor(EMISSION_SPEED_FACTOR) + , median_block_size_window(MEIDAN_BLOCK_SIZE_WINDOW) + , block_capacity_vote_window(BLOCK_CAPACITY_VOTE_WINDOW) + , max_header_size(MAX_HEADER_SIZE) + , block_capacity_vote_min(BLOCK_CAPACITY_VOTE_MIN) + , block_capacity_vote_max(BLOCK_CAPACITY_VOTE_MAX) + , number_of_decimal_places(DISPLAY_DECIMAL_POINT) + , min_dust_threshold(MIN_DUST_THRESHOLD) + , max_dust_threshold(MAX_DUST_THRESHOLD) + , self_dust_threshold(SELF_DUST_THRESHOLD) , difficulty_target(std::max(1, - parameters::DIFFICULTY_TARGET / - platform::get_time_multiplier_for_tests())) // multiplier can be != 1 only in testnet - , minimum_difficulty(net == "test" ? 2 : parameters::MINIMUM_DIFFICULTY) - , difficulty_window(expected_blocks_per_day()) - , difficulty_lag(parameters::DIFFICULTY_LAG) - , difficulty_cut(parameters::DIFFICULTY_CUT) - , max_block_size_initial(net != "main" ? 1024 * 1024 : parameters::MAX_BLOCK_SIZE_INITIAL) - , max_block_size_growth_per_year(net != "main" ? 0 : parameters::MAX_BLOCK_SIZE_GROWTH_PER_YEAR) - , locked_tx_allowed_delta_seconds(parameters::LOCKED_TX_ALLOWED_DELTA_SECONDS(difficulty_target)) - , locked_tx_allowed_delta_blocks(parameters::LOCKED_TX_ALLOWED_DELTA_BLOCKS) - , upgrade_height_v2(parameters::UPGRADE_HEIGHT_V2) - , upgrade_height_v3(parameters::UPGRADE_HEIGHT_V3) - , key_image_subgroup_checking_height(parameters::KEY_IMAGE_SUBGROUP_CHECKING_HEIGHT) + DIFFICULTY_TARGET / platform::get_time_multiplier_for_tests())) // multiplier can be != 1 only in testnet + , upgrade_heights{UPGRADE_HEIGHT_V2, UPGRADE_HEIGHT_V3} + , key_image_subgroup_checking_height(KEY_IMAGE_SUBGROUP_CHECKING_HEIGHT) + , amethyst_block_version(BLOCK_VERSION_AMETHYST) + , amethyst_transaction_version(TRANSACTION_VERSION_AMETHYST) , upgrade_from_major_version(3) - , upgrade_indicator_minor_version(4) + , upgrade_indicator_minor_version(5) , upgrade_desired_major_version(0) - , upgrade_voting_window(expected_blocks_per_day()) - , upgrade_votes_required(upgrade_voting_window * 9 / 10) - , upgrade_blocks_after_voting(expected_blocks_per_day() * 14) - , current_transaction_version(CURRENT_TRANSACTION_VERSION) { - if (net != "main") { - upgrade_height_v2 = 1; - upgrade_height_v3 = 1; + , upgrade_voting_window(UPGRADE_VOTING_WINDOW) + , upgrade_window(UPGRADE_WINDOW) { + if (net == "test") { + upgrade_heights = {1, 1}; // block 1 is already V3 + upgrade_desired_major_version = 4; + upgrade_voting_window = 30; + upgrade_window = 10; + } + if (net == "stage") { + upgrade_heights = {1, 1}; // block 1 is already V3 + upgrade_desired_major_version = 4; + upgrade_window = EXPECTED_NUMBER_OF_BLOCKS_PER_DAY; + } + { + BinaryArray miner_tx_blob; + invariant(from_hex(GENESIS_COINBASE_TX_HEX, &miner_tx_blob), + "Currency failed to parse coinbase tx from hard coded blob"); + seria::from_binary(genesis_block_template.base_transaction, miner_tx_blob); + // Demystified genesis block calculations below + // PublicKey genesis_output_key = + // common::pfh("9b2e4c0281c0b02e7c53291a94d1d0cbff8883f8024f5142ee494ffbbd088071"); + // PublicKey genesis_tx_public_key = + // common::pfh("3c086a48c15fb637a96991bc6d53caf77068b5ba6eeb3c82357228c49790584a"); + // Transaction base_transaction; + // base_transaction.version = 1; + // base_transaction.unlock_block_or_timestamp = mined_money_unlock_window; + // base_transaction.inputs.push_back(CoinbaseInput{0}); + // base_transaction.outputs.push_back(TransactionOutput{money_supply >> emission_speed_factor, + // KeyOutput{genesis_output_key}}); + // extra_add_transaction_public_key(base_transaction.extra, genesis_tx_public_key); + // invariant(miner_tx_blob == seria::to_binary(base_transaction), "Demystified transaction does not match + // original one"); } - // Hard code coinbase tx in genesis block, because through generating tx use - // random, but genesis should be always - // the same - // std::string genesis_coinbase_tx_hex = - // "010a01ff0001ffffffffffff0f029b2e4c0281c0b02e7c53291a94d1d0cbff8883f8024f" - // "5142ee494ffbbd08807121013c086a48c15fb637a96991bc6d53caf77068b5ba6eeb3c82357228c49790584a"; - // BinaryArray miner_tx_blob; - // invariant(from_hex(genesis_coinbase_tx_hex, miner_tx_blob), "Currency failed to parse coinbase tx from hard - // coded blob"); - // seria::from_binary(genesis_block_template.base_transaction, miner_tx_blob); - // Demystified genesis block calculations below genesis_block_template.major_version = 1; genesis_block_template.minor_version = 0; genesis_block_template.timestamp = 0; - genesis_block_template.nonce = 70; - PublicKey genesis_output_key = - common::pfh("9b2e4c0281c0b02e7c53291a94d1d0cbff8883f8024f5142ee494ffbbd088071"); - PublicKey genesis_tx_public_key = - common::pfh("3c086a48c15fb637a96991bc6d53caf77068b5ba6eeb3c82357228c49790584a"); - genesis_block_template.base_transaction.version = 1; - genesis_block_template.base_transaction.unlock_block_or_timestamp = mined_money_unlock_window; - genesis_block_template.base_transaction.inputs.push_back(CoinbaseInput{0}); - genesis_block_template.base_transaction.outputs.push_back( - TransactionOutput{money_supply >> emission_speed_factor, KeyOutput{genesis_output_key}}); - extra_add_transaction_public_key(genesis_block_template.base_transaction.extra, genesis_tx_public_key); + genesis_block_template.nonce = BinaryArray{70, 0, 0, 0}; if (net == "test") { - genesis_block_template.nonce += 1; + genesis_block_template.nonce.at(0) += 1; genesis_block_template.timestamp = platform::get_time_multiplier_for_tests() - 1; } if (net == "stage") - genesis_block_template.nonce += 2; + genesis_block_template.nonce.at(0) += 2; auto body_proxy = get_body_proxy_from_template(genesis_block_template); genesis_block_hash = get_block_hash(genesis_block_template, body_proxy); if (net == "main") { - checkpoints_begin = std::begin(CHECKPOINTS); - checkpoints_end = std::end(CHECKPOINTS); - checkpoint_keys_begin = std::begin(CHECKPOINT_PUBLIC_KEYS); - checkpoint_keys_end = std::end(CHECKPOINT_PUBLIC_KEYS); + checkpoints_begin = CHECKPOINTS; + checkpoints_end = CHECKPOINTS + sizeof(CHECKPOINTS) / sizeof(*CHECKPOINTS); + checkpoint_keys_begin = CHECKPOINT_PUBLIC_KEYS; + checkpoint_keys_end = CHECKPOINT_PUBLIC_KEYS + sizeof(CHECKPOINT_PUBLIC_KEYS) / sizeof(*CHECKPOINT_PUBLIC_KEYS); } if (net == "test") { - checkpoints_begin = nullptr; - checkpoints_end = nullptr; - checkpoint_keys_begin = std::begin(CHECKPOINT_PUBLIC_KEYS_TESTNET); - checkpoint_keys_end = std::end(CHECKPOINT_PUBLIC_KEYS_TESTNET); + checkpoint_keys_begin = CHECKPOINT_PUBLIC_KEYS_TESTNET; + checkpoint_keys_end = CHECKPOINT_PUBLIC_KEYS_TESTNET + + sizeof(CHECKPOINT_PUBLIC_KEYS_TESTNET) / sizeof(*CHECKPOINT_PUBLIC_KEYS_TESTNET); } if (net == "stage") { - checkpoints_begin = std::begin(CHECKPOINTS_STAGENET); - checkpoints_end = std::end(CHECKPOINTS_STAGENET); - checkpoint_keys_begin = std::begin(CHECKPOINT_PUBLIC_KEYS_STAGENET); - checkpoint_keys_end = std::end(CHECKPOINT_PUBLIC_KEYS_STAGENET); + checkpoints_begin = CHECKPOINTS_STAGENET; + checkpoints_end = CHECKPOINTS_STAGENET + sizeof(CHECKPOINTS_STAGENET) / sizeof(*CHECKPOINTS_STAGENET); + ; + checkpoint_keys_begin = CHECKPOINT_PUBLIC_KEYS_STAGENET; + checkpoint_keys_end = CHECKPOINT_PUBLIC_KEYS_STAGENET + + sizeof(CHECKPOINT_PUBLIC_KEYS_STAGENET) / sizeof(*CHECKPOINT_PUBLIC_KEYS_STAGENET); } + miner_tx_blob_reserved_size = + get_maximum_tx_size(1, get_max_coinbase_outputs(), 0) + 1 + TransactionExtraNonce::MAX_COUNT; // ~1k bytes } -uint32_t Currency::expected_blocks_per_day() const { +Height Currency::upgrade_votes_required() const { return upgrade_voting_window * UPGRADE_VOTING_PERCENT / 100; } + +Height Currency::timestamp_check_window(uint8_t block_major_version) const { + if (block_major_version >= amethyst_block_version) + return BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW; + return BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW_V1_3; +} + +bool Currency::is_transaction_unlocked(uint8_t block_major_version, BlockOrTimestamp unlock_time, Height block_height, + Timestamp block_time, Timestamp block_median_time) const { + if (block_major_version >= amethyst_block_version) { + if (unlock_time < max_block_height) // interpret as block index + return block_height >= unlock_time; + return block_median_time + (BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW * difficulty_target) / 2 >= unlock_time; + } + if (unlock_time < max_block_height) // interpret as block index + return block_height + LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time; + return block_time + LOCKED_TX_ALLOWED_DELTA_SECONDS(difficulty_target) >= unlock_time; // interpret as time +} + +bool Currency::is_upgrade_vote(uint8_t major, uint8_t minor) const { + if (major == upgrade_from_major_version) + return minor == upgrade_indicator_minor_version; +#if bytecoin_ALLOW_CM + // TODO - line below is "statement can be simplified", how? + if (upgrade_from_major_version >= amethyst_block_version && major == upgrade_from_major_version + 1) + return minor == upgrade_indicator_minor_version; +#endif + return false; +} + +Height Currency::expected_blocks_per_day() const { return 24 * 60 * 60 / difficulty_target / platform::get_time_multiplier_for_tests(); } -uint32_t Currency::expected_blocks_per_year() const { + +Height Currency::expected_blocks_per_year() const { return 365 * 24 * 60 * 60 / difficulty_target / platform::get_time_multiplier_for_tests(); } -bool Currency::is_in_sw_checkpoint_zone(Height height) const { return height <= last_sw_checkpoint().height; } +bool Currency::is_in_hard_checkpoint_zone(Height height) const { return height <= last_hard_checkpoint().height; } -bool Currency::check_sw_checkpoint(Height height, const Hash &h, bool &is_sw_checkpoint) const { +bool Currency::check_hard_checkpoint(Height height, const Hash &h, bool &is_hard_checkpoint) const { if (checkpoints_begin == checkpoints_end) { - is_sw_checkpoint = (height == 0); + is_hard_checkpoint = (height == 0); return height == 0 ? h == genesis_block_hash : true; } auto it = std::lower_bound( - checkpoints_begin, checkpoints_end, height, [](const SWCheckpoint &da, uint32_t ma) { return da.height < ma; }); - is_sw_checkpoint = false; + checkpoints_begin, checkpoints_end, height, [](const HardCheckpoint &da, Height ma) { return da.height < ma; }); + is_hard_checkpoint = false; if (it == checkpoints_end) return true; if (it->height != height) return true; - is_sw_checkpoint = true; + is_hard_checkpoint = true; return h == it->hash; } -SWCheckpoint Currency::last_sw_checkpoint() const { +HardCheckpoint Currency::last_hard_checkpoint() const { if (checkpoints_begin == checkpoints_end) - return SWCheckpoint{0, genesis_block_hash}; + return HardCheckpoint{0, genesis_block_hash}; return *(checkpoints_end - 1); } -PublicKey Currency::get_checkpoint_public_key(uint32_t key_id) const { - if (key_id >= checkpoint_keys_end - checkpoint_keys_begin) +PublicKey Currency::get_checkpoint_public_key(size_t key_id) const { + if (key_id >= get_checkpoint_keys_count()) return PublicKey{}; return checkpoint_keys_begin[key_id]; } uint8_t Currency::get_block_major_version_for_height(Height height) const { - if (height < upgrade_height_v2) - return 1; - if (height >= upgrade_height_v2 && height < upgrade_height_v3) - return 2; - return 3; // info.height >= currency.upgrade_height_v3 + for (size_t i = 0; i < upgrade_heights.size(); ++i) + if (height < upgrade_heights[i]) + return static_cast(i + 1); + return static_cast(upgrade_heights.size() + 1); } Difficulty Currency::get_minimum_difficulty(uint8_t block_major_version) const { - if (block_major_version == 1) - return parameters::MINIMUM_DIFFICULTY_V1; - return minimum_difficulty; + if (block_major_version == 1 || net == "test") + return MINIMUM_DIFFICULTY_V1; + return MINIMUM_DIFFICULTY; } -uint32_t Currency::get_minimum_size_median(uint8_t block_major_version) const { +Height Currency::difficulty_windows_plus_lag() const { return DIFFICULTY_WINDOW + DIFFICULTY_LAG; } + +size_t Currency::get_minimum_size_median(uint8_t block_major_version) const { if (block_major_version == 1) - return parameters::MINIMUM_SIZE_MEDIAN_V1; + return MINIMUM_SIZE_MEDIAN_V1; if (block_major_version == 2) - return parameters::MINIMUM_SIZE_MEDIAN_V2; - return minimum_size_median; + return MINIMUM_SIZE_MEDIAN_V2; + if (block_major_version == 3) + return MINIMUM_SIZE_MEDIAN_V3; + return 0; +} + +size_t Currency::max_block_transactions_cumulative_size(Height height) const { + if (net != "main") + return 1024 * 1024; + // if (MAX_BLOCK_SIZE_GROWTH_PER_YEAR == 0) + // return MAX_BLOCK_SIZE_INITIAL; + static_assert( + MAX_BLOCK_SIZE_GROWTH_PER_YEAR == 0 || + std::numeric_limits::max() <= std::numeric_limits::max() / MAX_BLOCK_SIZE_GROWTH_PER_YEAR, + "MAX_BLOCK_SIZE_GROWTH_PER_YEAR too large"); + uint64_t max_size = + MAX_BLOCK_SIZE_INITIAL + (uint64_t(height) * MAX_BLOCK_SIZE_GROWTH_PER_YEAR) / expected_blocks_per_year(); + invariant(max_size < std::numeric_limits::max(), ""); + return static_cast(max_size); } -void Currency::get_block_reward(uint8_t block_major_version, size_t effective_median_size, size_t current_block_size, - Amount already_generated_coins, Amount fee, Amount *reward, SignedAmount *emission_change) const { +size_t Currency::minimum_anonymity(uint8_t block_major_version) const { + if (block_major_version >= amethyst_block_version) + return MINIMUM_ANONYMITY; + return MINIMUM_ANONYMITY_V1_3; +} + +Amount Currency::get_base_block_reward( + uint8_t block_major_version, Height height, Amount already_generated_coins) const { invariant(already_generated_coins <= money_supply, ""); invariant(emission_speed_factor > 0 && emission_speed_factor <= 8 * sizeof(Amount), ""); - Amount base_reward = (money_supply - already_generated_coins) >> emission_speed_factor; + return (money_supply - already_generated_coins) >> emission_speed_factor; +} - Amount penalized_base_reward = get_penalized_amount(base_reward, effective_median_size, current_block_size); +Amount Currency::get_block_reward(uint8_t block_major_version, Height height, size_t effective_median_size, + size_t current_transactions_size, Amount already_generated_coins, Amount fee, SignedAmount *emission_change) const { + Amount base_reward = get_base_block_reward(block_major_version, height, already_generated_coins); + + Amount penalized_base_reward = get_penalized_amount(base_reward, effective_median_size, current_transactions_size); Amount penalized_fee = - block_major_version >= 2 ? get_penalized_amount(fee, effective_median_size, current_block_size) : fee; + block_major_version >= 2 ? get_penalized_amount(fee, effective_median_size, current_transactions_size) : fee; - *emission_change = penalized_base_reward - (fee - penalized_fee); - *reward = penalized_base_reward + penalized_fee; + if (emission_change) + *emission_change = penalized_base_reward - (fee - penalized_fee); + return penalized_base_reward + penalized_fee; } Height Currency::largest_window() const { - return std::max(difficulty_blocks_count(), std::max(reward_blocks_window, timestamp_check_window)); -} - -uint32_t Currency::max_block_cumulative_size(Height height) const { - if (max_block_size_growth_per_year == 0) - return max_block_size_initial; - invariant(height <= std::numeric_limits::max() / max_block_size_growth_per_year, ""); - uint64_t max_size = - max_block_size_initial + (uint64_t(height) * max_block_size_growth_per_year) / expected_blocks_per_year(); - invariant(max_size < std::numeric_limits::max(), ""); - return static_cast(max_size); + return std::max(block_capacity_vote_window, + std::max(difficulty_windows_plus_lag(), + std::max(median_block_size_window, + std::max(BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW, BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW_V1_3)))); } -uint32_t Currency::max_transaction_allowed_size(uint32_t effective_block_size_median) const { - invariant(effective_block_size_median * 2 > miner_tx_blob_reserved_size, ""); - - return std::min(max_tx_size, effective_block_size_median * 2 - miner_tx_blob_reserved_size); -} - -void Currency::construct_miner_tx(uint8_t block_major_version, Height height, size_t effective_median_size, - Amount already_generated_coins, size_t current_block_size, Amount fee, Hash mineproof_seed, - const AccountPublicAddress &miner_address, Transaction *tx, const BinaryArray &extra_nonce) const { - const size_t max_outs = get_max_amount_outputs(); +Transaction Currency::construct_miner_tx( + uint8_t block_major_version, Height height, Amount block_reward, const AccountAddress &miner_address) const { + Transaction tx; + const size_t max_outs = get_max_coinbase_outputs(); // If we wish to limit number of outputs, it makes sense to round miner reward to some arbitrary number // Though this solution will reduce number of coins to mix - tx->inputs.clear(); - tx->outputs.clear(); - tx->extra.clear(); - - tx->inputs.push_back(CoinbaseInput{height}); - KeyPair txkey = mineproof_seed == Hash{} ? crypto::random_keypair() - : TransactionBuilder::deterministic_keys_from_seed(*tx, mineproof_seed); + tx.inputs.push_back(InputCoinbase{height}); - extra_add_transaction_public_key(tx->extra, txkey.public_key); - if (!extra_nonce.empty()) - extra_add_nonce(tx->extra, extra_nonce); + Hash tx_inputs_hash = get_transaction_inputs_hash(tx); + KeyPair txkey = crypto::random_keypair(); - Amount block_reward; - SignedAmount emission_change; - get_block_reward(block_major_version, effective_median_size, current_block_size, already_generated_coins, fee, - &block_reward, &emission_change); + extra_add_transaction_public_key(tx.extra, txkey.public_key); std::vector out_amounts; - decompose_amount(block_reward, default_dust_threshold, &out_amounts); + decompose_amount(block_reward, min_dust_threshold, &out_amounts); - while (out_amounts.size() > max_outs) { - out_amounts[out_amounts.size() - 2] += out_amounts.back(); + while (out_amounts.size() > max_outs && out_amounts.size() > 2) { + out_amounts.at(out_amounts.size() - 2) += out_amounts.back(); out_amounts.pop_back(); } Amount summary_amounts = 0; - for (size_t no = 0; no < out_amounts.size(); no++) { - KeyDerivation derivation{}; - PublicKey out_ephemeral_pub_key{}; - - if (!crypto::generate_key_derivation(miner_address.view_public_key, txkey.secret_key, derivation)) - throw json_rpc::Error{json_rpc::INVALID_PARAMS, - "Miner address has invalid public key " + common::pod_to_hex(miner_address.view_public_key)}; - - if (!crypto::derive_public_key(derivation, no, miner_address.spend_public_key, out_ephemeral_pub_key)) - throw json_rpc::Error{json_rpc::INVALID_PARAMS, - "Miner address has invalid public key " + common::pod_to_hex(miner_address.view_public_key)}; - - KeyOutput tk; - tk.public_key = out_ephemeral_pub_key; - - TransactionOutput out; - summary_amounts += out.amount = out_amounts[no]; - out.target = tk; - tx->outputs.push_back(out); + for (size_t out_index = 0; out_index < out_amounts.size(); out_index++) { + Hash output_secret = crypto::rand(); + OutputKey tk = TransactionBuilder::create_output( + miner_address, txkey.secret_key, tx_inputs_hash, out_index, output_secret); + tk.amount = out_amounts.at(out_index); + summary_amounts += tk.amount; + tx.outputs.push_back(tk); } invariant(summary_amounts == block_reward, ""); - // logger(ERROR, BrightRed) << "Failed to construct miner tx, - // summary_amounts = " << summary_amounts << " not - // equal block_reward = " << block_reward; - tx->version = current_transaction_version; - tx->unlock_block_or_timestamp = height + mined_money_unlock_window; + tx.version = miner_address.type() == typeid(AccountAddressSimple) ? 1 : amethyst_transaction_version; + // if mining on old address, we maintain binary compatibility + if (block_major_version < amethyst_block_version) + tx.unlock_block_or_timestamp = height + mined_money_unlock_window; + return tx; } -uint64_t Currency::get_penalized_amount(uint64_t amount, size_t median_size, size_t current_block_size) { +Amount Currency::get_penalized_amount(uint64_t amount, size_t median_size, size_t current_block_size) { static_assert(sizeof(size_t) >= sizeof(uint32_t), "size_t is too small"); invariant(current_block_size <= 2 * median_size, ""); invariant(median_size <= std::numeric_limits::max(), ""); @@ -317,7 +342,7 @@ uint64_t Currency::get_penalized_amount(uint64_t amount, size_t median_size, siz uint64_t product_hi; uint64_t product_lo = - mul128(amount, current_block_size * (UINT64_C(2) * median_size - current_block_size), &product_hi); + mul128(amount, current_block_size * (uint64_t(2) * median_size - current_block_size), &product_hi); uint64_t penalized_amount_hi; uint64_t penalized_amount_lo; @@ -331,39 +356,53 @@ uint64_t Currency::get_penalized_amount(uint64_t amount, size_t median_size, siz return penalized_amount_lo; } -std::string Currency::get_account_address_as_str(uint64_t prefix, const AccountPublicAddress &adr) { - BinaryArray ba = seria::to_binary(adr); - return common::base58::encode_addr(prefix, ba); -} - -bool Currency::parse_account_address_string(uint64_t *prefix, AccountPublicAddress *adr, const std::string &str) { - BinaryArray data; - - if (!common::base58::decode_addr(str, prefix, &data)) - return false; - try { - seria::from_binary(*adr, data); - } catch (const std::exception &) { - return false; +std::string Currency::account_address_as_string(const AccountAddress &v_addr) const { + if (v_addr.type() == typeid(AccountAddressSimple)) { + auto &addr = boost::get(v_addr); + BinaryArray ba = seria::to_binary(addr); + return common::base58::encode_addr(ADDRESS_BASE58_PREFIX, ba); } - return key_isvalid(adr->spend_public_key) && key_isvalid(adr->view_public_key); -} - -std::string Currency::account_address_as_string(const AccountPublicAddress &account_public_address) const { - return get_account_address_as_str(public_address_base58_prefix, account_public_address); + if (v_addr.type() == typeid(AccountAddressUnlinkable)) { + auto &addr = boost::get(v_addr); + BinaryArray ba = seria::to_binary(addr); + if (addr.is_auditable) + return common::base58::encode_addr(ADDRESS_BASE58_PREFIX_AUDITABLE_UNLINKABLE, ba); + return common::base58::encode_addr(ADDRESS_BASE58_PREFIX_UNLINKABLE, ba); + } + throw std::runtime_error("Unknown address type"); } -bool Currency::parse_account_address_string(const std::string &str, AccountPublicAddress *addr) const { - uint64_t prefix; - if (!parse_account_address_string(&prefix, addr, str)) { +bool Currency::parse_account_address_string(const std::string &str, AccountAddress *v_addr) const { + BinaryArray tag; + BinaryArray data; + if (!common::base58::decode_addr(str, 2 * sizeof(PublicKey), &tag, &data)) return false; + if (tag == ADDRESS_BASE58_PREFIX_UNLINKABLE || tag == ADDRESS_BASE58_PREFIX_AUDITABLE_UNLINKABLE) { + AccountAddressUnlinkable addr; + addr.is_auditable = (tag == ADDRESS_BASE58_PREFIX_AUDITABLE_UNLINKABLE); + try { + seria::from_binary(addr, data); + } catch (const std::exception &) { + return false; + } + if (!key_isvalid(addr.s) || !key_isvalid(addr.sv)) + return false; + *v_addr = addr; + return true; } - if (prefix != public_address_base58_prefix) { - // logger(DEBUGGING) << "Wrong address prefix: " << prefix << ", expected - // " << m_publicAddressBase58Prefix; - return false; + if (tag == ADDRESS_BASE58_PREFIX) { + AccountAddressSimple addr; + try { + seria::from_binary(addr, data); + } catch (const std::exception &) { + return false; + } + if (!key_isvalid(addr.spend_public_key) || !key_isvalid(addr.view_public_key)) + return false; + *v_addr = addr; + return true; } - return true; + return false; } static std::string ffw(Amount am, size_t digs) { @@ -439,28 +478,26 @@ bool Currency::parse_amount(size_t number_of_decimal_places, const std::string & Difficulty Currency::next_difficulty( std::vector *timestamps, std::vector *cumulative_difficulties) const { - invariant(difficulty_window >= 2, "Bad DIFFICULTY_WINDOW"); - invariant(2 * difficulty_cut <= difficulty_window - 2, "Bad DIFFICULTY_WINDOW or DIFFICULTY_CUT"); - - if (timestamps->size() > difficulty_window) { - timestamps->resize(difficulty_window); - cumulative_difficulties->resize(difficulty_window); + if (timestamps->size() > DIFFICULTY_WINDOW) { + timestamps->resize(DIFFICULTY_WINDOW); + cumulative_difficulties->resize(DIFFICULTY_WINDOW); } - size_t length = timestamps->size(); - invariant(length == cumulative_difficulties->size() && length <= difficulty_window, ""); + const size_t length = timestamps->size(); + invariant(length == cumulative_difficulties->size() && length <= DIFFICULTY_WINDOW, ""); if (length <= 1) return 1; std::sort(timestamps->begin(), timestamps->end()); size_t cut_begin, cut_end; - if (length <= difficulty_window - 2 * difficulty_cut) { + const size_t inner_window = DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT; + if (length <= inner_window) { cut_begin = 0; cut_end = length; } else { - cut_begin = (length - (difficulty_window - 2 * difficulty_cut) + 1) / 2; - cut_end = cut_begin + (difficulty_window - 2 * difficulty_cut); + cut_begin = (length - inner_window + 1) / 2; + cut_end = cut_begin + inner_window; } invariant(cut_begin + 2 <= cut_end && cut_end <= length, "After difficulty cut at least 2 items should remain"); Timestamp time_span = timestamps->at(cut_end - 1) - timestamps->at(cut_begin); @@ -493,95 +530,67 @@ BinaryArray Currency::get_block_long_hashing_data(const BlockHeader &bh, const B common::VectorOutputStream stream(result); seria::BinaryOutputStream ba(stream); ba.begin_object(); - ser_members(const_cast(bh), ba, BlockSeriaType::LONG_BLOCKHASH, body_proxy); + ser_members(const_cast(bh), ba, BlockSeriaType::LONG_BLOCKHASH, body_proxy, genesis_block_hash); ba.end_object(); - if (bh.major_version == 1) - return result; - if (bh.is_merge_mined()) { - TransactionExtraMergeMiningTag mm_tag; - if (!extra_get_merge_mining_tag(bh.parent_block.base_transaction.extra, mm_tag)) { - // logger(ERROR) << "merge mining tag wasn't found in extra of the parent - // block miner transaction"; - return BinaryArray{}; - } - if (mm_tag.depth != bh.parent_block.blockchain_branch.size()) - return BinaryArray{}; - if (bh.parent_block.blockchain_branch.size() > 8 * sizeof(genesis_block_hash)) - return BinaryArray{}; - Hash aux_blocks_merkle_root = crypto::tree_hash_from_branch(bh.parent_block.blockchain_branch.data(), - bh.parent_block.blockchain_branch.size(), get_auxiliary_block_header_hash(bh, body_proxy), - &genesis_block_hash); - - if (aux_blocks_merkle_root != mm_tag.merkle_root) { - // logger(ERROR, BRIGHT_YELLOW) << "Aux block hash wasn't found in merkle - // tree"; - return BinaryArray{}; - } - return result; - } -#if bytecoin_ALLOW_CM - if (bh.is_cm_mined()) { - Hash merkle_root_hash = crypto::tree_hash_from_branch(bh.cm_merkle_branch.data(), - bh.cm_merkle_branch.size(), - get_auxiliary_block_header_hash(bh, body_proxy), - &genesis_block_hash); - // We should not allow adding merkle_root_hash twice to the long_hashing_array, so more - // flexible "pre_nonce" | root | "post_nonce" would be bad (allow mining of sidechains) - BinaryArray long_hashing_array = bh.cm_nonce; - common::append(long_hashing_array, std::begin(merkle_root_hash.data), std::end(merkle_root_hash.data)); - return long_hashing_array; - } -#endif - throw std::runtime_error("Unknown block major version."); + return result; } -bool Currency::is_dust(Amount amount) const { +bool Currency::amount_allowed_in_output(uint8_t block_major_version, Amount amount) const { + if (block_major_version < amethyst_block_version) + return true; auto pretty_it = std::lower_bound(std::begin(PRETTY_AMOUNTS), std::end(PRETTY_AMOUNTS), amount); if (pretty_it != std::end(Currency::PRETTY_AMOUNTS) && *pretty_it == amount) - return amount >= 30000000000000000; // We have just a couple of large coins - return amount < 1000000; - // if (amount > 1000000) - // return true; - // if (net == "main") // Enough dust particles in mainnet already - // return amount > 1000 && amount % 1000 != 0; - // return true; - // We changed dust definition - rebuilding wallets will rearrange spendable and spendable_dust balances + return true; + if (amount > min_dust_threshold) // "crazy" amounts + return false; + return amount < 1000 || amount % 1000 == 0; // 3-digit dust } -Hash bytecoin::get_transaction_inputs_hash(const TransactionPrefix &tx) { +/*bool Currency::is_dust(Amount amount) const { + auto pretty_it = std::lower_bound(std::begin(PRETTY_AMOUNTS), std::end(PRETTY_AMOUNTS), amount); + if (pretty_it != std::end(Currency::PRETTY_AMOUNTS) && *pretty_it == amount) + return amount > max_dust_threshold; // We have just a couple of large coins + if (amount > min_dust_threshold) + return true; + return amount > 1000 && amount % 1000 != 0; + // We changed dust definition - rebuilding wallet caches will rearrange spendable and spendable_dust balances +}*/ + +Hash cn::get_transaction_inputs_hash(const TransactionPrefix &tx) { BinaryArray ba = seria::to_binary(tx.inputs); return crypto::cn_fast_hash(ba.data(), ba.size()); } -Hash bytecoin::get_transaction_prefix_hash(const TransactionPrefix &tx) { +Hash cn::get_transaction_prefix_hash(const TransactionPrefix &tx) { BinaryArray ba = seria::to_binary(tx); return crypto::cn_fast_hash(ba.data(), ba.size()); } -Hash bytecoin::get_transaction_hash(const Transaction &tx) { +Hash cn::get_transaction_hash(const Transaction &tx) { + if (tx.version >= TRANSACTION_VERSION_AMETHYST) { + std::pair ha; + ha.first = get_transaction_prefix_hash(tx); + BinaryArray binary_sigs = seria::to_binary(tx.signatures, static_cast(tx)); + ha.second = crypto::cn_fast_hash(binary_sigs.data(), binary_sigs.size()); + BinaryArray ba = seria::to_binary(ha); + return crypto::cn_fast_hash(ba.data(), ba.size()); + } BinaryArray ba = seria::to_binary(tx); return crypto::cn_fast_hash(ba.data(), ba.size()); } -Hash bytecoin::get_block_hash(const BlockHeader &bh, const BlockBodyProxy &body_proxy) { - // common::BinaryArray result; - // common::VectorOutputStream stream(result); - // seria::BinaryOutputStream ba(stream); - // ba.begin_object(); - // ser_members(const_cast(bh), ba, BlockSeriaType::BLOCKHASH, body_proxy); - // ba.end_object(); +Hash cn::get_block_hash(const BlockHeader &bh, const BlockBodyProxy &body_proxy) { + // get_object_hash prepends array size before hashing. + // this was a mistake of initial cryptonote developers Hash ha2 = get_object_hash(seria::to_binary(bh, BlockSeriaType::BLOCKHASH, body_proxy)); - // std::cout << "ha: " << ha2 << " ba: " << common::to_hex(result.data(), result.size()) << std::endl; + // std::cout << "ha: " << ha2 << " ba: " << common::to_hex(seria::to_binary(bh, BlockSeriaType::BLOCKHASH, + // body_proxy)) << std::endl; return ha2; } -Hash bytecoin::get_auxiliary_block_header_hash(const BlockHeader &bh, const BlockBodyProxy &body_proxy) { - // common::BinaryArray result; - // common::VectorOutputStream stream(result); - // seria::BinaryOutputStream ba(stream); - // ba.begin_object(); - // ser(const_cast(bh), ba, BlockSeriaType::PREHASH, body_proxy); - // ba.end_object(); +Hash cn::get_auxiliary_block_header_hash(const BlockHeader &bh, const BlockBodyProxy &body_proxy) { + // get_object_hash prepends array size before hashing. + // this was a mistake of initial cryptonote developers Hash ha2 = get_object_hash(seria::to_binary(bh, BlockSeriaType::PREHASH, body_proxy)); // std::cout << "ha: " << ha2 << " ba: " << common::to_hex(result.data(), result.size()) << std::endl; return ha2; diff --git a/src/Core/Currency.hpp b/src/Core/Currency.hpp index adc7c467..e687b17b 100644 --- a/src/Core/Currency.hpp +++ b/src/Core/Currency.hpp @@ -10,9 +10,9 @@ #include "Difficulty.hpp" #include "crypto/hash.hpp" -namespace bytecoin { +namespace cn { -class Currency { // Consensus calcs depend on those parameters +class Currency { // Consensus calculations depend on those parameters public: static const std::vector PRETTY_AMOUNTS; static const std::vector DECIMAL_PLACES; @@ -24,83 +24,76 @@ class Currency { // Consensus calcs depend on those parameters Hash genesis_block_hash{}; Height max_block_height; - uint32_t max_block_blob_size; - uint32_t max_tx_size; - uint64_t public_address_base58_prefix; Height mined_money_unlock_window; Height largest_window() const; // for limit on caching of headers - Height timestamp_check_window; + Height timestamp_check_window(uint8_t block_major_version) const; Timestamp block_future_time_limit; Amount money_supply; unsigned int emission_speed_factor; - Height reward_blocks_window; - uint32_t minimum_size_median; - uint32_t get_minimum_size_median(uint8_t block_major_version) const; + Height median_block_size_window; + Height block_capacity_vote_window; + size_t max_header_size; + size_t block_capacity_vote_min; + size_t block_capacity_vote_max; + size_t miner_tx_blob_reserved_size; + size_t get_recommended_max_transaction_size() const { + return block_capacity_vote_min - miner_tx_blob_reserved_size; + } + size_t get_minimum_size_median(uint8_t block_major_version) const; - uint32_t miner_tx_blob_reserved_size; + size_t max_block_transactions_cumulative_size(Height height) const; // Legacy checks + size_t minimum_anonymity(uint8_t block_major_version) const; size_t number_of_decimal_places; Amount coin() const { return DECIMAL_PLACES.at(number_of_decimal_places); } - size_t get_max_amount_outputs() const { return 14; } // 2 groups of 3 digits + 12 single digits - - Amount default_dust_threshold; + size_t get_max_amount_outputs() const { return 15; } // 2 groups of 3 digits + 13 single digits + size_t get_max_coinbase_outputs() const { return 10; } + Amount min_dust_threshold; + Amount max_dust_threshold; Amount self_dust_threshold; Timestamp difficulty_target; - Difficulty minimum_difficulty; Difficulty get_minimum_difficulty(uint8_t block_major_version) const; - Height difficulty_window; - Height difficulty_lag; - size_t difficulty_cut; - Height difficulty_blocks_count() const { return difficulty_window + difficulty_lag; } - uint32_t expected_blocks_per_day() const; - uint32_t expected_blocks_per_year() const; - uint32_t max_block_size_initial; - uint32_t max_block_size_growth_per_year; - - Timestamp locked_tx_allowed_delta_seconds; - Height locked_tx_allowed_delta_blocks; - - Height upgrade_height_v2; // height of first v2 block - Height upgrade_height_v3; // height of first v3 block + Height difficulty_windows_plus_lag() const; + Height expected_blocks_per_day() const; + Height expected_blocks_per_year() const; + + std::vector upgrade_heights; // Height of first V2 bloc, first V3 block, etc Height key_image_subgroup_checking_height; uint8_t get_block_major_version_for_height(Height) const; + uint8_t amethyst_block_version; + uint8_t amethyst_transaction_version; // upgrade voting threshold must not be reached before or at last sw checkpoint! uint8_t upgrade_from_major_version; uint8_t upgrade_indicator_minor_version; + bool is_upgrade_vote(uint8_t major, uint8_t minor) const; uint8_t upgrade_desired_major_version; Height upgrade_voting_window; - Height upgrade_votes_required; - Height upgrade_blocks_after_voting; - - uint8_t current_transaction_version; - - size_t sw_checkpoint_count() const { return checkpoints_end - checkpoints_begin; } - bool is_in_sw_checkpoint_zone(Height height) const; - bool check_sw_checkpoint(Height height, const Hash &h, bool &is_sw_checkpoint) const; - SWCheckpoint last_sw_checkpoint() const; - PublicKey get_checkpoint_public_key(uint32_t key_id) const; - uint32_t get_checkpoint_keys_count() const { - return static_cast(checkpoint_keys_end - checkpoint_keys_begin); - } - - void get_block_reward(uint8_t block_major_version, size_t effective_median_size, size_t current_block_size, - Amount already_generated_coins, Amount fee, Amount *reward, SignedAmount *emission_change) const; - uint32_t max_block_cumulative_size(Height height) const; - uint32_t max_transaction_allowed_size(uint32_t effective_block_size_median) const; - void construct_miner_tx(uint8_t block_major_version, Height height, size_t effective_median_size, - Amount already_generated_coins, size_t current_block_size, Amount fee, Hash mineproof_seed, - const AccountPublicAddress &miner_address, Transaction *tx, - const BinaryArray &extra_nonce = BinaryArray()) const; - - std::string account_address_as_string(const AccountPublicAddress &account_public_address) const; - bool parse_account_address_string(const std::string &str, AccountPublicAddress *addr) const; + Height upgrade_votes_required() const; + Height upgrade_window; + + size_t hard_checkpoint_count() const { return checkpoints_end - checkpoints_begin; } + bool is_in_hard_checkpoint_zone(Height height) const; + bool check_hard_checkpoint(Height height, const Hash &h, bool &is_hard_checkpoint) const; + HardCheckpoint last_hard_checkpoint() const; + PublicKey get_checkpoint_public_key(size_t key_id) const; + size_t get_checkpoint_keys_count() const { return checkpoint_keys_end - checkpoint_keys_begin; } + + Amount get_base_block_reward(uint8_t block_major_version, Height height, Amount already_generated_coins) const; + Amount get_block_reward(uint8_t block_major_version, Height height, size_t effective_median_size, + size_t current_transactions_size, Amount already_generated_coins, Amount fee, + SignedAmount *emission_change = nullptr) const; + Transaction construct_miner_tx( + uint8_t block_major_version, Height height, Amount block_reward, const AccountAddress &miner_address) const; + + std::string account_address_as_string(const AccountAddress &account_public_address) const; + bool parse_account_address_string(const std::string &str, AccountAddress *addr) const; std::string format_amount(Amount amount) const { return format_amount(number_of_decimal_places, amount); } std::string format_amount(SignedAmount amount) const { return format_amount(number_of_decimal_places, amount); } @@ -115,28 +108,24 @@ class Currency { // Consensus calcs depend on those parameters BinaryArray get_block_long_hashing_data(const BlockHeader &, const BlockBodyProxy &) const; - bool is_transaction_spend_time(BlockOrTimestamp unlock_time) const { return unlock_time >= max_block_height; } - bool is_transaction_spend_time_block(BlockOrTimestamp unlock_time) const { return unlock_time < max_block_height; } - bool is_transaction_spend_time_unlocked( - BlockOrTimestamp unlock_time, Height block_height, Timestamp block_time) const { - if (unlock_time < max_block_height) { // interpret as block index - return block_height + locked_tx_allowed_delta_blocks >= unlock_time; - } // else interpret as time - return block_time + locked_tx_allowed_delta_seconds >= unlock_time; - } - bool is_dust(Amount am) const; - static uint64_t get_penalized_amount(uint64_t amount, size_t median_size, size_t current_block_size); - static std::string get_account_address_as_str(uint64_t prefix, const AccountPublicAddress &adr); - static bool parse_account_address_string(uint64_t *prefix, AccountPublicAddress *adr, const std::string &str); + bool is_block_or_timestamp_timestamp(BlockOrTimestamp unlock_time) const { return unlock_time >= max_block_height; } + bool is_block_or_timestamp_block(BlockOrTimestamp unlock_time) const { return unlock_time < max_block_height; } + bool is_transaction_unlocked(uint8_t block_major_version, BlockOrTimestamp unlock_time, Height block_height, + Timestamp block_time, Timestamp block_median_time) const; + + // bool is_dust(Amount amount) const; + bool amount_allowed_in_output(uint8_t block_major_version, Amount amount) const; + + static uint64_t get_penalized_amount(uint64_t amount, size_t median_size, size_t current_transactions_size); static std::string format_amount(size_t number_of_decimal_places, Amount); static std::string format_amount(size_t number_of_decimal_places, SignedAmount); static bool parse_amount(size_t number_of_decimal_places, const std::string &, Amount *); private: - const PublicKey *checkpoint_keys_begin = nullptr; - const PublicKey *checkpoint_keys_end = nullptr; - const SWCheckpoint *checkpoints_begin = nullptr; - const SWCheckpoint *checkpoints_end = nullptr; + const PublicKey *checkpoint_keys_begin = nullptr; + const PublicKey *checkpoint_keys_end = nullptr; + const HardCheckpoint *checkpoints_begin = nullptr; + const HardCheckpoint *checkpoints_end = nullptr; }; // we should probaly find better place for these global funs @@ -148,4 +137,4 @@ Hash get_block_hash(const BlockHeader &, const BlockBodyProxy &); Hash get_auxiliary_block_header_hash(const BlockHeader &, const BlockBodyProxy &); // Auxilary hash, or prehash - inserted into MM or CM tree -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/Difficulty.cpp b/src/Core/Difficulty.cpp index a7994475..418b0f23 100644 --- a/src/Core/Difficulty.cpp +++ b/src/Core/Difficulty.cpp @@ -14,13 +14,13 @@ #include "crypto/int-util.h" #include "seria/ISeria.hpp" -using namespace bytecoin; +using namespace cn; // C99 6.2.5.9 - this is actually good working code by C standard // static bool cadd(uint64_t a, uint64_t b) { return a + b < a; } static bool cadc(uint64_t a, uint64_t b, bool c) { return a + b < a || (c && a + b == (uint64_t)-1); } -bool bytecoin::check_hash(const crypto::Hash &hash, Difficulty difficulty) { +bool cn::check_hash(const crypto::Hash &hash, Difficulty difficulty) { uint64_t hash64[4]; for (size_t i = 0; i != 4; ++i) hash64[i] = common::uint_le_from_bytes(hash.data + 8 * i, 8); diff --git a/src/Core/Difficulty.hpp b/src/Core/Difficulty.hpp index c28cb4f9..76464be5 100644 --- a/src/Core/Difficulty.hpp +++ b/src/Core/Difficulty.hpp @@ -9,9 +9,9 @@ #include "CryptoNote.hpp" #include "common/Int128.hpp" -namespace bytecoin { +namespace cn { bool check_hash(const crypto::Hash &hash, Difficulty difficulty); typedef common::Uint128 CumulativeDifficulty; -} +} // namespace cn diff --git a/src/Core/Multicore.cpp b/src/Core/Multicore.cpp index c8603414..35ffdcb3 100644 --- a/src/Core/Multicore.cpp +++ b/src/Core/Multicore.cpp @@ -3,11 +3,79 @@ #include "Multicore.hpp" #include "BlockChainState.hpp" +#include "Config.hpp" +#include "CryptoNoteTools.hpp" #include "Currency.hpp" #include "TransactionExtra.hpp" #include "crypto/crypto.hpp" +#include "platform/Network.hpp" -using namespace bytecoin; +using namespace cn; + +BlockPreparatorMulticore::BlockPreparatorMulticore(const Currency ¤cy, platform::EventLoop *main_loop) + : currency(currency), main_loop(main_loop) { + auto th_count = std::max(2, 3 * std::thread::hardware_concurrency() / 4); + // we use more energy but have the same speed when using hyperthreading + // std::cout << "Starting multicore ring checker using " << th_count << "/" << std::thread::hardware_concurrency() + // << " cpus" << std::endl; + for (size_t i = 0; i != th_count; ++i) + threads.emplace_back(&BlockPreparatorMulticore::thread_run, this); +} +BlockPreparatorMulticore::~BlockPreparatorMulticore() { + { + std::unique_lock lock(mu); + quit = true; + have_work.notify_all(); + } + for (auto &&th : threads) + th.join(); +} +void BlockPreparatorMulticore::thread_run() { + crypto::CryptoNightContext ctx; + while (true) { + std::tuple local_work; + { + std::unique_lock lock(mu); + if (quit) + return; + if (work.empty()) { + have_work.wait(lock); + continue; + } + local_work = std::move(work.front()); + work.pop_front(); + } + PreparedBlock pb(std::move(std::get<2>(local_work)), currency, std::get<1>(local_work) ? &ctx : nullptr); + { + std::unique_lock lock(mu); + prepared_blocks[std::get<0>(local_work)] = std::move(pb); + main_loop->wake(); // so we start processing on_idle + // prepared_blocks_ready.notify_all(); + } + } +} + +void BlockPreparatorMulticore::add_block(Hash bid, bool check_pow, RawBlock &&rb) { + std::unique_lock lock(mu); + work.push_back(std::make_tuple(bid, check_pow, std::move(rb))); + have_work.notify_all(); +} + +bool BlockPreparatorMulticore::get_prepared_block(Hash bid, PreparedBlock *pb) { + std::unique_lock lock(mu); + auto pid = prepared_blocks.find(bid); + if (pid == prepared_blocks.end()) + return false; + *pb = std::move(pid->second); + pid = prepared_blocks.erase(pid); + return true; +} + +bool BlockPreparatorMulticore::has_prepared_block(Hash bid) const { + std::unique_lock lock(mu); + auto pid = prepared_blocks.find(bid); + return pid != prepared_blocks.end(); +} RingCheckerMulticore::RingCheckerMulticore() { auto th_count = std::max(2, 3 * std::thread::hardware_concurrency() / 4); @@ -31,34 +99,43 @@ RingCheckerMulticore::~RingCheckerMulticore() { void RingCheckerMulticore::thread_run() { while (true) { RingSignatureArg arg; - int local_work_counter = 0; + RingSignatureArg3 arg3; + Height newest_referenced_height = 0; + int local_work_counter = 0; { std::unique_lock lock(mu); if (quit) return; - if (args.empty()) { + if (args.empty() && args3.empty()) { have_work.wait(lock); continue; } local_work_counter = work_counter; - arg = std::move(args.front()); - args.pop_front(); + if (!args.empty()) { + arg = std::move(args.front()); + newest_referenced_height = arg.newest_referenced_height; + args.pop_front(); + } else { + arg3 = std::move(args3.front()); + newest_referenced_height = arg3.newest_referenced_height; + args3.pop_front(); + } + } + bool result = false; + if (!arg.output_keys.empty()) { + result = crypto::check_ring_signature(arg.tx_prefix_hash, arg.key_image, arg.output_keys.data(), + arg.output_keys.size(), arg.input_signature, arg.key_image_subgroup_check); + } else { + result = crypto::check_ring_signature3( + arg3.tx_prefix_hash, arg3.key_images, arg3.output_keys, arg3.input_signature); } - std::vector output_key_pointers; - output_key_pointers.reserve(arg.output_keys.size()); - std::for_each(arg.output_keys.begin(), arg.output_keys.end(), - [&output_key_pointers](const PublicKey &key) { output_key_pointers.push_back(&key); }); - bool key_corrupted = false; - bool result = check_ring_signature(arg.tx_prefix_hash, arg.key_image, output_key_pointers.data(), - output_key_pointers.size(), arg.signatures.data(), arg.key_image_subgroup_check, &key_corrupted); { std::unique_lock lock(mu); if (local_work_counter == work_counter) { ready_counter += 1; - if (!result && key_corrupted) // TODO - db corrupted - errors.push_back("INPUT_CORRUPTED_SIGNATURES"); - if (!result && !key_corrupted) - errors.push_back("INPUT_INVALID_SIGNATURES"); + if (!result) + errors.push_back(ConsensusErrorBadOutputOrSignature{ + "Bad signature or output reference changed", newest_referenced_height}); result_ready.notify_all(); } } @@ -67,73 +144,92 @@ void RingCheckerMulticore::thread_run() { void RingCheckerMulticore::cancel_work() { std::unique_lock lock(mu); args.clear(); + args3.clear(); work_counter += 1; } -std::string RingCheckerMulticore::start_work_get_error(IBlockChainState *state, const Currency ¤cy, - const Block &block, Height unlock_height, Timestamp unlock_timestamp, bool key_image_subgroup_check) { +void RingCheckerMulticore::start_work(IBlockChainState *state, const Currency ¤cy, const Block &block, + Height unlock_height, Timestamp block_timestamp, Timestamp block_median_timestamp, bool key_image_subgroup_check) { { std::unique_lock lock(mu); args.clear(); + args3.clear(); errors.clear(); - // args.reserve(block.transactions.size()); ready_counter = 0; work_counter += 1; } total_counter = 0; for (auto &&transaction : block.transactions) { Hash tx_prefix_hash = get_transaction_prefix_hash(transaction); - size_t input_index = 0; - for (const auto &input : transaction.inputs) { - if (input.type() == typeid(CoinbaseInput)) { - } else if (input.type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(input); - RingSignatureArg arg; - arg.key_image_subgroup_check = key_image_subgroup_check; - arg.tx_prefix_hash = tx_prefix_hash; - arg.key_image = in.key_image; - arg.signatures = transaction.signatures[input_index]; - Height height = 0; + RingSignatureArg3 arg3; + for (size_t input_index = 0; input_index != transaction.inputs.size(); ++input_index) { + const auto &input = transaction.inputs.at(input_index); + Height newest_referenced_height = 0; + if (input.type() == typeid(InputKey)) { + const InputKey &in = boost::get(input); + Height height = 0; if (state->read_keyimage(in.key_image, &height)) - return "INPUT_KEYIMAGE_ALREADY_SPENT"; - if (in.output_indexes.empty()) - return "INPUT_UNKNOWN_TYPE"; - std::vector global_indexes(in.output_indexes.size()); - global_indexes[0] = in.output_indexes[0]; - for (size_t i = 1; i < in.output_indexes.size(); ++i) { - global_indexes[i] = global_indexes[i - 1] + in.output_indexes[i]; - } - arg.output_keys.resize(global_indexes.size()); + throw ConsensusErrorOutputSpent("Output already spent", in.key_image, height); + std::vector global_indexes; + if (!relative_output_offsets_to_absolute(&global_indexes, in.output_indexes)) + throw ConsensusError("Output indexes invalid in input"); + std::vector output_keys(global_indexes.size()); for (size_t i = 0; i != global_indexes.size(); ++i) { IBlockChainState::UnlockTimePublickKeyHeightSpent unp; if (!state->read_amount_output(in.amount, global_indexes[i], &unp)) - return "INPUT_INVALID_GLOBAL_INDEX"; - if (!currency.is_transaction_spend_time_unlocked( - unp.unlock_block_or_timestamp, unlock_height, unlock_timestamp)) - return "INPUT_SPEND_LOCKED_OUT"; - arg.output_keys[i] = unp.public_key; + throw ConsensusErrorOutputDoesNotExist("Output does not exist", input_index, global_indexes[i]); + if (unp.auditable && global_indexes.size() != 1) + throw ConsensusErrorBadOutputOrSignature("Auditable output mixed", unp.height); + if (!currency.is_transaction_unlocked(block.header.major_version, unp.unlock_block_or_timestamp, + unlock_height, block_timestamp, block_median_timestamp)) + throw ConsensusErrorBadOutputOrSignature("Output locked", unp.height); + output_keys[i] = unp.public_key; + newest_referenced_height = std::max(newest_referenced_height, unp.height); } // As soon as first arg is ready, other thread can start work while we // continue reading from slow DB - total_counter += 1; - std::unique_lock lock(mu); - args.push_back(std::move(arg)); - have_work.notify_all(); + if (transaction.signatures.type() == typeid(RingSignatures)) { + auto &signatures = boost::get(transaction.signatures); + RingSignatureArg arg; + arg.key_image_subgroup_check = key_image_subgroup_check; + arg.tx_prefix_hash = tx_prefix_hash; + arg.newest_referenced_height = newest_referenced_height; + arg.key_image = in.key_image; + arg.output_keys = std::move(output_keys); + arg.input_signature = signatures.signatures.at(input_index); + total_counter += 1; + std::unique_lock lock(mu); + args.push_back(std::move(arg)); + have_work.notify_all(); + } else if (transaction.signatures.type() == typeid(RingSignature3)) { + auto &signatures = boost::get(transaction.signatures); + arg3.output_keys.push_back(std::move(output_keys)); + arg3.newest_referenced_height = std::max(arg3.newest_referenced_height, newest_referenced_height); + arg3.key_images.push_back(in.key_image); + if (arg3.input_signature.r.empty()) + arg3.input_signature = signatures; + } else + throw ConsensusError("Unknown signatures type"); } - input_index++; + } + if (!arg3.output_keys.empty()) { + arg3.tx_prefix_hash = tx_prefix_hash; + total_counter += 1; + std::unique_lock lock(mu); + args3.push_back(std::move(arg3)); + have_work.notify_all(); } } - return std::string(); } -bool RingCheckerMulticore::signatures_valid() const { +std::vector RingCheckerMulticore::move_errors() { while (true) { std::unique_lock lock(mu); if (ready_counter != total_counter) { result_ready.wait(lock); continue; } - return errors.empty(); + return std::move(errors); } } @@ -156,42 +252,48 @@ WalletPreparatorMulticore::~WalletPreparatorMulticore() { th.join(); } -PreparedWalletTransaction::PreparedWalletTransaction(TransactionPrefix &&ttx, const SecretKey &view_secret_key) - : tx(std::move(ttx)) { +PreparedWalletTransaction::PreparedWalletTransaction( + TransactionPrefix &&ttx, TransactionSignatures &&sigs, const Wallet::OutputHandler &o_handler) + : tx(std::move(ttx)), sigs(std::move(sigs)) { + // We ignore results of most crypto calls here and absence of tx_public_key + // All errors will lead to spend_key not found in our wallet PublicKey tx_public_key = extra_get_transaction_public_key(tx.extra); - if (!generate_key_derivation(tx_public_key, view_secret_key, derivation)) - return; + prefix_hash = get_transaction_prefix_hash(tx); + inputs_hash = get_transaction_inputs_hash(tx); + KeyPair tx_keys; - size_t key_index = 0; - uint32_t out_index = 0; - spend_keys.reserve(tx.outputs.size()); - for (const auto &output : tx.outputs) { - if (output.target.type() == typeid(KeyOutput)) { - const KeyOutput &key_output = boost::get(output.target); - PublicKey spend_key; - underive_public_key(derivation, key_index, key_output.public_key, - spend_key); // error indicated by spend_key not in our wallet - spend_keys.push_back(spend_key); - ++key_index; - } - ++out_index; + spend_keys.resize(tx.outputs.size()); + output_secret_scalars.resize(tx.outputs.size()); + for (size_t out_index = 0; out_index != tx.outputs.size(); ++out_index) { + const auto &output = tx.outputs.at(out_index); + if (output.type() != typeid(OutputKey)) + continue; + const auto &key_output = boost::get(output); + o_handler(tx_public_key, &derivation, inputs_hash, out_index, key_output, &spend_keys.at(out_index), + &output_secret_scalars.at(out_index)); } } +PreparedWalletTransaction::PreparedWalletTransaction(Transaction &&tx, const Wallet::OutputHandler &o_handler) + : PreparedWalletTransaction(std::move(static_cast(tx)), std::move(tx.signatures), o_handler) { +} + PreparedWalletBlock::PreparedWalletBlock(BlockTemplate &&bc_header, std::vector &&raw_transactions, - Hash base_transaction_hash, const SecretKey &view_secret_key) + std::vector &&signatures, Hash base_transaction_hash, const Wallet::OutputHandler &o_handler) : base_transaction_hash(base_transaction_hash) { - header = bc_header; - base_transaction = PreparedWalletTransaction(std::move(bc_header.base_transaction), view_secret_key); + header = bc_header; + base_transaction = + PreparedWalletTransaction(std::move(bc_header.base_transaction), TransactionSignatures{}, o_handler); transactions.reserve(raw_transactions.size()); for (size_t tx_index = 0; tx_index != raw_transactions.size(); ++tx_index) { - transactions.emplace_back(std::move(raw_transactions.at(tx_index)), view_secret_key); + transactions.emplace_back(std::move(raw_transactions.at(tx_index)), + tx_index < signatures.size() ? std::move(signatures.at(tx_index)) : TransactionSignatures{}, o_handler); } } void WalletPreparatorMulticore::thread_run() { while (true) { - SecretKey view_secret_key; + Wallet::OutputHandler o_handler; Height height = 0; int local_work_counter = 0; api::RawBlock sync_block; @@ -205,14 +307,14 @@ void WalletPreparatorMulticore::thread_run() { continue; } local_work_counter = work_counter; - view_secret_key = work_secret_key; + o_handler = m_o_handler; height = work.start_height; sync_block = std::move(work.blocks.front()); work.start_height += 1; work.blocks.erase(work.blocks.begin()); } PreparedWalletBlock result(std::move(sync_block.raw_header), std::move(sync_block.raw_transactions), - sync_block.transactions.at(0).hash, view_secret_key); + std::move(sync_block.signatures), sync_block.transactions.at(0).hash, o_handler); { std::unique_lock lock(mu); if (local_work_counter == work_counter) { @@ -225,16 +327,16 @@ void WalletPreparatorMulticore::thread_run() { void WalletPreparatorMulticore::cancel_work() { std::unique_lock lock(mu); - work = api::bytecoind::SyncBlocks::Response(); + work = api::cnd::SyncBlocks::Response(); prepared_blocks.clear(); work_counter += 1; } -void WalletPreparatorMulticore::start_work(const api::bytecoind::SyncBlocks::Response &new_work, - const SecretKey &view_secret_key) { +void WalletPreparatorMulticore::start_work( + const api::cnd::SyncBlocks::Response &new_work, Wallet::OutputHandler &&o_handler) { std::unique_lock lock(mu); - work = new_work; - work_secret_key = view_secret_key; + work = new_work; + m_o_handler = std::move(o_handler); work_counter += 1; have_work.notify_all(); } diff --git a/src/Core/Multicore.hpp b/src/Core/Multicore.hpp index 614d828a..4917d91d 100644 --- a/src/Core/Multicore.hpp +++ b/src/Core/Multicore.hpp @@ -7,23 +7,61 @@ #include #include #include +#include "BlockChain.hpp" // for PreparedBlock #include "CryptoNote.hpp" +#include "Wallet.hpp" // for OutputHandler #include "rpc_api.hpp" // Experimental machinery to offload heavy calcs to other cores // Without making any critical part of the Core multithreaded // We do it by confining threads to "boxes" -namespace bytecoin { +namespace platform { +class EventLoop; +} +namespace cn { class IBlockChainState; // We will read keyimages and outputs from it class Currency; +class BlockPreparatorMulticore { + const Currency ¤cy; + + std::vector threads; + mutable std::mutex mu; + std::condition_variable have_work; + platform::EventLoop *main_loop = nullptr; + // std::condition_variable prepared_blocks_ready; + bool quit = false; + + std::deque> work; + std::map prepared_blocks; + + void thread_run(); + +public: + explicit BlockPreparatorMulticore(const Currency ¤cy, platform::EventLoop *main_loop); + ~BlockPreparatorMulticore(); + + void add_block(Hash bid, bool check_pow, RawBlock &&rb); + bool get_prepared_block(Hash bid, PreparedBlock *pb); + bool has_prepared_block(Hash bid) const; +}; + struct RingSignatureArg { Hash tx_prefix_hash; + Height newest_referenced_height = 0; KeyImage key_image; bool key_image_subgroup_check = false; std::vector output_keys; - std::vector signatures; + RingSignature input_signature; +}; + +struct RingSignatureArg3 { + Hash tx_prefix_hash; + Height newest_referenced_height = 0; + std::vector key_images; + std::vector> output_keys; + RingSignature3 input_signature; }; class RingCheckerMulticore { @@ -35,9 +73,10 @@ class RingCheckerMulticore { size_t total_counter = 0; size_t ready_counter = 0; - std::vector errors; + std::vector errors; std::deque args; + std::deque args3; int work_counter = 0; void thread_run(); @@ -45,18 +84,25 @@ class RingCheckerMulticore { RingCheckerMulticore(); ~RingCheckerMulticore(); void cancel_work(); - std::string start_work_get_error(IBlockChainState *state, const Currency ¤cy, const Block &block, - Height unlock_height, Timestamp unlock_timestamp, bool key_image_subgroup_check); // can fail immediately - bool signatures_valid() const; + void start_work(IBlockChainState *state, const Currency ¤cy, const Block &block, Height unlock_height, + Timestamp block_timestamp, Timestamp block_median_timestamp, + bool key_image_subgroup_check); // can throw ConsensusError immediately + std::vector move_errors(); }; struct PreparedWalletTransaction { TransactionPrefix tx; - KeyDerivation derivation; + TransactionSignatures sigs; + Hash prefix_hash; + Hash inputs_hash; + boost::optional derivation; // Will be assigned on first actual use std::vector spend_keys; + std::vector output_secret_scalars; - PreparedWalletTransaction() {} - PreparedWalletTransaction(TransactionPrefix &&tx, const SecretKey &view_secret_key); + PreparedWalletTransaction() = default; + PreparedWalletTransaction( + TransactionPrefix &&tx, TransactionSignatures &&sigs, const Wallet::OutputHandler &o_handler); + PreparedWalletTransaction(Transaction &&tx, const Wallet::OutputHandler &o_handler); }; struct PreparedWalletBlock { @@ -64,9 +110,10 @@ struct PreparedWalletBlock { PreparedWalletTransaction base_transaction; Hash base_transaction_hash; std::vector transactions; - PreparedWalletBlock() {} + PreparedWalletBlock() = default; PreparedWalletBlock(BlockTemplate &&bc_header, std::vector &&raw_transactions, - Hash base_transaction_hash, const SecretKey &view_secret_key); + std::vector &&signatures, Hash base_transaction_hash, + const Wallet::OutputHandler &o_handler); }; class WalletPreparatorMulticore { @@ -77,16 +124,16 @@ class WalletPreparatorMulticore { bool quit = false; std::map prepared_blocks; - api::bytecoind::SyncBlocks::Response work; + api::cnd::SyncBlocks::Response work; int work_counter = 0; - SecretKey work_secret_key; + Wallet::OutputHandler m_o_handler; void thread_run(); public: WalletPreparatorMulticore(); ~WalletPreparatorMulticore(); void cancel_work(); - void start_work(const api::bytecoind::SyncBlocks::Response &new_work, const SecretKey &view_secret_key); + void start_work(const api::cnd::SyncBlocks::Response &new_work, Wallet::OutputHandler &&o_handler); PreparedWalletBlock get_ready_work(Height height); }; -} +} // namespace cn diff --git a/src/Core/Node.cpp b/src/Core/Node.cpp index 264d8241..309a16e9 100644 --- a/src/Core/Node.cpp +++ b/src/Core/Node.cpp @@ -5,11 +5,15 @@ #include #include #include +#include "BlockChainFileFormat.hpp" #include "Config.hpp" #include "CryptoNoteTools.hpp" #include "TransactionExtra.hpp" #include "common/JsonValue.hpp" +#include "http/Client.hpp" +#include "http/Server.hpp" #include "platform/PathTools.hpp" +#include "platform/PreventSleep.hpp" #include "platform/Time.hpp" #include "seria/BinaryInputStream.hpp" #include "seria/BinaryOutputStream.hpp" @@ -17,7 +21,7 @@ #include "seria/KVBinaryOutputStream.hpp" #include "version.hpp" -using namespace bytecoin; +using namespace cn; Node::Node(logging::ILogger &log, const Config &config, BlockChainState &block_chain) : m_block_chain(block_chain) @@ -30,44 +34,46 @@ Node::Node(logging::ILogger &log, const Config &config, BlockChainState &block_c , m_multicast_timer(std::bind(&Node::send_multicast, this)) , m_start_time(m_p2p.get_local_time()) , m_commit_timer(std::bind(&Node::db_commit, this)) - , m_downloader(this, block_chain) - , m_downloader_v3(this, block_chain) { - const std::string old_path = platform::get_default_data_directory(config.crypto_note_name); + , log_request_timestamp(std::chrono::steady_clock::now()) + , log_response_timestamp(std::chrono::steady_clock::now()) + , m_pow_checker(block_chain.get_currency(), platform::EventLoop::current()) { + const std::string old_path = platform::get_default_data_directory(CRYPTONOTE_NAME); const std::string new_path = config.get_data_folder(); m_block_chain_reader1 = std::make_unique( - block_chain.get_currency(), new_path + "/blockindexes.bin", new_path + "/blocks.bin"); + block_chain.get_currency(), new_path + config.block_indexes_file_name, new_path + config.blocks_file_name); if (m_block_chain_reader1->get_block_count() <= block_chain.get_tip_height()) m_block_chain_reader1.reset(); if (!config.bytecoind_bind_ip.empty() && config.bytecoind_bind_port != 0) - m_api.reset(new http::Server(config.bytecoind_bind_ip, config.bytecoind_bind_port, + m_api = std::make_unique(config.bytecoind_bind_ip, config.bytecoind_bind_port, std::bind(&Node::on_api_http_request, this, _1, _2, _3), std::bind(&Node::on_api_http_disconnect, this, _1), config.ssl_certificate_pem_file, - config.ssl_certificate_password ? config.ssl_certificate_password.get() : std::string())); + config.ssl_certificate_password ? config.ssl_certificate_password.get() : std::string()); - m_commit_timer.once(m_config.db_commit_period_blockchain); + m_commit_timer.once(float(m_config.db_commit_period_blockchain)); advance_long_poll(); send_multicast(); } +Node::~Node() {} // we have unique_ptr to incomplete type + void Node::send_multicast() { if (m_config.multicast_period == 0) return; - std::cout << "sending multicast about node listening on port=" << m_config.p2p_external_port << std::endl; - BinaryArray ha = P2PProtocolNew::create_multicast_announce( - m_block_chain.get_currency().genesis_block_hash, m_config.p2p_external_port); + // std::cout << "sending multicast about node listening on port=" << m_config.p2p_external_port << std::endl; + BinaryArray ha = P2PProtocolBasic::create_multicast_announce( + m_config.network_id, m_block_chain.get_currency().genesis_block_hash, m_config.p2p_external_port); platform::UDPMulticast::send(m_config.multicast_address, m_config.multicast_port, ha.data(), ha.size()); m_multicast_timer.once(m_config.multicast_period); } void Node::on_multicast(const std::string &addr, const unsigned char *data, size_t size) { - // std::cout << " on_multicast from=" << addr << " size=" << size << std::endl; NetworkAddress na; - na.port = P2PProtocolNew::parse_multicast_announce(data, size, m_block_chain.get_currency().genesis_block_hash); + na.port = P2PProtocolBasic::parse_multicast_announce( + data, size, m_config.network_id, m_block_chain.get_currency().genesis_block_hash); if (!na.port) return; if (common::parse_ip_address(addr, &na.ip)) { - // std::cout << "* good on_multicast from=" << na << " size=" << size << std::endl; // TODO - remove if (m_peer_db.add_incoming_peer(na, m_p2p.get_local_time())) m_log(logging::INFO) << "Adding peer from multicast announce addr=" << na << std::endl; } @@ -81,13 +87,38 @@ void Node::on_multicast(const std::string &addr, const unsigned char *data, size void Node::db_commit() { m_block_chain.db_commit(); - m_commit_timer.once(m_config.db_commit_period_blockchain); + m_commit_timer.once(float(m_config.db_commit_period_blockchain)); +} + +void Node::remove_chain_block(std::map::iterator it) { + invariant(it->second.chain_counter > 0, ""); + it->second.chain_counter -= 1; + if (it->second.chain_counter == 0 && !it->second.preparing) + chain_blocks.erase(it); +} + +void Node::advance_all_downloads() { + for (auto &&who : m_broadcast_protocols) + who->advance_blocks(); } bool Node::on_idle() { + auto idle_start = std::chrono::steady_clock::now(); + Hash was_top_bid = m_block_chain.get_tip_bid(); + bool on_idle_result = false; if (!m_block_chain_reader1 && !m_block_chain_reader2 && - m_block_chain.get_tip_height() >= m_block_chain.internal_import_known_height()) - return m_downloader.on_idle(); + m_block_chain.get_tip_height() >= m_block_chain.internal_import_known_height()) { + for (size_t s = 0; s != 10; ++s) { + bool on_idle_result_s = false; + std::vector bp_copy{m_broadcast_protocols.begin(), m_broadcast_protocols.end()}; + // We need bp_copy because on_idle can disconnect, modifying m_broadcast_protocols + for (auto &&who : bp_copy) + on_idle_result_s = who->on_idle(idle_start) | on_idle_result_s; + if (!on_idle_result_s) + break; + on_idle_result = true; + } + } if (m_block_chain.get_tip_height() < m_block_chain.internal_import_known_height()) m_block_chain.internal_import(); else { @@ -98,43 +129,16 @@ bool Node::on_idle() { m_block_chain_reader2.reset(); } } - advance_long_poll(); - m_downloader.advance_download(); - return true; -} - -void Node::sync_transactions(P2PProtocolBytecoin *who) { - NOTIFY_REQUEST_TX_POOL::request msg; - auto mytxs = m_block_chain.get_memory_state_transactions(); - msg.txs.reserve(mytxs.size()); - for (auto &&tx : mytxs) { - msg.txs.push_back(tx.first); + if (m_block_chain.get_tip_bid() != was_top_bid) { + advance_long_poll(); } - BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_REQUEST_TX_POOL::ID, LevinProtocol::encode(msg), false); - who->send(std::move(raw_msg)); -} - -bool Node::check_trust(const np::ProofOfTrust &tr) { - uint64_t local_time = platform::now_unix_timestamp(); - uint64_t time_delta = local_time > tr.time ? local_time - tr.time : tr.time - local_time; - - if (time_delta > 24 * 60 * 60) - return false; - if (m_last_stat_request_time >= tr.time) - return false; - if (m_p2p.get_unique_number() != tr.peer_id) - return false; - - Hash h = tr.get_hash(); - if (!crypto::check_signature(h, m_config.trusted_public_key, tr.sign)) - return false; - m_last_stat_request_time = tr.time; - return true; + advance_all_downloads(); + return on_idle_result; } -bool Node::check_trust(const ProofOfTrustLegacy &tr) { - uint64_t local_time = platform::now_unix_timestamp(); - uint64_t time_delta = local_time > tr.time ? local_time - tr.time : tr.time - local_time; +bool Node::check_trust(const p2p::ProofOfTrust &tr) { + Timestamp local_time = platform::now_unix_timestamp(); + Timestamp time_delta = local_time > tr.time ? local_time - tr.time : tr.time - local_time; if (time_delta > 24 * 60 * 60) return false; @@ -151,18 +155,18 @@ bool Node::check_trust(const ProofOfTrustLegacy &tr) { } void Node::advance_long_poll() { const auto now = m_p2p.get_local_time(); - if (!prevent_sleep && m_block_chain.get_tip().timestamp < now - 86400) - prevent_sleep = std::make_unique("Downloading blockchain"); - if (prevent_sleep && + if (!m_prevent_sleep && m_block_chain.get_tip().timestamp < now - 86400) + m_prevent_sleep = std::make_unique("Downloading blockchain"); + if (m_prevent_sleep && m_block_chain.get_tip().timestamp > now - m_block_chain.get_currency().block_future_time_limit * 2) - prevent_sleep = nullptr; + m_prevent_sleep = nullptr; if (m_long_poll_http_clients.empty()) return; - const api::bytecoind::GetStatus::Response resp = create_status_response(); + const api::cnd::GetStatus::Response resp = create_status_response(); for (auto lit = m_long_poll_http_clients.begin(); lit != m_long_poll_http_clients.end();) { - const bool method_status = lit->original_json_request.get_method() == api::bytecoind::GetStatus::method() || - lit->original_json_request.get_method() == api::bytecoind::GetStatus::method2(); + const bool method_status = lit->original_json_request.get_method() == api::cnd::GetStatus::method() || + lit->original_json_request.get_method() == api::cnd::GetStatus::method2(); if (method_status && !resp.ready_for_longpoll(lit->original_get_status)) { ++lit; continue; @@ -173,7 +177,7 @@ void Node::advance_long_poll() { continue; } const common::JsonValue &jid = lit->original_json_request.get_id().get(); - http::ResponseData last_http_response; + http::ResponseBody last_http_response; last_http_response.r.headers.push_back({"Content-Type", "application/json; charset=utf-8"}); last_http_response.r.status = 200; last_http_response.r.http_version_major = lit->original_request.r.http_version_major; @@ -183,10 +187,10 @@ void Node::advance_long_poll() { last_http_response.set_body(json_rpc::create_response_body(resp, jid)); } else { try { - api::bytecoind::GetBlockTemplate::Request gbt_req; + api::cnd::GetBlockTemplate::Request gbt_req; lit->original_json_request.load_params(gbt_req); - api::bytecoind::GetBlockTemplate::Response gbt_res; - getblocktemplate(std::move(gbt_req), gbt_res); + api::cnd::GetBlockTemplate::Response gbt_res; + getblocktemplate(gbt_req, gbt_res); last_http_response.set_body(json_rpc::create_response_body(gbt_res, jid)); } catch (const json_rpc::Error &err) { last_http_response.set_body(json_rpc::create_error_response_body(err, jid)); @@ -205,12 +209,12 @@ static const std::string beautiful_index_start = -bytecoind • version +)" CRYPTONOTE_NAME R"(d • version )"; static const std::string beautiful_index_finish = " "; static const std::string robots_txt = "User-agent: *\r\nDisallow: /"; -bool Node::on_api_http_request(http::Client *who, http::RequestData &&request, http::ResponseData &response) { +bool Node::on_api_http_request(http::Client *who, http::RequestBody &&request, http::ResponseBody &response) { response.r.add_headers_nocache(); if (request.r.uri == "/robots.txt") { response.r.headers.push_back({"Content-Type", "text/plain; charset=UTF-8"}); @@ -236,13 +240,13 @@ bool Node::on_api_http_request(http::Client *who, http::RequestData &&request, h response.set_body(std::move(body)); return true; } - if (request.r.uri == api::bytecoind::url()) { + if (request.r.uri == api::cnd::url()) { if (!on_json_rpc(who, std::move(request), response)) return false; response.r.status = 200; return true; } - if (request.r.uri == api::bytecoind::binary_url()) { + if (request.r.uri == api::cnd::binary_url()) { if (!on_binary_rpc(who, std::move(request), response)) return false; response.r.status = 200; @@ -262,52 +266,57 @@ void Node::on_api_http_disconnect(http::Client *who) { } const std::unordered_map Node::m_binaryrpc_handlers = { - {api::bytecoind::SyncBlocks::method(), json_rpc::make_binary_member_method(&Node::on_sync_blocks)}, - {api::bytecoind::SyncMemPool::method(), json_rpc::make_binary_member_method(&Node::on_sync_mempool)}}; + {api::cnd::SyncBlocks::bin_method(), json_rpc::make_binary_member_method(&Node::on_sync_blocks)}, + {api::cnd::SyncMemPool::bin_method(), json_rpc::make_binary_member_method(&Node::on_sync_mempool)}}; std::unordered_map Node::m_jsonrpc_handlers = { - {api::bytecoind::GetLastBlockHeaderLegacy::method(), json_rpc::make_member_method(&Node::on_get_last_block_header)}, - {api::bytecoind::GetBlockHeaderByHashLegacy::method(), - json_rpc::make_member_method(&Node::on_get_block_header_by_hash)}, - {api::bytecoind::GetBlockHeaderByHeightLegacy::method(), + {api::cnd::GetLastBlockHeaderLegacy::method(), json_rpc::make_member_method(&Node::on_get_last_block_header)}, + {api::cnd::GetBlockHeaderByHashLegacy::method(), json_rpc::make_member_method(&Node::on_get_block_header_by_hash)}, + {api::cnd::GetBlockHeaderByHeightLegacy::method(), json_rpc::make_member_method(&Node::on_get_block_header_by_height)}, - {api::bytecoind::GetBlockTemplate::method(), json_rpc::make_member_method(&Node::on_getblocktemplate)}, - {api::bytecoind::GetBlockTemplate::method_legacy(), json_rpc::make_member_method(&Node::on_getblocktemplate)}, - {api::bytecoind::GetCurrencyId::method(), json_rpc::make_member_method(&Node::on_get_currency_id)}, - {api::bytecoind::GetCurrencyId::method_legacy(), json_rpc::make_member_method(&Node::on_get_currency_id)}, - {api::bytecoind::SubmitBlock::method(), json_rpc::make_member_method(&Node::on_submitblock)}, - {api::bytecoind::SubmitBlockLegacy::method(), json_rpc::make_member_method(&Node::on_submitblock_legacy)}, - {api::bytecoind::GetRandomOutputs::method(), json_rpc::make_member_method(&Node::on_get_random_outputs)}, - {api::bytecoind::GetStatus::method(), json_rpc::make_member_method(&Node::on_get_status)}, - {api::bytecoind::GetStatus::method2(), json_rpc::make_member_method(&Node::on_get_status)}, - {api::bytecoind::GetStatistics::method(), json_rpc::make_member_method(&Node::on_get_statistics)}, - {api::bytecoind::GetArchive::method(), json_rpc::make_member_method(&Node::on_get_archive)}, - {api::bytecoind::SendTransaction::method(), json_rpc::make_member_method(&Node::handle_send_transaction)}, - {api::bytecoind::CheckSendproof::method(), json_rpc::make_member_method(&Node::handle_check_sendproof)}, - {api::bytecoind::SyncBlocks::method(), json_rpc::make_member_method(&Node::on_sync_blocks)}, - {api::bytecoind::GetRawBlock::method(), json_rpc::make_member_method(&Node::on_get_raw_block)}, - {api::bytecoind::GetBlockHeader::method(), json_rpc::make_member_method(&Node::on_get_block_header)}, - {api::bytecoind::GetRawTransaction::method(), json_rpc::make_member_method(&Node::on_get_raw_transaction)}, - {api::bytecoind::SyncMemPool::method(), json_rpc::make_member_method(&Node::on_sync_mempool)}}; - -bool Node::on_get_random_outputs(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetRandomOutputs::Request &&request, api::bytecoind::GetRandomOutputs::Response &response) { + {api::cnd::GetBlockTemplate::method(), json_rpc::make_member_method(&Node::on_getblocktemplate)}, + {api::cnd::GetBlockTemplate::method_legacy(), json_rpc::make_member_method(&Node::on_getblocktemplate)}, + {api::cnd::GetCurrencyId::method(), json_rpc::make_member_method(&Node::on_get_currency_id)}, + {api::cnd::GetCurrencyId::method_legacy(), json_rpc::make_member_method(&Node::on_get_currency_id)}, + {api::cnd::SubmitBlock::method(), json_rpc::make_member_method(&Node::on_submitblock)}, + {api::cnd::SubmitBlockLegacy::method(), json_rpc::make_member_method(&Node::on_submitblock_legacy)}, + {api::cnd::GetRandomOutputs::method(), json_rpc::make_member_method(&Node::on_get_random_outputs)}, + {api::cnd::GetStatus::method(), json_rpc::make_member_method(&Node::on_get_status)}, + {api::cnd::GetStatus::method2(), json_rpc::make_member_method(&Node::on_get_status)}, + {api::cnd::GetStatistics::method(), json_rpc::make_member_method(&Node::on_get_statistics)}, + {api::cnd::GetArchive::method(), json_rpc::make_member_method(&Node::on_get_archive)}, + {api::cnd::SendTransaction::method(), json_rpc::make_member_method(&Node::on_send_transaction)}, + {api::cnd::CheckSendproof::method(), json_rpc::make_member_method(&Node::on_check_sendproof)}, + {api::cnd::SyncBlocks::method(), json_rpc::make_member_method(&Node::on_sync_blocks)}, + {api::cnd::GetRawBlock::method(), json_rpc::make_member_method(&Node::on_get_raw_block)}, + {api::cnd::GetBlockHeader::method(), json_rpc::make_member_method(&Node::on_get_block_header)}, + {api::cnd::GetRawTransaction::method(), json_rpc::make_member_method(&Node::on_get_raw_transaction)}, + {api::cnd::SyncMemPool::method(), json_rpc::make_member_method(&Node::on_sync_mempool)}}; + +bool Node::on_get_random_outputs(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetRandomOutputs::Request &&request, api::cnd::GetRandomOutputs::Response &response) { Height confirmed_height_or_depth = api::ErrorWrongHeight::fix_height_or_depth( request.confirmed_height_or_depth, m_block_chain.get_tip_height(), true, false); - api::BlockHeader tip_header = m_block_chain.get_tip(); + api::BlockHeader confirmed_header = m_block_chain.get_tip(); + Hash confirmed_hash; + invariant(m_block_chain.get_chain(confirmed_height_or_depth, &confirmed_hash), ""); + invariant(m_block_chain.get_header(confirmed_hash, &confirmed_header), ""); for (uint64_t amount : request.amounts) { - auto random_outputs = m_block_chain.get_random_outputs( - amount, request.output_count, confirmed_height_or_depth, tip_header.timestamp); + auto random_outputs = + m_block_chain.get_random_outputs(confirmed_header.major_version, amount, request.output_count, + confirmed_height_or_depth, confirmed_header.timestamp, confirmed_header.timestamp_median); auto &outs = response.outputs[amount]; outs.insert(outs.end(), random_outputs.begin(), random_outputs.end()); } return true; } -api::bytecoind::GetStatus::Response Node::create_status_response() const { - api::bytecoind::GetStatus::Response res; +api::cnd::GetStatus::Response Node::create_status_response() const { + api::cnd::GetStatus::Response res; res.top_block_height = m_block_chain.get_tip_height(); - res.top_known_block_height = m_downloader.get_known_block_count(res.top_block_height); + res.top_known_block_height = res.top_block_height; + for (auto &&gc : m_broadcast_protocols) + res.top_known_block_height = std::max(res.top_known_block_height, gc->get_peer_sync_data().current_height); res.top_known_block_height = std::max(res.top_known_block_height, m_block_chain.internal_import_known_height()); if (m_block_chain_reader1) @@ -316,12 +325,7 @@ api::bytecoind::GetStatus::Response Node::create_status_response() const { if (m_block_chain_reader2) res.top_known_block_height = std::max(res.top_known_block_height, m_block_chain_reader2->get_block_count()); - for (auto &&pb : broadcast_protocols) - if (pb->is_incoming()) - res.incoming_peer_count += 1; - else - res.outgoing_peer_count += 1; - for (auto &&pb : broadcast_protocols_new) + for (auto &&pb : m_broadcast_protocols) if (pb->is_incoming()) res.incoming_peer_count += 1; else @@ -333,60 +337,29 @@ api::bytecoind::GetStatus::Response Node::create_status_response() const { res.top_block_difficulty = tip.difficulty; res.top_block_cumulative_difficulty = tip.cumulative_difficulty; res.recommended_fee_per_byte = m_block_chain.get_currency().coin() / 1000000; // TODO - calculate - res.next_block_effective_median_size = m_block_chain.get_next_effective_median_size(); + res.recommended_max_transaction_size = m_block_chain.get_currency().get_recommended_max_transaction_size(); res.transaction_pool_version = m_block_chain.get_tx_pool_version(); return res; } void Node::broadcast(P2PProtocolBytecoin *exclude, const BinaryArray &data) { - for (auto &&p : broadcast_protocols) + for (auto &&p : m_broadcast_protocols) if (p != exclude) p->P2PProtocol::send(BinaryArray(data)); // Move is impossible here } -void Node::broadcast_new(P2PProtocolBytecoinNew *exclude, const BinaryArray &binary_header) { - np::RelayBlockHeader msg; - msg.binary_header = binary_header; - // TODO - do not forget to check - msg.top_block_desc.cd = m_block_chain.get_tip_cumulative_difficulty(); - msg.top_block_desc.height = m_block_chain.get_tip_height(); - msg.top_block_desc.hash = m_block_chain.get_tip_bid(); - BinaryArray body = seria::to_binary_kv(msg); - BinaryArray header = P2PProtocolBytecoinNew::create_header(np::RelayBlockHeader::ID, body.size()); - for (auto &&p : broadcast_protocols_new) - if (p != exclude) { - p->P2PProtocol::send(BinaryArray(header)); - p->P2PProtocol::send(BinaryArray(body)); - } -} -void Node::broadcast_new(P2PProtocolBytecoinNew *exclude, const std::vector &transaction_descs) { - np::RelayTransactionDescs msg; - msg.transaction_descs = transaction_descs; - // TODO - split into chunks - msg.top_block_desc.cd = m_block_chain.get_tip_cumulative_difficulty(); - msg.top_block_desc.height = m_block_chain.get_tip_height(); - msg.top_block_desc.hash = m_block_chain.get_tip_bid(); - BinaryArray body = seria::to_binary_kv(msg); - BinaryArray header = P2PProtocolBytecoinNew::create_header(np::RelayBlockHeader::ID, body.size()); - for (auto &&p : broadcast_protocols_new) - if (p != exclude) { - p->P2PProtocol::send(BinaryArray(header)); - p->P2PProtocol::send(BinaryArray(body)); - } +void Node::broadcast(P2PProtocolBytecoin *exclude, const BinaryArray &data_v1, const BinaryArray &data_v4) { + for (auto &&p : m_broadcast_protocols) + if (p != exclude) + p->P2PProtocol::send(BinaryArray( + p->get_peer_version() >= P2PProtocolVersion::AMETHYST ? data_v4 : data_v1)); // Move is impossible here } -// void Node::broadcast_new(P2PProtocolBytecoinNew * exclude, const BinaryArray &data){ -// for(auto && p : broadcast_protocols_new) -// if( p != exclude ) -// p->P2PProtocol::send(BinaryArray(data)); // Move is impossible here -//} - -bool Node::on_get_status(http::Client *who, http::RequestData &&raw_request, json_rpc::Request &&raw_js_request, - api::bytecoind::GetStatus::Request &&req, api::bytecoind::GetStatus::Response &res) { +bool Node::on_get_status(http::Client *who, http::RequestBody &&raw_request, json_rpc::Request &&raw_js_request, + api::cnd::GetStatus::Request &&req, api::cnd::GetStatus::Response &res) { res = create_status_response(); if (!res.ready_for_longpoll(req)) { // m_log(logging::INFO) << "on_get_status will long poll, json=" - //<< - // raw_request.body << std::endl; + // << raw_request.body << std::endl; LongPollClient lpc; lpc.original_who = who; lpc.original_request = raw_request; @@ -398,30 +371,27 @@ bool Node::on_get_status(http::Client *who, http::RequestData &&raw_request, jso return true; } -api::bytecoind::GetStatistics::Response Node::create_statistics_response() const { - api::bytecoind::GetStatistics::Response res; +api::cnd::GetStatistics::Response Node::create_statistics_response(const api::cnd::GetStatistics::Request &req) const { + api::cnd::GetStatistics::Response res; res.peer_id = m_p2p.get_unique_number(); - for (auto &&p : broadcast_protocols_new) { - np::ConnectionDesc desc; - desc.address = p->get_address(); - desc.is_incoming = p->is_incoming(); - desc.p2p_version = p->get_other_peer_desc().p2p_version; - desc.peer_id = p->get_other_peer_desc().peer_id; - desc.top_block_desc = p->get_other_top_block_desc(); - res.connections.push_back(desc); + if (req.need_connected_peers) { + for (auto &&p : m_broadcast_protocols) { + ConnectionDesc desc; + desc.address = p->get_address(); + desc.is_incoming = p->is_incoming(); + desc.p2p_version = p->get_peer_version(); + desc.peer_id = p->get_peer_unique_number(); + desc.top_block_desc.hash = p->get_peer_sync_data().top_id; + desc.top_block_desc.height = p->get_peer_sync_data().current_height; + res.connected_peers.push_back(desc); + } } - for (auto &&p : broadcast_protocols) { - np::ConnectionDesc desc; - desc.address = p->get_address(); - desc.is_incoming = p->is_incoming(); - desc.p2p_version = p->get_version(); - desc.peer_id = p->get_last_received_unique_number(); - desc.top_block_desc.hash = p->get_last_received_sync_data().top_id; - desc.top_block_desc.height = p->get_last_received_sync_data().current_height; - res.connections.push_back(desc); + if (req.need_peer_lists) { + res.peer_list_gray = m_peer_db.get_peer_list_gray(); + res.peer_list_gray = m_peer_db.get_peer_list_white(); } res.platform = platform::get_platform_name(); - res.version = bytecoin::app_version(); + res.version = cn::app_version(); res.net = m_config.net; res.genesis_block_hash = m_block_chain.get_currency().genesis_block_hash; res.start_time = m_start_time; @@ -429,18 +399,18 @@ api::bytecoind::GetStatistics::Response Node::create_statistics_response() const return res; } -bool Node::on_get_statistics(http::Client *, http::RequestData &&http_request, json_rpc::Request &&, - api::bytecoind::GetStatistics::Request &&, api::bytecoind::GetStatistics::Response &res) { +bool Node::on_get_statistics(http::Client *, http::RequestBody &&http_request, json_rpc::Request &&, + api::cnd::GetStatistics::Request &&req, api::cnd::GetStatistics::Response &res) { bool good_auth_private = m_config.bytecoind_authorization_private.empty() || http_request.r.basic_authorization == m_config.bytecoind_authorization_private; if (!good_auth_private) throw http::ErrorAuthorization("Statistics"); - res = create_statistics_response(); + res = create_statistics_response(req); return true; } -bool Node::on_get_archive(http::Client *, http::RequestData &&http_request, json_rpc::Request &&, - api::bytecoind::GetArchive::Request &&req, api::bytecoind::GetArchive::Response &resp) { +bool Node::on_get_archive(http::Client *, http::RequestBody &&http_request, json_rpc::Request &&, + api::cnd::GetArchive::Request &&req, api::cnd::GetArchive::Response &resp) { bool good_auth_private = m_config.bytecoind_authorization_private.empty() || http_request.r.basic_authorization == m_config.bytecoind_authorization_private; if (!good_auth_private) @@ -452,84 +422,78 @@ bool Node::on_get_archive(http::Client *, http::RequestData &&http_request, json static void fill_transaction_info(const TransactionPrefix &tx, api::Transaction *api_tx) { api_tx->unlock_block_or_timestamp = tx.unlock_block_or_timestamp; api_tx->extra = tx.extra; - api_tx->anonymity = std::numeric_limits::max(); + api_tx->anonymity = std::numeric_limits::max(); api_tx->public_key = extra_get_transaction_public_key(tx.extra); extra_get_payment_id(tx.extra, api_tx->payment_id); Amount input_amount = 0; for (const auto &input : tx.inputs) { - if (input.type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(input); - api_tx->anonymity = std::min(api_tx->anonymity, static_cast(in.output_indexes.size() - 1)); + if (input.type() == typeid(InputKey)) { + const InputKey &in = boost::get(input); + api_tx->anonymity = std::min(api_tx->anonymity, in.output_indexes.size() - 1); input_amount += in.amount; } } - Amount output_amount = 0; - for (const auto &output : tx.outputs) { - if (output.target.type() == typeid(KeyOutput)) { - // const KeyOutput &key_output = boost::get(output.target); - output_amount += output.amount; - } - } - api_tx->amount = output_amount; + Amount output_amount = get_tx_sum_outputs(tx); + api_tx->amount = output_amount; if (input_amount >= output_amount) api_tx->fee = input_amount - output_amount; - if (api_tx->anonymity == std::numeric_limits::max()) + if (api_tx->anonymity == std::numeric_limits::max()) api_tx->anonymity = 0; // No key inputs } -bool Node::on_sync_blocks(http::Client *, http::RequestData &&, json_rpc::Request &&json_req, - api::bytecoind::SyncBlocks::Request &&req, api::bytecoind::SyncBlocks::Response &res) { +bool Node::on_sync_blocks(http::Client *, http::RequestBody &&, json_rpc::Request &&json_req, + api::cnd::SyncBlocks::Request &&req, api::cnd::SyncBlocks::Response &res) { if (req.sparse_chain.empty()) throw std::runtime_error("Empty sparse chain - must include at least genesis block"); + if (req.sparse_chain.back() == Hash{}) // We allow to ask for "whatever genesis bid. Useful for explorer, etc." + req.sparse_chain.back() = m_block_chain.get_genesis_bid(); if (req.sparse_chain.back() != m_block_chain.get_genesis_bid()) throw std::runtime_error( "Wrong currency - different genesis block. Must be " + common::pod_to_hex(m_block_chain.get_genesis_bid())); - if (req.max_count > api::bytecoind::SyncBlocks::Request::MAX_COUNT) - throw std::runtime_error( - "Too big max_count - must be <= " + common::to_string(api::bytecoind::SyncBlocks::Request::MAX_COUNT)); + if (req.max_count > m_config.rpc_sync_blocks_max_count) + req.max_count = m_config.rpc_sync_blocks_max_count; auto first_block_timestamp = req.first_block_timestamp < m_block_chain.get_currency().block_future_time_limit ? 0 : req.first_block_timestamp - m_block_chain.get_currency().block_future_time_limit; Height full_offset = m_block_chain.get_timestamp_lower_bound_height(first_block_timestamp); Height start_height; - std::vector supplement = m_block_chain.get_sync_headers_chain(req.sparse_chain, &start_height, req.max_count); - if (full_offset >= start_height + supplement.size()) { + std::vector subchain = m_block_chain.get_sync_headers_chain(req.sparse_chain, &start_height, req.max_count); + if (full_offset >= start_height + subchain.size()) { start_height = full_offset; - supplement.clear(); - while (supplement.size() < req.max_count) { + subchain.clear(); + while (subchain.size() < req.max_count) { Hash ha; - if (!m_block_chain.read_chain(start_height + static_cast(supplement.size()), &ha)) + if (!m_block_chain.get_chain(start_height + static_cast(subchain.size()), &ha)) break; - supplement.push_back(ha); + subchain.push_back(ha); } } else if (full_offset > start_height) { - supplement.erase(supplement.begin(), supplement.begin() + (full_offset - start_height)); + subchain.erase(subchain.begin(), subchain.begin() + (full_offset - start_height)); start_height = full_offset; } res.start_height = start_height; - res.blocks.resize(supplement.size()); - for (size_t i = 0; i != supplement.size(); ++i) { - const auto bhash = supplement[i]; - auto &res_block = res.blocks[i]; + res.blocks.resize(subchain.size()); + size_t total_size = 0; + for (size_t i = 0; i != subchain.size(); ++i) { + const auto &bhash = subchain[i]; + auto &res_block = res.blocks[i]; invariant( - m_block_chain.read_header(bhash, &res_block.header), "Block header must be there, but it is not there"); - m_block_chain.fix_block_sizes(&res_block.header); + m_block_chain.get_header(bhash, &res_block.header), "Block header must be there, but it is not there"); + // BlockChainState::BlockGlobalIndices output_indexes; // if (res.blocks[i].header.timestamp >= req.first_block_timestamp) // // commented out becuase empty Block cannot be serialized { RawBlock rb; - invariant(m_block_chain.read_block(bhash, &rb), "Block must be there, but it is not there"); - Block block; - invariant(block.from_raw_block(rb), "RawBlock failed to convert into block"); + invariant(m_block_chain.get_block(bhash, &rb), "Block must be there, but it is not there"); + Block block(rb); res_block.transactions.resize(block.transactions.size() + 1); res_block.transactions.at(0).hash = get_transaction_hash(block.header.base_transaction); - res_block.transactions.at(0).size = - static_cast(seria::binary_size(block.header.base_transaction)); + res_block.transactions.at(0).size = seria::binary_size(block.header.base_transaction); if (req.need_redundant_data) { fill_transaction_info(block.header.base_transaction, &res_block.transactions.at(0)); - res_block.transactions.at(0).block_height = start_height + static_cast(i); + res_block.transactions.at(0).block_height = start_height + static_cast(i); res_block.transactions.at(0).block_hash = bhash; res_block.transactions.at(0).coinbase = true; res_block.transactions.at(0).timestamp = block.header.timestamp; @@ -538,11 +502,10 @@ bool Node::on_sync_blocks(http::Client *, http::RequestData &&, json_rpc::Reques res_block.raw_transactions.reserve(block.transactions.size()); for (size_t tx_index = 0; tx_index != block.transactions.size(); ++tx_index) { res_block.transactions.at(tx_index + 1).hash = res_block.raw_header.transaction_hashes.at(tx_index); - res_block.transactions.at(tx_index + 1).size = - static_cast(rb.transactions.at(tx_index).size()); + res_block.transactions.at(tx_index + 1).size = rb.transactions.at(tx_index).size(); if (req.need_redundant_data) { fill_transaction_info(block.transactions.at(tx_index), &res_block.transactions.at(tx_index + 1)); - res_block.transactions.at(tx_index + 1).block_height = start_height + static_cast(i); + res_block.transactions.at(tx_index + 1).block_height = start_height + static_cast(i); res_block.transactions.at(tx_index + 1).block_hash = bhash; res_block.transactions.at(tx_index + 1).timestamp = res_block.raw_header.timestamp; } @@ -553,13 +516,18 @@ bool Node::on_sync_blocks(http::Client *, http::RequestData &&, json_rpc::Reques invariant(m_block_chain.read_block_output_global_indices(bhash, &res_block.output_indexes), "Invariant dead - bid is in chain but blockchain has no block indices"); } + total_size += res_block.header.transactions_size; + if (total_size >= req.max_size) { + res.blocks.resize(i + 1); + break; + } } res.status = create_status_response(); return true; } -bool Node::on_sync_mempool(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::SyncMemPool::Request &&req, api::bytecoind::SyncMemPool::Response &res) { +bool Node::on_sync_mempool(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::SyncMemPool::Request &&req, api::cnd::SyncMemPool::Response &res) { const auto &pool = m_block_chain.get_memory_state_transactions(); for (auto &&ex : req.known_hashes) if (pool.count(ex) == 0) @@ -574,61 +542,57 @@ bool Node::on_sync_mempool(http::Client *, http::RequestData &&, json_rpc::Reque fill_transaction_info(tx.second.tx, &res.added_transactions.back()); res.added_transactions.back().hash = tx.first; res.added_transactions.back().timestamp = tx.second.timestamp; + res.added_transactions.back().amount = tx.second.amount; res.added_transactions.back().fee = tx.second.fee; - res.added_transactions.back().size = static_cast(tx.second.binary_tx.size()); + res.added_transactions.back().size = tx.second.binary_tx.size(); } res.status = create_status_response(); return true; } -bool Node::on_get_block_header(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetBlockHeader::Request &&request, api::bytecoind::GetBlockHeader::Response &response) { +bool Node::on_get_block_header(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetBlockHeader::Request &&request, api::cnd::GetBlockHeader::Response &response) { if (request.hash != Hash{} && request.height_or_depth != std::numeric_limits::max()) throw json_rpc::Error( json_rpc::INVALID_REQUEST, "You cannot specify both hash and height_or_depth to this method"); if (request.hash != Hash{}) { - if (!m_block_chain.read_header(request.hash, &response.block_header)) + if (!m_block_chain.get_header(request.hash, &response.block_header)) throw api::ErrorHashNotFound("Block not found in either main or side chains", request.hash); } else { Height height_or_depth = api::ErrorWrongHeight::fix_height_or_depth( request.height_or_depth, m_block_chain.get_tip_height(), true, true); invariant( - m_block_chain.read_chain(height_or_depth, &request.hash), ""); // after fix_height it must always succeed - invariant(m_block_chain.read_header(request.hash, &response.block_header), ""); + m_block_chain.get_chain(height_or_depth, &request.hash), ""); // after fix_height it must always succeed + invariant(m_block_chain.get_header(request.hash, &response.block_header), ""); } - m_block_chain.fix_block_sizes(&response.block_header); response.orphan_status = !m_block_chain.in_chain(response.block_header.height, response.block_header.hash); response.depth = api::HeightOrDepth(response.block_header.height) - api::HeightOrDepth(m_block_chain.get_tip_height()) - 1; return true; } -bool Node::on_get_raw_block(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetRawBlock::Request &&request, api::bytecoind::GetRawBlock::Response &response) { +bool Node::on_get_raw_block(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetRawBlock::Request &&request, api::cnd::GetRawBlock::Response &response) { if (request.hash != Hash{} && request.height_or_depth != std::numeric_limits::max()) throw json_rpc::Error( json_rpc::INVALID_REQUEST, "You cannot specify both hash and height_or_depth to this method"); if (request.hash != Hash{}) { - if (!m_block_chain.read_header(request.hash, &response.block.header)) + if (!m_block_chain.get_header(request.hash, &response.block.header)) throw api::ErrorHashNotFound("Block not found in either main or side chains", request.hash); } else { Height height_or_depth = api::ErrorWrongHeight::fix_height_or_depth( request.height_or_depth, m_block_chain.get_tip_height(), true, true); invariant( - m_block_chain.read_chain(height_or_depth, &request.hash), ""); // after fix_height it must always succeed - invariant(m_block_chain.read_header(request.hash, &response.block.header), ""); + m_block_chain.get_chain(height_or_depth, &request.hash), ""); // after fix_height it must always succeed + invariant(m_block_chain.get_header(request.hash, &response.block.header), ""); } - m_block_chain.fix_block_sizes(&response.block.header); - - // BlockChainState::BlockGlobalIndices output_indexes; RawBlock rb; - invariant(m_block_chain.read_block(request.hash, &rb), "Block must be there, but it is not there"); - Block block; - invariant(block.from_raw_block(rb), "RawBlock failed to convert into block"); + invariant(m_block_chain.get_block(request.hash, &rb), "Block must be there, but it is not there"); + Block block(rb); api::RawBlock &b = response.block; b.transactions.resize(block.transactions.size() + 1); b.transactions.at(0).hash = get_transaction_hash(block.header.base_transaction); - b.transactions.at(0).size = static_cast(seria::binary_size(block.header.base_transaction)); + b.transactions.at(0).size = seria::binary_size(block.header.base_transaction); fill_transaction_info(block.header.base_transaction, &b.transactions.at(0)); b.transactions.at(0).block_height = b.header.height; b.transactions.at(0).block_hash = b.header.hash; @@ -638,7 +602,7 @@ bool Node::on_get_raw_block(http::Client *, http::RequestData &&, json_rpc::Requ b.raw_transactions.reserve(block.transactions.size()); for (size_t tx_index = 0; tx_index != block.transactions.size(); ++tx_index) { b.transactions.at(tx_index + 1).hash = b.raw_header.transaction_hashes.at(tx_index); - b.transactions.at(tx_index + 1).size = static_cast(rb.transactions.at(tx_index).size()); + b.transactions.at(tx_index + 1).size = rb.transactions.at(tx_index).size(); fill_transaction_info(block.transactions.at(tx_index), &b.transactions.at(tx_index + 1)); b.transactions.at(tx_index + 1).block_height = b.header.height; b.transactions.at(tx_index + 1).block_hash = b.header.hash; @@ -654,12 +618,12 @@ bool Node::on_get_raw_block(http::Client *, http::RequestData &&, json_rpc::Requ return true; } -bool Node::on_get_raw_transaction(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetRawTransaction::Request &&req, api::bytecoind::GetRawTransaction::Response &res) { +bool Node::on_get_raw_transaction(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetRawTransaction::Request &&req, api::cnd::GetRawTransaction::Response &res) { const auto &pool = m_block_chain.get_memory_state_transactions(); auto tit = pool.find(req.hash); if (tit != pool.end()) { - res.raw_transaction = static_cast(tit->second.tx); + res.raw_transaction = static_cast(tit->second.tx); if (req.need_signatures) res.signatures = tit->second.tx.signatures; fill_transaction_info(tit->second.tx, &res.transaction); @@ -667,17 +631,17 @@ bool Node::on_get_raw_transaction(http::Client *, http::RequestData &&, json_rpc res.transaction.hash = req.hash; res.transaction.block_height = m_block_chain.get_tip_height() + 1; res.transaction.timestamp = tit->second.timestamp; - res.transaction.size = static_cast(tit->second.binary_tx.size()); + res.transaction.size = tit->second.binary_tx.size(); return true; } BinaryArray binary_tx; Transaction tx; size_t index_in_block = 0; - if (m_block_chain.read_transaction( + if (m_block_chain.get_transaction( req.hash, &binary_tx, &res.transaction.block_height, &res.transaction.block_hash, &index_in_block)) { - res.transaction.size = static_cast(binary_tx.size()); + res.transaction.size = binary_tx.size(); seria::from_binary(tx, binary_tx); - res.raw_transaction = static_cast(tx); // TODO - std::move? + res.raw_transaction = static_cast(tx); if (req.need_signatures) res.signatures = tx.signatures; fill_transaction_info(tx, &res.transaction); @@ -690,128 +654,205 @@ bool Node::on_get_raw_transaction(http::Client *, http::RequestData &&, json_rpc req.hash); } -bool Node::handle_send_transaction(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::SendTransaction::Request &&request, api::bytecoind::SendTransaction::Response &response) { +bool Node::on_send_transaction(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::SendTransaction::Request &&request, api::cnd::SendTransaction::Response &response) { response.send_result = "broadcast"; - NOTIFY_NEW_TRANSACTIONS::request msg; - Height conflict_height = - m_block_chain.get_currency().max_block_height; // So will not be accidentally viewed as confirmed + p2p::RelayTransactions::Notify msg; + p2p::RelayTransactions::Notify msg_v4; + // Height conflict_height = + // m_block_chain.get_currency().max_block_height; // So will not be accidentally viewed as confirmed Transaction tx; try { seria::from_binary(tx, request.binary_transaction); + const Hash tid = get_transaction_hash(tx); + if (m_block_chain.add_transaction(tid, tx, request.binary_transaction, m_p2p.get_local_time(), "json_rpc")) { + msg.txs.push_back(request.binary_transaction); + TransactionDesc desc; + desc.hash = tid; + desc.size = request.binary_transaction.size(); + desc.fee = get_tx_fee(tx); + Height newest_referenced_height = 0; + invariant(m_block_chain.get_largest_referenced_height(tx, &newest_referenced_height), ""); + invariant(m_block_chain.get_chain(newest_referenced_height, &desc.newest_referenced_block), ""); + msg_v4.transaction_descs.push_back(desc); + + BinaryArray raw_msg = LevinProtocol::send(msg); + BinaryArray raw_msg_v4 = LevinProtocol::send(msg_v4); + broadcast(nullptr, raw_msg, raw_msg_v4); + advance_long_poll(); + } + } catch (const ConsensusErrorOutputDoesNotExist &ex) { + throw api::cnd::SendTransaction::Error(api::cnd::SendTransaction::WRONG_OUTPUT_REFERENCE, common::what(ex), + m_block_chain.get_currency().max_block_height); + } catch (const ConsensusErrorBadOutputOrSignature &ex) { + throw api::cnd::SendTransaction::Error( + api::cnd::SendTransaction::WRONG_OUTPUT_REFERENCE, common::what(ex), ex.conflict_height); + } catch (const ConsensusErrorOutputSpent &ex) { + throw api::cnd::SendTransaction::Error( + api::cnd::SendTransaction::OUTPUT_ALREADY_SPENT, common::what(ex), ex.conflict_height); } catch (const std::exception &ex) { - std::throw_with_nested(api::bytecoind::SendTransaction::Error( - api::bytecoind::SendTransaction::INVALID_TRANSACTION_BINARY_FORMAT, common::what(ex), conflict_height)); - } - const Hash tid = get_transaction_hash(tx); - auto action = m_block_chain.add_transaction( - tid, tx, request.binary_transaction, m_p2p.get_local_time(), &conflict_height, "json_rpc"); - switch (action) { - case AddTransactionResult::BAN: - throw json_rpc::Error( - api::bytecoind::SendTransaction::INVALID_TRANSACTION_BINARY_FORMAT, "Binary transaction format is wrong"); - case AddTransactionResult::BROADCAST_ALL: { - msg.txs.push_back(request.binary_transaction); - BinaryArray raw_msg = - LevinProtocol::send_message(NOTIFY_NEW_TRANSACTIONS::ID, LevinProtocol::encode(msg), false); - broadcast(nullptr, raw_msg); - // broadcast_new(nullptr, ); // TODO - broadcast transaction - advance_long_poll(); - break; - } - case AddTransactionResult::ALREADY_IN_POOL: - break; - case AddTransactionResult::INCREASE_FEE: - break; - case AddTransactionResult::FAILED_TO_REDO: - throw api::bytecoind::SendTransaction::Error(api::bytecoind::SendTransaction::WRONG_OUTPUT_REFERENCE, - "Transaction references outputs changed during reorganization or signature wrong", conflict_height); - case AddTransactionResult::OUTPUT_ALREADY_SPENT: - throw api::bytecoind::SendTransaction::Error(api::bytecoind::SendTransaction::OUTPUT_ALREADY_SPENT, - "One of referenced outputs is already spent", conflict_height); + std::throw_with_nested(api::cnd::SendTransaction::Error( + api::cnd::SendTransaction::INVALID_TRANSACTION_BINARY_FORMAT, common::what(ex), 0)); } return true; } -bool Node::handle_check_sendproof(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::CheckSendproof::Request &&request, api::bytecoind::CheckSendproof::Response &response) { - SendProof sp; +bool Node::on_check_sendproof(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::CheckSendproof::Request &&request, api::cnd::CheckSendproof::Response &response) { + Sendproof sp; try { seria::from_json_value(sp, common::JsonValue::from_string(request.sendproof), m_block_chain.get_currency()); - // seria::JsonInputStreamValue s(); - // s.begin_object(); - // ser_members(sp, s, ); - // s.end_object(); } catch (const std::exception &ex) { - std::throw_with_nested(api::bytecoind::CheckSendproof::Error(api::bytecoind::CheckSendproof::FAILED_TO_PARSE, - "Failed to parse proof object ex.what=" + common::what(ex))); + std::throw_with_nested(api::cnd::CheckSendproof::Error( + api::cnd::CheckSendproof::FAILED_TO_PARSE, "Failed to parse proof object ex.what=" + common::what(ex))); } + response.transaction_hash = sp.transaction_hash; + response.address = m_block_chain.get_currency().account_address_as_string(sp.address); + response.message = sp.message; + response.amount = sp.amount; + BinaryArray binary_tx; Height height = 0; Hash block_hash; size_t index_in_block = 0; - if (!m_block_chain.read_transaction(sp.transaction_hash, &binary_tx, &height, &block_hash, &index_in_block)) { - throw api::bytecoind::CheckSendproof::Error( - api::bytecoind::CheckSendproof::NOT_IN_MAIN_CHAIN, "Transaction is not in main chain"); + if (!m_block_chain.get_transaction(sp.transaction_hash, &binary_tx, &height, &block_hash, &index_in_block)) { + throw api::cnd::CheckSendproof::Error( + api::cnd::CheckSendproof::NOT_IN_MAIN_CHAIN, "Transaction is not in main chain"); } Transaction tx; seria::from_binary(tx, binary_tx); - PublicKey tx_public_key = extra_get_transaction_public_key(tx.extra); - Hash message_hash = crypto::cn_fast_hash(sp.message.data(), sp.message.size()); - if (!crypto::check_sendproof( - tx_public_key, sp.address.view_public_key, sp.derivation, message_hash, sp.signature)) { - throw api::bytecoind::CheckSendproof::Error(api::bytecoind::CheckSendproof::WRONG_SIGNATURE, - "Proof object does not match transaction or was tampered with"); + Hash message_hash = crypto::cn_fast_hash(sp.message.data(), sp.message.size()); + if (sp.address.type() == typeid(AccountAddressSimple)) { + auto &addr = boost::get(sp.address); + auto &var = boost::get(sp.proof); + PublicKey tx_public_key = extra_get_transaction_public_key(tx.extra); + if (!crypto::check_sendproof( + tx_public_key, addr.view_public_key, var.derivation, message_hash, var.signature)) { + throw api::cnd::CheckSendproof::Error(api::cnd::CheckSendproof::WRONG_SIGNATURE, + "Proof object does not match transaction or was tampered with"); + } + Amount total_amount = 0; + size_t out_index = 0; + for (const auto &output : tx.outputs) { + if (output.type() == typeid(OutputKey)) { + const auto &key_output = boost::get(output); + const PublicKey spend_key = underive_public_key(var.derivation, out_index, key_output.public_key); + if (spend_key == addr.spend_public_key) { + total_amount += key_output.amount; + response.output_indexes.push_back(out_index); + } + } + ++out_index; + } + if (total_amount == 0) + throw api::cnd::CheckSendproof::Error(api::cnd::CheckSendproof::ADDRESS_NOT_IN_TRANSACTION, + "No outputs found in transaction for the address being proofed"); + if (total_amount != sp.amount) + throw api::cnd::CheckSendproof::Error(api::cnd::CheckSendproof::WRONG_AMOUNT, + "Wrong amount in outputs, actual amount is " + common::to_string(total_amount)); + return true; } - Amount total_amount = 0; - size_t key_index = 0; - uint32_t out_index = 0; - for (const auto &output : tx.outputs) { - if (output.target.type() == typeid(KeyOutput)) { - const KeyOutput &key_output = boost::get(output.target); - PublicKey spend_key; - if (underive_public_key(sp.derivation, key_index, key_output.public_key, spend_key) && - spend_key == sp.address.spend_public_key) { - total_amount += output.amount; + if (sp.address.type() == typeid(AccountAddressUnlinkable)) { + auto &addr = boost::get(sp.address); + auto &var = boost::get(sp.proof); + Hash tx_inputs_hash = get_transaction_inputs_hash(tx); + for (size_t oi = 1; oi < var.elements.size(); ++oi) { + if (var.elements.at(oi).out_index <= var.elements.at(oi - 1).out_index) + throw api::cnd::CheckSendproof::Error( + api::cnd::CheckSendproof::WRONG_SIGNATURE, "Proof object elements are not in ascending order"); + } + Amount total_amount = 0; + size_t out_index = 0; + for (const auto &output : tx.outputs) { + if (output.type() == typeid(OutputKey)) { + const auto &key_output = boost::get(output); + for (size_t oi = 0; oi != var.elements.size(); ++oi) { + const auto &el = var.elements.at(oi); + if (el.out_index == out_index) { + if (addr.is_auditable != key_output.is_auditable) + throw api::cnd::CheckSendproof::Error(api::cnd::CheckSendproof::WRONG_SIGNATURE, + "Auditability of coin does not match auditability of address"); + Hash output_secret = crypto::cn_fast_hash(el.q.data, sizeof(el.q.data)); + if (!crypto::check_signature(output_secret, el.q, el.signature)) + throw api::cnd::CheckSendproof::Error( + api::cnd::CheckSendproof::WRONG_SIGNATURE, "Proof object element signature wrong"); + AccountAddressUnlinkable address; + if (!crypto::unlinkable_underive_address(output_secret, tx_inputs_hash, out_index, + key_output.public_key, key_output.encrypted_secret, &address.s, &address.sv) || + address != addr) { + throw api::cnd::CheckSendproof::Error(api::cnd::CheckSendproof::WRONG_SIGNATURE, + "Proof unlinkable address derivation failed"); + } + total_amount += key_output.amount; + response.output_indexes.push_back(out_index); + var.elements.erase(var.elements.begin() + oi); + break; + } + } } - ++key_index; + ++out_index; } - ++out_index; + if (!var.elements.empty()) + throw api::cnd::CheckSendproof::Error( + api::cnd::CheckSendproof::WRONG_SIGNATURE, "Proof object contains excess elements"); + if (total_amount == 0) + throw api::cnd::CheckSendproof::Error(api::cnd::CheckSendproof::ADDRESS_NOT_IN_TRANSACTION, + "No outputs found in transaction for the address being proofed"); + if (total_amount != sp.amount) + throw api::cnd::CheckSendproof::Error(api::cnd::CheckSendproof::WRONG_AMOUNT, + "Wrong amount in outputs, actual amount is " + common::to_string(total_amount)); + return true; } - if (total_amount == 0) - throw api::bytecoind::CheckSendproof::Error(api::bytecoind::CheckSendproof::ADDRESS_NOT_IN_TRANSACTION, - "No outputs found in transaction for the address being proofed"); - if (total_amount != sp.amount) - throw api::bytecoind::CheckSendproof::Error(api::bytecoind::CheckSendproof::WRONG_AMOUNT, - "Wrong amount in outputs, actual amount is " + common::to_string(total_amount)); - response.transaction_hash = sp.transaction_hash; - response.address = m_block_chain.get_currency().account_address_as_string(sp.address); - response.message = sp.message; - response.amount = sp.amount; - return true; + return false; } void Node::submit_block(const BinaryArray &blockblob, api::BlockHeader *info) { BlockTemplate block_template; seria::from_binary(block_template, blockblob); RawBlock raw_block; - // api::BlockHeader info; - auto broad = m_block_chain.add_mined_block(blockblob, &raw_block, info); - if (broad == BroadcastAction::BAN) - throw json_rpc::Error{api::bytecoind::SubmitBlock::BLOCK_NOT_ACCEPTED, "Block not accepted"}; - NOTIFY_NEW_BLOCK::request msg; - msg.b = RawBlockLegacy{raw_block.block, raw_block.transactions}; + try { + if (!m_block_chain.add_mined_block(blockblob, &raw_block, info)) + return; + } catch (const std::exception &ex) { + throw json_rpc::Error{ + api::cnd::SubmitBlock::BLOCK_NOT_ACCEPTED, "Block not accepted, reason=" + common::what(ex)}; + } + for (auto who : m_broadcast_protocols) + who->advance_transactions(); + p2p::RelayBlock::Notify msg; + msg.b = std::move(raw_block); // RawBlockLegacy{raw_block.block, raw_block.transactions}; msg.hop = 1; - msg.current_blockchain_height = m_block_chain.get_tip_height() + 1; // TODO check - BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_NEW_BLOCK::ID, LevinProtocol::encode(msg), false); - broadcast(nullptr, raw_msg); - broadcast_new(nullptr, blockblob); + msg.current_blockchain_height = m_block_chain.get_tip_height(); + msg.top_id = m_block_chain.get_tip_bid(); + p2p::RelayBlock::Notify msg_v4; + msg_v4.b.block = msg.b.block; + msg_v4.current_blockchain_height = msg.current_blockchain_height; + msg_v4.top_id = msg.top_id; + msg_v4.hop = msg.hop; + + msg.top_id = Hash{}; // TODO - uncomment after 3.4 fork. This is workaround of bug in 3.2 + + BinaryArray raw_msg = LevinProtocol::send(msg); + BinaryArray raw_msg_v4 = LevinProtocol::send(msg_v4); + broadcast(nullptr, raw_msg, raw_msg_v4); advance_long_poll(); } -bool Node::on_submitblock(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::SubmitBlock::Request &&req, api::bytecoind::SubmitBlock::Response &res) { +bool Node::on_submitblock(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::SubmitBlock::Request &&req, api::cnd::SubmitBlock::Response &res) { + if (!req.cm_nonce.empty()) { + // Experimental, a bit hacky + BlockTemplate bt; + seria::from_binary(bt, req.blocktemplate_blob); + bt.major_version += 1; + bt.nonce = req.cm_nonce; + bt.cm_merkle_branch = req.cm_merkle_branch; + req.blocktemplate_blob = seria::to_binary(bt); + // auto body_proxy = get_body_proxy_from_template(bt); + // auto cm_prehash = get_auxiliary_block_header_hash(bt, body_proxy); + // std::cout << "submit CM data " << body_proxy.transactions_merkle_root << " " << cm_prehash << std::endl; + } submit_block(req.blocktemplate_blob, &res.block_header); res.orphan_status = !m_block_chain.in_chain(res.block_header.height, res.block_header.hash); res.depth = api::HeightOrDepth(res.block_header.height) - api::HeightOrDepth(m_block_chain.get_tip_height()) - 1; diff --git a/src/Core/Node.hpp b/src/Core/Node.hpp index dcf2dcfc..449361f8 100644 --- a/src/Core/Node.hpp +++ b/src/Core/Node.hpp @@ -8,85 +8,81 @@ #include #include #include -#include "BlockChainFileFormat.hpp" #include "BlockChainState.hpp" #include "http/BinaryRpc.hpp" #include "http/JsonRpc.hpp" -#include "http/Server.hpp" #include "p2p/P2P.hpp" -#include "p2p/P2PClientBasic.hpp" -#include "p2p/P2PClientNew.hpp" -#include "platform/PreventSleep.hpp" +#include "p2p/P2PProtocolBasic.hpp" #include "rpc_api.hpp" -namespace bytecoin { - -// a bit different commit periods to make most commits not simultaneous -static const float SYNC_TIMEOUT = 20; // If sync does not return, select different sync node after -static const int DOWNLOAD_CONCURRENCY = 4; -static const int DOWNLOAD_QUEUE = 10; // number of block requests sent before receiving reply -static const int DOWNLOAD_BLOCK_WINDOW = DOWNLOAD_CONCURRENCY * DOWNLOAD_QUEUE * 2; -static const float RETRY_DOWNLOAD_SECONDS = 10; +namespace http { +class Server; +class Client; +} // namespace http +namespace platform { +class PreventSleep; +} +namespace cn { +class LegacyBlockChainReader; class Node { public: - typedef std::function + typedef std::function JSONRPCHandlerFunction; typedef std::function BINARYRPCHandlerFunction; explicit Node(logging::ILogger &, const Config &, BlockChainState &); + ~Node(); bool on_idle(); // binary method - bool on_sync_blocks(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::SyncBlocks::Request &&, api::bytecoind::SyncBlocks::Response &); - bool on_sync_mempool(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::SyncMemPool::Request &&, api::bytecoind::SyncMemPool::Response &); - - bool on_get_raw_transaction(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetRawTransaction::Request &&, api::bytecoind::GetRawTransaction::Response &); - bool on_get_raw_block(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetRawBlock::Request &&, api::bytecoind::GetRawBlock::Response &); - bool on_get_block_header(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetBlockHeader::Request &&, api::bytecoind::GetBlockHeader::Response &); - - api::bytecoind::GetStatus::Response create_status_response() const; - api::bytecoind::GetStatistics::Response create_statistics_response() const; + bool on_sync_blocks(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::cnd::SyncBlocks::Request &&, + api::cnd::SyncBlocks::Response &); + bool on_sync_mempool(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::cnd::SyncMemPool::Request &&, + api::cnd::SyncMemPool::Response &); + + bool on_get_raw_transaction(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetRawTransaction::Request &&, api::cnd::GetRawTransaction::Response &); + bool on_get_raw_block(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::cnd::GetRawBlock::Request &&, + api::cnd::GetRawBlock::Response &); + bool on_get_block_header(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetBlockHeader::Request &&, api::cnd::GetBlockHeader::Response &); + + api::cnd::GetStatus::Response create_status_response() const; + api::cnd::GetStatistics::Response create_statistics_response(const api::cnd::GetStatistics::Request &) const; // json_rpc_node - bool on_get_status(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetStatus::Request &&, api::bytecoind::GetStatus::Response &); - bool on_get_statistics(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetStatistics::Request &&, api::bytecoind::GetStatistics::Response &); - bool on_get_archive(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetArchive::Request &&, api::bytecoind::GetArchive::Response &); - bool on_get_random_outputs(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetRandomOutputs::Request &&, api::bytecoind::GetRandomOutputs::Response &); - bool handle_send_transaction(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::SendTransaction::Request &&, api::bytecoind::SendTransaction::Response &); - bool handle_check_sendproof(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::CheckSendproof::Request &&, api::bytecoind::CheckSendproof::Response &); - bool on_getblocktemplate(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetBlockTemplate::Request &&r, api::bytecoind::GetBlockTemplate::Response &); - void getblocktemplate( - const api::bytecoind::GetBlockTemplate::Request &, api::bytecoind::GetBlockTemplate::Response &); - bool on_get_currency_id(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetCurrencyId::Request &&, api::bytecoind::GetCurrencyId::Response &); + bool on_get_status(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::cnd::GetStatus::Request &&, + api::cnd::GetStatus::Response &); + bool on_get_statistics(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetStatistics::Request &&, api::cnd::GetStatistics::Response &); + bool on_get_archive(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::cnd::GetArchive::Request &&, + api::cnd::GetArchive::Response &); + bool on_get_random_outputs(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetRandomOutputs::Request &&, api::cnd::GetRandomOutputs::Response &); + bool on_send_transaction(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::SendTransaction::Request &&, api::cnd::SendTransaction::Response &); + bool on_check_sendproof(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::CheckSendproof::Request &&, api::cnd::CheckSendproof::Response &); + bool on_getblocktemplate(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetBlockTemplate::Request &&r, api::cnd::GetBlockTemplate::Response &); + void getblocktemplate(const api::cnd::GetBlockTemplate::Request &, api::cnd::GetBlockTemplate::Response &); + bool on_get_currency_id(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetCurrencyId::Request &&, api::cnd::GetCurrencyId::Response &); void submit_block(const BinaryArray &blockblob, api::BlockHeader *info); - bool on_submitblock(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::SubmitBlock::Request &&, api::bytecoind::SubmitBlock::Response &); - bool on_submitblock_legacy(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::SubmitBlockLegacy::Request &&, api::bytecoind::SubmitBlockLegacy::Response &); - bool on_get_last_block_header(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetLastBlockHeaderLegacy::Request &&, api::bytecoind::GetLastBlockHeaderLegacy::Response &); - bool on_get_block_header_by_hash(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetBlockHeaderByHashLegacy::Request &&, api::bytecoind::GetBlockHeaderByHashLegacy::Response &); - bool on_get_block_header_by_height(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetBlockHeaderByHeightLegacy::Request &&, - api::bytecoind::GetBlockHeaderByHeightLegacy::Response &); - - bool on_json_rpc(http::Client *, http::RequestData &&, http::ResponseData &); - bool on_binary_rpc(http::Client *, http::RequestData &&, http::ResponseData &); + bool on_submitblock(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::cnd::SubmitBlock::Request &&, + api::cnd::SubmitBlock::Response &); + bool on_submitblock_legacy(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::SubmitBlockLegacy::Request &&, api::cnd::SubmitBlockLegacy::Response &); + bool on_get_last_block_header(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetLastBlockHeaderLegacy::Request &&, api::cnd::GetLastBlockHeaderLegacy::Response &); + bool on_get_block_header_by_hash(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetBlockHeaderByHashLegacy::Request &&, api::cnd::GetBlockHeaderByHashLegacy::Response &); + bool on_get_block_header_by_height(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetBlockHeaderByHeightLegacy::Request &&, api::cnd::GetBlockHeaderByHeightLegacy::Response &); + + bool on_json_rpc(http::Client *, http::RequestBody &&, http::ResponseBody &); + bool on_binary_rpc(http::Client *, http::RequestBody &&, http::ResponseBody &); BlockChainState &m_block_chain; const Config &m_config; @@ -99,9 +95,9 @@ class Node { std::unique_ptr m_prevent_sleep; struct LongPollClient { http::Client *original_who = nullptr; - http::RequestData original_request; + http::RequestBody original_request; json_rpc::Request original_json_request; - api::bytecoind::GetStatus::Request original_get_status; + api::cnd::GetStatus::Request original_get_status; }; std::list m_long_poll_http_clients; void advance_long_poll(); @@ -117,242 +113,103 @@ class Node { const Timestamp m_start_time; platform::Timer m_commit_timer; - std::unique_ptr prevent_sleep; void db_commit(); - bool check_trust(const np::ProofOfTrust &); - bool check_trust(const ProofOfTrustLegacy &); - uint64_t m_last_stat_request_time = 0; // TODO - Timestamp type after getting rid of old p2p + bool check_trust(const p2p::ProofOfTrust &); + Timestamp m_last_stat_request_time = 0; // Prevent replay attacks by only trusting requests with timestamp > than previous request + class P2PProtocolBytecoin; + struct DownloadInfo { + size_t chain_counter = 0; + P2PProtocolBytecoin *who_downloading = nullptr; + Height expected_height = 0; // Set during download + bool preparing = false; + }; + std::map chain_blocks; + void remove_chain_block(std::map::iterator it); + std::map downloading_transactions; + class P2PProtocolBytecoin : public P2PProtocolBasic { Node *const m_node; void after_handshake(); - protected: - virtual void on_disconnect(const std::string &ban_reason) override; - - virtual void on_msg_bytes(size_t downloaded, size_t uploaded) override; - virtual CORE_SYNC_DATA get_sync_data() const override; - virtual std::vector get_peers_to_share() const override; - - virtual void on_immediate_protocol_switch(unsigned char first_byte) override; - virtual void on_first_message_after_handshake() override; - virtual void on_msg_handshake(COMMAND_HANDSHAKE::request &&) override; - virtual void on_msg_handshake(COMMAND_HANDSHAKE::response &&) override; - virtual void on_msg_notify_request_chain(NOTIFY_REQUEST_CHAIN::request &&) override; - virtual void on_msg_notify_request_chain(NOTIFY_RESPONSE_CHAIN_ENTRY::request &&) override; - virtual void on_msg_notify_request_objects(NOTIFY_REQUEST_GET_OBJECTS::request &&) override; - virtual void on_msg_notify_request_objects(NOTIFY_RESPONSE_GET_OBJECTS::request &&) override; - virtual void on_msg_notify_request_tx_pool(NOTIFY_REQUEST_TX_POOL::request &&) override; - virtual void on_msg_timed_sync(COMMAND_TIMED_SYNC::request &&) override; - virtual void on_msg_timed_sync(COMMAND_TIMED_SYNC::response &&) override; - virtual void on_msg_notify_new_block(NOTIFY_NEW_BLOCK::request &&) override; - virtual void on_msg_notify_new_transactions(NOTIFY_NEW_TRANSACTIONS::request &&) override; - virtual void on_msg_notify_checkpoint(NOTIFY_CHECKPOINT::request &&) override; -#if bytecoin_ALLOW_DEBUG_COMMANDS - virtual void on_msg_network_state(COMMAND_REQUEST_NETWORK_STATE::request &&) override; - virtual void on_msg_stat_info(COMMAND_REQUEST_STAT_INFO::request &&) override; -#endif - public: - explicit P2PProtocolBytecoin(Node *node, P2PClient *client) - : P2PProtocolBasic(node->m_config, node->m_p2p.get_unique_number(), client), m_node(node) {} - ~P2PProtocolBytecoin(); - Node *get_node() const { return m_node; } - }; - std::unique_ptr client_factory(P2PClient *client) { - return std::make_unique(this, client); - } - class DownloaderV11 { // torrent-style sync&download from legacy v1 clients - Node *const m_node; - BlockChainState &m_block_chain; - - std::map m_good_clients; // -> # of downloading blocks - size_t total_downloading_blocks = 0; - std::list m_who_downloaded_block; - P2PProtocolBytecoin *m_chain_client = nullptr; - bool m_chain_request_sent = false; - platform::Timer m_chain_timer; // If m_chain_client does not respond for long, disconnect it - - struct DownloadCell { - Hash bid; - Height expected_height = 0; - NetworkAddress bid_source; // for banning culprit in case of a problem - NetworkAddress block_source; // for banning culprit in case of a problem - P2PProtocolBytecoin *downloading_client = nullptr; - std::chrono::steady_clock::time_point request_time; - RawBlock rb; - enum Status { DOWNLOADING, DOWNLOADED, PREPARING, PREPARED } status = DOWNLOADING; - bool protect_from_disconnect = false; - PreparedBlock pb; - }; - std::deque - m_download_chain; // ~20-1000 of blocks we wish to have downloading (depending on current median size) - // Height m_protected_start = 0; - Height m_chain_start_height = 0; - std::deque m_chain; // 10k-20k of hashes of the next wanted blocks - NetworkAddress chain_source; // for banning culprit in case of a problem + bool m_chain_request_sent = false; + platform::Timer m_chain_timer; platform::Timer m_download_timer; - std::chrono::steady_clock::time_point log_request_timestamp; - std::chrono::steady_clock::time_point log_response_timestamp; - - // multicore preparator - std::vector threads; - std::mutex mu; - std::map prepared_blocks; - std::deque> work; - std::condition_variable have_work; - platform::EventLoop *main_loop = nullptr; - bool quit = false; - void add_work(std::tuple &&wo); - void thread_run(); - - void start_download(DownloadCell &dc, P2PProtocolBytecoin *who); - void stop_download(DownloadCell &dc, bool success); + size_t m_downloading_block_count = 0; void on_chain_timer(); void on_download_timer(); - void advance_chain(); - - public: - DownloaderV11(Node *node, BlockChainState &block_chain); - ~DownloaderV11(); - - void advance_download(); - bool on_idle(); - - uint32_t get_known_block_count(uint32_t my) const; - void on_connect(P2PProtocolBytecoin *); - void on_disconnect(P2PProtocolBytecoin *); - const std::map &get_good_clients() const { return m_good_clients; } - void on_msg_notify_request_chain(P2PProtocolBytecoin *, const NOTIFY_RESPONSE_CHAIN_ENTRY::request &); - void on_msg_notify_request_objects(P2PProtocolBytecoin *, const NOTIFY_RESPONSE_GET_OBJECTS::request &); - void on_msg_timed_sync(const CORE_SYNC_DATA &payload_data); - }; - class P2PProtocolBytecoinNew : public P2PProtocolNew { - Node *const m_node; - void after_handshake(); - - void on_download_timer(); + Hash m_previous_chain_hash; + std::deque::iterator> m_chain; + size_t m_chain_start_height = 0; + + bool m_syncpool_equest_sent = false; + std::pair syncpool_start{std::numeric_limits::max(), Hash{}}; + size_t m_downloading_transaction_count = 0; + platform::Timer m_syncpool_timer; + platform::Timer m_download_transactions_timer; + std::map m_transaction_descs; + void on_syncpool_timer(); + void on_download_transactions_timer(); + void transaction_download_finished(const Hash &tid, bool success); + bool on_transaction_descs(const std::vector &descs); protected: void on_disconnect(const std::string &ban_reason) override; - void on_msg_bytes(size_t, size_t) override; + void on_msg_bytes(size_t downloaded, size_t uploaded) override; + CoreSyncData get_my_sync_data() const override; + std::vector get_peers_to_share(bool lots) const override; - void on_msg_handshake(np::Handshake::Request &&req) override; - void on_msg_handshake(np::Handshake::Response &&req) override; - void on_msg_find_diff(np::FindDiff::Request &&) override; - void on_msg_find_diff(np::FindDiff::Response &&) override; - void on_msg_sync_headers(np::SyncHeaders::Request &&) override; - void on_msg_sync_headers(np::SyncHeaders::Response &&) override; - void on_msg_get_transactions(np::GetTransactions::Request &&) override; - void on_msg_get_transactions(np::GetTransactions::Response &&) override; - void on_msg_get_pool_hashes(np::GetPoolHashes::Request &&) override; - void on_msg_get_pool_hashes(np::GetPoolHashes::Response &&) override; - void on_msg_relay_block_header(np::RelayBlockHeader &&) override; - void on_msg_relay_transaction_desc(np::RelayTransactionDescs &&) override; + void on_first_message_after_handshake() override; + void on_msg_handshake(p2p::Handshake::Request &&) override; + void on_msg_handshake(p2p::Handshake::Response &&) override; + void on_msg_notify_request_chain(p2p::GetChainRequest::Notify &&) override; + void on_msg_notify_request_chain(p2p::GetChainResponse::Notify &&) override; + void on_msg_notify_request_objects(p2p::GetObjectsRequest::Notify &&) override; + void on_msg_notify_request_objects(p2p::GetObjectsResponse::Notify &&) override; + void on_msg_notify_request_tx_pool(p2p::SyncPool::Notify &&) override; + void on_msg_notify_request_tx_pool(p2p::SyncPool::Request &&) override; + void on_msg_notify_request_tx_pool(p2p::SyncPool::Response &&) override; + void on_msg_timed_sync(p2p::TimedSync::Request &&) override; + void on_msg_timed_sync(p2p::TimedSync::Response &&) override; + void on_msg_notify_new_block(p2p::RelayBlock::Notify &&) override; + void on_msg_notify_new_transactions(p2p::RelayTransactions::Notify &&) override; + void on_msg_notify_checkpoint(p2p::Checkpoint::Notify &&) override; #if bytecoin_ALLOW_DEBUG_COMMANDS - void on_msg_get_peer_statistics(np::GetPeerStatistics::Request &&) override; + void on_msg_stat_info(p2p::GetStatInfo::Request &&) override; #endif - void on_first_message_after_handshake() override; - np::TopBlockDesc get_top_block_desc() const override; - std::vector get_peers_to_share() const override; - public: - explicit P2PProtocolBytecoinNew(Node *node, P2PClient *client) - : P2PProtocolNew( - node->m_config, node->m_block_chain.get_currency(), node->m_p2p.get_unique_number(), client) - , m_node(node) - , m_download_timer(std::bind(&P2PProtocolBytecoinNew::on_download_timer, this)) {} + explicit P2PProtocolBytecoin(Node *node, P2PClient *client); + ~P2PProtocolBytecoin() override; Node *get_node() const { return m_node; } - - std::set> can_download_blocks; - std::set downloading_blocks; - platform::Timer m_download_timer; // Reset when start download or receive block - }; - class DownloaderV3 { // torrent-style sync&download from new v3 clients - Node *const m_node; - BlockChainState &m_block_chain; - - // std::set> fill_can_download(Hash hash)const; - - std::map m_good_clients; // -> # of downloading blocks - size_t total_downloading_blocks = 0; - std::list m_who_downloaded_block; - P2PProtocolBytecoinNew *m_find_diff_client = nullptr; - int m_find_diff_iteration = 0; - Hash m_find_diff_bid; - P2PProtocolBytecoinNew *m_sync_headers_client = nullptr; - Hash m_sync_headers_previous_block_hash; - platform::Timer m_chain_timer; // If m_chain_client does not respond for long, disconnect it - - struct DownloadCell { - Hash bid; - Height expected_height = 0; - NetworkAddress block_source; // for banning culprit in case of a problem - P2PProtocolBytecoinNew *downloading_client = nullptr; - std::chrono::steady_clock::time_point request_time; - RawBlock rb; - enum Status { DOWNLOADING, DOWNLOADED, PREPARING, PREPARED } status = DOWNLOADING; - bool protect_from_disconnect = false; - PreparedBlock pb; - }; - std::deque - m_download_chain; // ~20-1000 of blocks we wish to have downloading (depending on current median size) - platform::Timer m_download_timer; - std::chrono::steady_clock::time_point log_request_timestamp; - std::chrono::steady_clock::time_point log_response_timestamp; - - // multicore preparator - std::vector threads; - std::mutex mu; - std::map prepared_blocks; - std::deque> work; - std::condition_variable have_work; - platform::EventLoop *main_loop = nullptr; - bool quit = false; - void add_work(std::tuple &&wo); - void thread_run(); - - void start_download(DownloadCell &dc, P2PProtocolBytecoinNew *who); - void stop_download(DownloadCell &dc, bool success); - void on_chain_timer(); - void on_download_timer(); void advance_chain(); + void advance_blocks(); + bool on_idle(std::chrono::steady_clock::time_point idle_start); + void advance_transactions(); + }; + std::unique_ptr client_factory(P2PClient *client) { + return std::make_unique(this, client); + } - public: - DownloaderV3(Node *node, BlockChainState &block_chain); - ~DownloaderV3(); - - void advance_download(); - bool on_idle(); + std::chrono::steady_clock::time_point log_request_timestamp; + std::chrono::steady_clock::time_point log_response_timestamp; - uint32_t get_known_block_count(uint32_t my) const; - void on_connect(P2PProtocolBytecoinNew *); - void on_disconnect(P2PProtocolBytecoinNew *); - const std::map &get_good_clients() const { return m_good_clients; } + void advance_all_downloads(); + std::set m_broadcast_protocols; - void on_msg_find_diff(P2PProtocolBytecoinNew *, np::FindDiff::Response &&resp); - void on_msg_sync_headers(P2PProtocolBytecoinNew *, np::SyncHeaders::Response &&resp); - void on_msg_get_transactions(P2PProtocolBytecoinNew *, np::GetTransactions::Response &&resp); - }; + BlockPreparatorMulticore m_pow_checker; + // TODO - periodically clear m_pow_checker of blocks that were not asked - std::set broadcast_protocols; void broadcast(P2PProtocolBytecoin *exclude, const BinaryArray &data); - std::set broadcast_protocols_new; - void broadcast_new(P2PProtocolBytecoinNew *exclude, const BinaryArray &binary_header); - void broadcast_new(P2PProtocolBytecoinNew *exclude, const std::vector &transaction_descs); - DownloaderV11 m_downloader; - DownloaderV3 m_downloader_v3; + void broadcast(P2PProtocolBytecoin *exclude, const BinaryArray &data_v1, const BinaryArray &data_v4); - bool on_api_http_request(http::Client *, http::RequestData &&, http::ResponseData &); + bool on_api_http_request(http::Client *, http::RequestBody &&, http::ResponseBody &); void on_api_http_disconnect(http::Client *); - void sync_transactions(P2PProtocolBytecoin *); - void sync_transactions(P2PProtocolBytecoinNew *) {} - static std::unordered_map m_jsonrpc_handlers; static const std::unordered_map m_binaryrpc_handlers; }; -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/NodeDownloader.cpp b/src/Core/NodeDownloader.cpp deleted file mode 100644 index 6cbd852f..00000000 --- a/src/Core/NodeDownloader.cpp +++ /dev/null @@ -1,485 +0,0 @@ -// Copyright (c) 2012-2018, The CryptoNote developers, The Bytecoin developers. -// Licensed under the GNU Lesser General Public License. See LICENSE for details. - -#include -#include "Config.hpp" -#include "CryptoNoteTools.hpp" -#include "Node.hpp" -#include "seria/BinaryInputStream.hpp" -#include "seria/BinaryOutputStream.hpp" - -using namespace bytecoin; - -static const bool multicore = true; - -Node::DownloaderV11::DownloaderV11(Node *node, BlockChainState &block_chain) - : m_node(node) - , m_block_chain(block_chain) - , m_chain_timer(std::bind(&DownloaderV11::on_chain_timer, this)) - , m_download_timer(std::bind(&DownloaderV11::on_download_timer, this)) - , log_request_timestamp(std::chrono::steady_clock::now()) - , log_response_timestamp(std::chrono::steady_clock::now()) { - if (multicore) { - auto th_count = std::max(2, std::thread::hardware_concurrency() / 2); - // we use more energy but have the same speed when using hyperthreading - // std::cout << "Starting multicore POW checker using " << th_count << "/" << - // std::thread::hardware_concurrency() - // << " cpus" << std::endl; - for (size_t i = 0; i != th_count; ++i) - threads.emplace_back(&DownloaderV11::thread_run, this); - main_loop = platform::EventLoop::current(); - } - m_download_timer.once(SYNC_TIMEOUT / 8); // just several ticks per SYNC_TIMEOUT -} - -Node::DownloaderV11::~DownloaderV11() { - { - std::unique_lock lock(mu); - quit = true; - have_work.notify_all(); - } - for (auto &&th : threads) - th.join(); -} - -void Node::DownloaderV11::add_work(std::tuple &&wo) { - std::unique_lock lock(mu); - work.push_back(std::move(wo)); - have_work.notify_all(); -} - -void Node::DownloaderV11::thread_run() { - crypto::CryptoNightContext hash_crypto_context; - while (true) { - std::tuple wo; - { - std::unique_lock lock(mu); - if (quit) - return; - if (work.empty()) { - have_work.wait(lock); - continue; - } - wo = std::move(work.front()); - work.pop_front(); - } - PreparedBlock result(std::move(std::get<2>(wo)), - m_node->m_block_chain.get_currency(), - std::get<1>(wo) ? &hash_crypto_context : nullptr); - { - std::unique_lock lock(mu); - prepared_blocks[std::get<0>(wo)] = std::move(result); - main_loop->wake(); // so we start processing on_idle - } - } -} - -uint32_t Node::DownloaderV11::get_known_block_count(uint32_t my) const { - for (auto &&gc : m_good_clients) - my = std::max(my, gc.first->get_last_received_sync_data().current_height); - return my; -} - -void Node::DownloaderV11::on_connect(P2PProtocolBytecoin *who) { - if (who->is_incoming()) // Never sync from incoming - return; - m_node->m_log(logging::TRACE) << "DownloaderV11::on_connect " << who->get_address() << std::endl; - invariant(m_good_clients.insert(std::make_pair(who, 0)).second, ""); - if (who->get_last_received_sync_data().current_height == m_block_chain.get_tip_height()) { - m_node->m_log(logging::TRACE) << "DownloaderV11::on_connect sync_transactions to " << who->get_address() - << " our pool size=" - << m_node->m_block_chain.get_memory_state_transactions().size() << std::endl; - m_node->sync_transactions(who); - // If we at same height, sync tx now, otherwise will sync after we reach same height - } - advance_download(); -} - -void Node::DownloaderV11::on_disconnect(P2PProtocolBytecoin *who) { - if (who->is_incoming()) - return; - if (m_good_clients.count(who) == 0) // Remove only if we have it added - return; - m_node->m_log(logging::TRACE) << "DownloaderV11::on_disconnect " << who->get_address() << std::endl; - invariant(total_downloading_blocks >= m_good_clients[who], "total_downloading_blocks mismatch in disconnect"); - total_downloading_blocks -= m_good_clients[who]; - m_good_clients.erase(who); - for (auto lit = m_who_downloaded_block.begin(); lit != m_who_downloaded_block.end();) - if (*lit == who) - lit = m_who_downloaded_block.erase(lit); - else - ++lit; - for (auto &&dc : m_download_chain) { - if (dc.status == DownloadCell::DOWNLOADING && dc.downloading_client == who) - dc.downloading_client = nullptr; - } - if (m_chain_client && m_chain_client == who) { - m_chain_timer.cancel(); - m_chain_client = nullptr; - m_chain_request_sent = false; - m_node->m_log(logging::TRACE) << "DownloaderV11::on_disconnect m_chain_client reset to 0" << std::endl; - } - advance_download(); -} - -void Node::DownloaderV11::on_chain_timer() { - if (m_chain_client) { - m_node->m_log(logging::TRACE) << "DownloaderV11::on_chain_timer" << std::endl; - m_chain_client->disconnect(std::string()); - } -} - -void Node::DownloaderV11::on_msg_notify_request_chain(P2PProtocolBytecoin *who, - const NOTIFY_RESPONSE_CHAIN_ENTRY::request &req) { - if (m_chain_client != who || !m_chain_request_sent) - return; // TODO - who just sent us chain we did not ask, ban - m_chain_request_sent = false; - m_chain_timer.cancel(); - m_node->m_log(logging::INFO) << "DownloaderV11 received chain from " << who->get_address() - << " start_height=" << req.start_height << " length=" << req.m_block_ids.size() - << std::endl; - m_chain_start_height = req.start_height; - chain_source = m_chain_client->get_address(); - m_chain.assign(req.m_block_ids.begin(), req.m_block_ids.end()); - // Hash last_downloaded_block = m_chain.empty() ? Hash{} : m_chain.back(); - std::set downloading_bids; - for (auto &&dc : m_download_chain) - downloading_bids.insert(dc.bid); - while (!m_chain.empty() && - (m_node->m_block_chain.has_block(m_chain.front()) || downloading_bids.count(m_chain.front()) != 0)) { - m_chain.pop_front(); - m_chain_start_height += 1; - } // We stop removing as soon as we find new block, because wrong order might prevent us from applying blocks - if (req.m_block_ids.size() != m_chain.size() + 1) { - m_node->m_log(logging::INFO) << "DownloaderV11 truncated chain length=" << m_chain.size() << std::endl; - } - if (req.m_block_ids.empty()) { // Most likely peer is 3.2.0 - const auto now = m_node->m_p2p.get_local_time(); - m_node->m_log(logging::INFO) << "DownloaderV11 truncated chain to zero, delaying connect to " - << who->get_address() << std::endl; - m_node->m_peer_db.delay_connection_attempt(who->get_address(), now); - who->disconnect(std::string()); // Will recursively call advance_chain again - return; - } - advance_download(); -} - -static const size_t GOOD_LAG = 5; // lagging by 5 blocks is ok for us - -void Node::DownloaderV11::advance_chain() { - if (!m_chain.empty() || !m_download_chain.empty() || m_chain_request_sent) - return; - m_chain_client = nullptr; - std::vector lagging_clients; - std::vector worth_clients; - const auto now = m_node->m_p2p.get_local_time(); - for (auto &&who : m_good_clients) { - if (who.first->get_last_received_sync_data().current_height + GOOD_LAG < m_node->m_block_chain.get_tip_height()) - lagging_clients.push_back(who.first); - api::BlockHeader info; - if (!m_node->m_block_chain.read_header(who.first->get_last_received_sync_data().top_id, &info)) - worth_clients.push_back(who.first); - } - if (lagging_clients.size() > m_node->m_config.p2p_max_outgoing_connections / 4) { - auto who = lagging_clients.front(); - m_node->m_peer_db.delay_connection_attempt(who->get_address(), now); - m_node->m_log(logging::INFO) << "DownloaderV11 disconnecting lagging client " << who->get_address() - << std::endl; - who->disconnect(std::string()); // Will recursively call advance_chain again - return; - } - if (worth_clients.empty()) - return; // We hope to get more connections soon - m_chain_client = worth_clients.at(crypto::rand() % worth_clients.size()); - m_chain_request_sent = true; - NOTIFY_REQUEST_CHAIN::request msg; - msg.block_ids = m_block_chain.get_sparse_chain(); - - m_node->m_log(logging::INFO) << "DownloaderV11::advance_chain Requesting chain from " - << m_chain_client->get_address() - << " remote height=" << m_chain_client->get_last_received_sync_data().current_height - << " our height=" << m_block_chain.get_tip_height() << std::endl; - BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_REQUEST_CHAIN::ID, LevinProtocol::encode(msg), false); - m_chain_client->send(std::move(raw_msg)); - m_chain_timer.once(SYNC_TIMEOUT); -} - -void Node::DownloaderV11::start_download(DownloadCell &dc, P2PProtocolBytecoin *who) { - auto idea_now = std::chrono::steady_clock::now(); - dc.downloading_client = who; - dc.block_source = who->get_address(); - dc.request_time = idea_now; - m_good_clients[dc.downloading_client] += 1; - total_downloading_blocks += 1; - NOTIFY_REQUEST_GET_OBJECTS::request msg; - msg.blocks.push_back(dc.bid); - if (std::chrono::duration_cast(idea_now - log_request_timestamp).count() > 1000) { - log_request_timestamp = idea_now; - std::cout << "Requesting block " << dc.expected_height << " from " << dc.downloading_client->get_address() - << std::endl; - } - m_node->m_log(logging::TRACE) << "DownloaderV11::advance_download requesting block " << dc.expected_height - << " hash=" << dc.bid << " from " << dc.downloading_client->get_address() - << std::endl; - BinaryArray raw_msg = - LevinProtocol::send_message(NOTIFY_REQUEST_GET_OBJECTS::ID, LevinProtocol::encode(msg), false); - dc.downloading_client->send(std::move(raw_msg)); -} - -void Node::DownloaderV11::stop_download(DownloadCell &dc, bool success) { - if (dc.status != DownloadCell::DOWNLOADING || !dc.downloading_client) - return; - auto git = m_good_clients.find(dc.downloading_client); - invariant(git != m_good_clients.end() && git->second != 0 && total_downloading_blocks != 0, - "DownloadCell reference to good client not found"); - git->second -= 1; - total_downloading_blocks -= 1; - if (success) { - dc.status = DownloadCell::DOWNLOADED; - m_who_downloaded_block.push_back(dc.downloading_client); - } - dc.downloading_client = nullptr; -} - -void Node::DownloaderV11::on_msg_notify_request_objects(P2PProtocolBytecoin *who, - const NOTIFY_RESPONSE_GET_OBJECTS::request &req) { - for (auto &&rb : req.blocks) { - Hash bid; - try { - BlockTemplate bheader; - seria::from_binary(bheader, rb.block); - auto body_proxy = get_body_proxy_from_template(bheader); - bid = bytecoin::get_block_hash(bheader, body_proxy); - } catch (const std::exception &ex) { - m_node->m_log(logging::INFO) << "Exception " << common::what(ex) - << " while parsing returned block, banning " << who->get_address() - << std::endl; - who->disconnect(std::string()); - break; - } - bool cell_found = false; - for (auto &&dc : m_download_chain) { - if (dc.status != DownloadCell::DOWNLOADING || dc.downloading_client != who || dc.bid != bid) - continue; // downloaded or downloading - stop_download(dc, true); - dc.rb.block = rb.block; // TODO - std::move - dc.rb.transactions = rb.transactions; // TODO - std::move - auto now = std::chrono::steady_clock::now(); - if (std::chrono::duration_cast(now - log_response_timestamp).count() > 1000) { - log_response_timestamp = now; - std::cout << "Received block with height=" << dc.expected_height - << " (queue=" << total_downloading_blocks << ") from " << who->get_address() << std::endl; - } - m_node->m_log(logging::TRACE) - << "DownloaderV11 received block with height=" << dc.expected_height << " hash=" << dc.bid - << " (queue=" << total_downloading_blocks << ") from " << who->get_address() << std::endl; - cell_found = true; - if (multicore) { - dc.status = DownloadCell::PREPARING; - add_work(std::tuple(dc.bid, - !m_node->m_block_chain.get_currency().is_in_sw_checkpoint_zone(dc.expected_height), - std::move(dc.rb))); - } else { - dc.pb = PreparedBlock(std::move(dc.rb), m_node->m_block_chain.get_currency(), nullptr); - dc.status = DownloadCell::PREPARED; - } - break; - } - if (!cell_found) { - m_node->m_log(logging::INFO) << "DownloaderV11 received stray block from " << who->get_address() - << std::endl; - // who->disconnect(std::string()); - // break; - } - } - for (auto &&bid : req.missed_ids) { - for (size_t dit_counter = 0; dit_counter != m_download_chain.size(); ++dit_counter) { - auto &dit = m_download_chain.at(dit_counter); - if (dit.status != DownloadCell::DOWNLOADING || dit.downloading_client != who || dit.bid != bid) - continue; // downloaded or downloading - stop_download(dit, false); - if (!m_chain_client || m_chain_client == who) { - m_node->m_log(logging::INFO) - << "DownloaderV11 cannot download block from any connected client, cleaning chain" << std::endl; - while (m_download_chain.size() > dit_counter) { - stop_download(m_download_chain.back(), false); - m_download_chain.pop_back(); - } - m_chain.clear(); - advance_download(); - return; - } - start_download(dit, m_chain_client); - } - } - advance_download(); -} - -bool Node::DownloaderV11::on_idle() { - int added_counter = 0; - if (multicore) { - std::unique_lock lock(mu); - for (auto &&pb : prepared_blocks) { - for (auto &&dc : m_download_chain) - if (dc.status == DownloadCell::PREPARING && dc.bid == pb.first) { - dc.pb = std::move(pb.second); - dc.status = DownloadCell::PREPARED; - break; - } - } - prepared_blocks.clear(); - } - auto idea_start = std::chrono::high_resolution_clock::now(); - while (!m_download_chain.empty() && m_download_chain.front().status == DownloadCell::PREPARED) { - DownloadCell dc = std::move(m_download_chain.front()); - m_download_chain.pop_front(); - api::BlockHeader info; - auto action = m_block_chain.add_block( - dc.pb, &info, common::ip_address_and_port_to_string(dc.block_source.ip, dc.block_source.port)); - if (action == BroadcastAction::BAN) { - m_node->m_log(logging::INFO) << "DownloaderV11 DownloadCell BAN height=" << dc.expected_height - << " wb=" << dc.bid << std::endl; - // TODO - ban client who gave us chain - // continue; - } - // if (action == BroadcastAction::NOTHING) - // std::cout << "BroadcastAction::NOTHING height=" << info.height << " cd=" << - // info.cumulative_difficulty.lo - // << std::endl; - if (action == BroadcastAction::BROADCAST_ALL) { - // std::cout << "BroadcastAction::BROADCAST_ALL height=" << info.height - // << " cd=" << info.cumulative_difficulty.lo << std::endl; - if (m_download_chain.empty()) { - // We do not want to broadcast too often during download - m_node->m_log(logging::INFO) << "Added last (from batch) downloaded block height=" << info.height - << " bid=" << info.hash << std::endl; - COMMAND_TIMED_SYNC::request req; - req.payload_data = - CORE_SYNC_DATA{m_node->m_block_chain.get_tip_height(), m_node->m_block_chain.get_tip_bid()}; - BinaryArray raw_msg = - LevinProtocol::send_message(COMMAND_TIMED_SYNC::ID, LevinProtocol::encode(req), true); - m_node->broadcast( - nullptr, raw_msg); // nullptr - we can not always know which connection was block source - // m_node->broadcast_new(nullptr, raw_msg); // TODO nullptr - we can not always know which - // connection was block source - } - } - added_counter += 1; - auto idea_ms = std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - idea_start); - if (idea_ms.count() > 100) - break; - } - if (added_counter) { - m_node->advance_long_poll(); - advance_download(); - if (m_download_chain.empty()) - for (auto &&who : m_good_clients) { - if (who.first->get_last_received_sync_data().current_height == m_node->m_block_chain.get_tip_height()) { - m_node->m_log(logging::TRACE) - << "DownloaderV11::on_idle sync_transactions to " << who.first->get_address() - << " our pool size=" << m_node->m_block_chain.get_memory_state_transactions().size() - << std::endl; - m_node->sync_transactions(who.first); - break; // TODO - sync with all nodes - } - } - } - - return !m_download_chain.empty() && m_download_chain.front().status == DownloadCell::PREPARED; -} - -void Node::DownloaderV11::on_download_timer() { - m_download_timer.once(SYNC_TIMEOUT / 8); // just several ticks per SYNC_TIMEOUT - auto idea_now = std::chrono::steady_clock::now(); - if (!m_download_chain.empty() && m_download_chain.front().status == DownloadCell::DOWNLOADING && - m_download_chain.front().downloading_client && m_download_chain.front().protect_from_disconnect && - std::chrono::duration_cast(idea_now - m_download_chain.front().request_time).count() > - SYNC_TIMEOUT) { - auto who = m_download_chain.front().downloading_client; - m_node->m_peer_db.delay_connection_attempt(who->get_address(), m_node->m_p2p.get_local_time()); - m_node->m_log(logging::INFO) << "DownloaderV11 disconnecting protected slacker " << who->get_address() - << std::endl; - who->disconnect(std::string()); - } -} - -void Node::DownloaderV11::advance_download() { - if (m_node->m_block_chain_reader1 || m_node->m_block_chain_reader2 || - m_block_chain.get_tip_height() < m_block_chain.internal_import_known_height()) - return; - const size_t TOTAL_DOWNLOAD_BLOCKS = 400; // TODO - dynamic count - const size_t TOTAL_DOWNLOAD_WINDOW = 2000; // TODO - dynamic count - while (m_download_chain.size() < TOTAL_DOWNLOAD_WINDOW && !m_chain.empty()) { - m_download_chain.push_back(DownloadCell()); - m_download_chain.back().bid = m_chain.front(); - m_download_chain.back().expected_height = m_chain_start_height; - m_download_chain.back().bid_source = chain_source; - m_chain.pop_front(); - m_chain_start_height += 1; - } - advance_chain(); - - while (m_who_downloaded_block.size() > TOTAL_DOWNLOAD_BLOCKS) - m_who_downloaded_block.pop_front(); - std::map who_downloaded_counter; - for (auto lit = m_who_downloaded_block.begin(); lit != m_who_downloaded_block.end(); ++lit) - who_downloaded_counter[*lit] += 1; - auto idea_now = std::chrono::steady_clock::now(); - for (size_t dit_counter = 0; dit_counter != m_download_chain.size(); ++dit_counter) { - auto &dit = m_download_chain.at(dit_counter); - if (dit.status != DownloadCell::DOWNLOADING || dit.downloading_client) - continue; // downloaded or downloading - if (total_downloading_blocks >= TOTAL_DOWNLOAD_BLOCKS) - break; - P2PProtocolBytecoin *ready_client = nullptr; - size_t ready_counter = std::numeric_limits::max(); - size_t ready_speed = 1; - for (auto &&who : m_good_clients) { - size_t speed = - std::max(1, std::min(TOTAL_DOWNLOAD_BLOCKS / 4, who_downloaded_counter[who.first])); - // We clamp speed so that if even 1 downloaded all blocks, we will give - // small % of blocks to other peers - if (who.second * ready_speed < ready_counter * speed && - who.first->get_last_received_sync_data().current_height >= dit.expected_height) { - ready_client = who.first; - ready_counter = who.second; - ready_speed = speed; - } - } - if (!ready_client && m_chain_client) - ready_client = m_chain_client; - if (!ready_client) { // Cannot download chain from any client - m_node->m_log(logging::INFO) - << "DownloaderV11::advance_download cannot download blocks from any connected client, cleaning chain" - << std::endl; - while (m_download_chain.size() > dit_counter) { - stop_download(m_download_chain.back(), false); - m_download_chain.pop_back(); - } - m_chain.clear(); - advance_chain(); - return; - } - start_download(dit, ready_client); - } - const bool bad_timeout = - !m_download_chain.empty() && m_download_chain.front().status == DownloadCell::DOWNLOADING && - m_download_chain.front().downloading_client && !m_download_chain.front().protect_from_disconnect && - std::chrono::duration_cast(idea_now - m_download_chain.front().request_time).count() > - 2 * SYNC_TIMEOUT; - const bool bad_relatively_slow = - total_downloading_blocks < TOTAL_DOWNLOAD_BLOCKS && m_download_chain.size() >= TOTAL_DOWNLOAD_WINDOW && - m_good_clients.size() > 1 && m_download_chain.front().status == DownloadCell::DOWNLOADING && - m_download_chain.front().downloading_client && !m_download_chain.front().protect_from_disconnect; - if (bad_relatively_slow || bad_timeout) { - auto who = m_download_chain.front().downloading_client; - for (auto &&dc : m_download_chain) - if (dc.downloading_client == who) - dc.protect_from_disconnect = true; - m_node->m_peer_db.delay_connection_attempt(who->get_address(), m_node->m_p2p.get_local_time()); - m_node->m_log(logging::INFO) << "DownloaderV11::advance_download disconnecting slacker " << who->get_address() - << std::endl; - who->disconnect(std::string()); - } -} diff --git a/src/Core/NodeDownloaderV3.cpp b/src/Core/NodeDownloaderV3.cpp deleted file mode 100644 index 04974629..00000000 --- a/src/Core/NodeDownloaderV3.cpp +++ /dev/null @@ -1,619 +0,0 @@ -// Copyright (c) 2012-2018, The CryptoNote developers, The Bytecoin developers. -// Licensed under the GNU Lesser General Public License. See LICENSE for details. - -#include -#include "Config.hpp" -#include "Node.hpp" -#include "seria/BinaryInputStream.hpp" -#include "seria/BinaryOutputStream.hpp" - -using namespace bytecoin; - -static const bool multicore = true; - -Node::DownloaderV3::DownloaderV3(Node *node, BlockChainState &block_chain) - : m_node(node) - , m_block_chain(block_chain) - , m_chain_timer(std::bind(&DownloaderV3::on_chain_timer, this)) - , m_download_timer(std::bind(&DownloaderV3::on_download_timer, this)) - , log_request_timestamp(std::chrono::steady_clock::now()) - , log_response_timestamp(std::chrono::steady_clock::now()) { - if (multicore) { - auto th_count = std::max(2, std::thread::hardware_concurrency() / 2); - // we use more energy but have the same speed when using hyperthreading - // std::cout << "Starting multicore POW checker using " << th_count << "/" << - // std::thread::hardware_concurrency() - // << " cpus" << std::endl; - for (size_t i = 0; i != th_count; ++i) - threads.emplace_back(&DownloaderV3::thread_run, this); - main_loop = platform::EventLoop::current(); - } - m_download_timer.once(SYNC_TIMEOUT / 8); // just several ticks per SYNC_TIMEOUT -} - -Node::DownloaderV3::~DownloaderV3() { - { - std::unique_lock lock(mu); - quit = true; - have_work.notify_all(); - } - for (auto &&th : threads) - th.join(); -} - -// std::set> Node::DownloaderV3::fill_can_download(Hash hash)const{ -// -//} - -void Node::DownloaderV3::add_work(std::tuple &&wo) { - std::unique_lock lock(mu); - work.push_back(std::move(wo)); - have_work.notify_all(); -} - -void Node::DownloaderV3::thread_run() { - crypto::CryptoNightContext hash_crypto_context; - while (true) { - std::tuple wo; - { - std::unique_lock lock(mu); - if (quit) - return; - if (work.empty()) { - have_work.wait(lock); - continue; - } - wo = std::move(work.front()); - work.pop_front(); - } - PreparedBlock result(std::move(std::get<2>(wo)), - m_node->m_block_chain.get_currency(), - std::get<1>(wo) ? &hash_crypto_context : nullptr); - { - std::unique_lock lock(mu); - prepared_blocks[std::get<0>(wo)] = std::move(result); - main_loop->wake(); // so we start processing on_idle - } - } -} - -uint32_t Node::DownloaderV3::get_known_block_count(uint32_t my) const { - for (auto &&gc : m_good_clients) - my = std::max(my, gc.first->get_other_top_block_desc().height); - return my; -} - -void Node::DownloaderV3::on_connect(P2PProtocolBytecoinNew *who) { - if (who->is_incoming()) // Never sync from incoming - return; - m_node->m_log(logging::TRACE) << "DownloaderV3::on_connect " << who->get_address() << std::endl; - invariant(m_good_clients.insert(std::make_pair(who, 0)).second, ""); - // compare height, not hashes. This syncs most good transactions between short splits - if (who->get_other_top_block_desc().height == m_block_chain.get_tip_height()) { - m_node->m_log(logging::TRACE) << "DownloaderV3::on_connect sync_transactions to " << who->get_address() - << " our pool size=" - << m_node->m_block_chain.get_memory_state_transactions().size() << std::endl; - m_node->sync_transactions(who); - // If we at same height, sync tx now, otherwise will sync after we reach same height - } - // who->can_download_blocks = m_block_chain.fill_can_download(who->get_other_top_block_desc().hash); - advance_download(); -} - -void Node::DownloaderV3::on_disconnect(P2PProtocolBytecoinNew *who) { - if (who->is_incoming()) - return; - if (m_good_clients.count(who) == 0) // Remove only if we have it added - return; - m_node->m_log(logging::TRACE) << "DownloaderV3::on_disconnect " << who->get_address() << std::endl; - invariant(total_downloading_blocks >= m_good_clients[who], "total_downloading_blocks mismatch in disconnect"); - total_downloading_blocks -= m_good_clients[who]; - m_good_clients.erase(who); - for (auto lit = m_who_downloaded_block.begin(); lit != m_who_downloaded_block.end();) - if (*lit == who) - lit = m_who_downloaded_block.erase(lit); - else - ++lit; - for (auto &&dc : m_download_chain) { - if (dc.status == DownloadCell::DOWNLOADING && dc.downloading_client == who) - dc.downloading_client = nullptr; - } - if (m_find_diff_client && m_find_diff_client == who) { - m_chain_timer.cancel(); - m_find_diff_client = nullptr; - m_node->m_log(logging::TRACE) << "DownloaderV3::on_disconnect find_diff_client reset to 0" << std::endl; - } - if (m_sync_headers_client && m_sync_headers_client == who) { - m_chain_timer.cancel(); - m_sync_headers_client = nullptr; - m_node->m_log(logging::TRACE) << "DownloaderV3::on_disconnect sync_headers_client reset to 0" << std::endl; - } - advance_download(); -} - -void Node::DownloaderV3::on_chain_timer() { - if (m_find_diff_client) { - m_node->m_log(logging::TRACE) << "DownloaderV3::on_chain_timer find_diff_client disconnect" << std::endl; - m_find_diff_client->disconnect(std::string()); - return; - } - if (m_sync_headers_client) { - m_node->m_log(logging::TRACE) << "DownloaderV3::on_chain_timer m_sync_headers_client disconnect" << std::endl; - m_sync_headers_client->disconnect(std::string()); - return; - } -} - -void Node::DownloaderV3::on_msg_find_diff(P2PProtocolBytecoinNew *who, np::FindDiff::Response &&resp) { - m_chain_timer.cancel(); - if (resp.sparse_chain.size() > np::FindDiff::Response::MAX_SPARSE_CHAIN_LENGTH) - return who->disconnect("MAX_SPARSE_CHAIN_LENGTH violation"); - if (who != m_find_diff_client) - return who->disconnect("Stray FindDiff Response"); - api::BlockHeader desired_header; - api::BlockHeader have_header; - if (resp.sparse_chain.size() < 2) - return who->disconnect("FindDiff sparse_chain length < 2"); - if (resp.sparse_chain.at(0).hash != m_find_diff_bid) - return who->disconnect("FindDiff sparse_chain does not start with desired_bid"); - if (m_node->m_block_chain.read_header(resp.sparse_chain.at(0).hash, &desired_header)) { - m_node->m_log(logging::INFO) << "DownloaderV3::on_msg_find_diff already have desired_bid from " - << m_sync_headers_client->get_address() << " m_find_diff_bid=" << m_find_diff_bid - << " remote height=" << m_sync_headers_client->get_other_top_block_desc().height - << " our height=" << m_block_chain.get_tip_height() << std::endl; - // While we were searching for diff, we got the desired header already - m_find_diff_client = nullptr; - advance_chain(); - return; - } - SWCheckpoint desired_sw = resp.sparse_chain.at(0); - resp.sparse_chain.erase(resp.sparse_chain.begin()); - if (!m_node->m_block_chain.read_header(resp.sparse_chain.back().hash, &have_header)) - return who->disconnect("FindDiff sparse_chain does not contain any bid we have"); - SWCheckpoint have_sw = resp.sparse_chain.back(); - resp.sparse_chain.pop_back(); - while (!resp.sparse_chain.empty() && - !m_node->m_block_chain.read_header(resp.sparse_chain.at(0).hash, &desired_header)) { - desired_sw = resp.sparse_chain.at(0); - resp.sparse_chain.erase(resp.sparse_chain.begin()); - } - while ( - !resp.sparse_chain.empty() && m_node->m_block_chain.read_header(resp.sparse_chain.back().hash, &have_header)) { - have_sw = resp.sparse_chain.back(); - resp.sparse_chain.pop_back(); - } - if (!resp.sparse_chain.empty()) - return who->disconnect("FindDiff sparse_chain with wrong order"); - if (desired_sw.height == have_sw.height + 1) { // Found! - m_sync_headers_client = m_find_diff_client; - m_find_diff_client = nullptr; - m_sync_headers_previous_block_hash = have_sw.hash; - np::SyncHeaders::Request fd; - fd.previous_hash = m_sync_headers_previous_block_hash; - fd.max_count = np::SyncHeaders::Request::GOOD_COUNT; - BinaryArray msg = seria::to_binary_kv(fd); - m_find_diff_client->send(P2PProtocolBytecoinNew::create_header(np::SyncHeaders::Request::ID, msg.size())); - m_find_diff_client->send(std::move(msg)); - m_node->m_log(logging::INFO) << "DownloaderV3::advance_chain SyncHeaders::Request from " - << m_sync_headers_client->get_address() - << " remote height=" << m_sync_headers_client->get_other_top_block_desc().height - << " our height=" << m_block_chain.get_tip_height() << std::endl; - m_chain_timer.once(SYNC_TIMEOUT); - return; - } - m_find_diff_iteration += 1; - np::FindDiff::Request fd; - fd.gap_start.push_back(have_sw.hash); - fd.desired_bid = m_find_diff_bid; - BinaryArray msg = seria::to_binary_kv(fd); - m_find_diff_client->send(P2PProtocolBytecoinNew::create_header(np::FindDiff::Request::ID, msg.size())); - m_find_diff_client->send(std::move(msg)); - m_node->m_log(logging::INFO) << "DownloaderV3::advance_chain next FindDiff::Request from " - << m_find_diff_client->get_address() - << " remote height=" << m_find_diff_client->get_other_top_block_desc().height - << " our height=" << m_block_chain.get_tip_height() << std::endl; - m_chain_timer.once(SYNC_TIMEOUT); -} -void Node::DownloaderV3::on_msg_sync_headers(P2PProtocolBytecoinNew *who, np::SyncHeaders::Response &&resp) { - m_chain_timer.cancel(); - if (resp.binary_headers.size() > np::SyncHeaders::Request::GOOD_COUNT) - return who->disconnect("SyncHeaders binary_headers too much headers returned"); - if (resp.binary_headers.size() == 0) { // Peer switched chain - m_node->m_log(logging::INFO) << "DownloaderV3::on_msg_sync_headers peer switched chains " - << m_sync_headers_client->get_address() - << " m_sync_headers_previous_block_hash=" << m_sync_headers_previous_block_hash - << " remote height=" << m_sync_headers_client->get_other_top_block_desc().height - << " our height=" << m_block_chain.get_tip_height() << std::endl; - m_sync_headers_client = nullptr; - advance_chain(); - return; - } - for (const auto &bh : resp.binary_headers) { - BlockTemplate block_header; - seria::from_binary(block_header, bh); - api::BlockHeader info; - // if (m_block_chain.add_header(block_header, &info) == BroadcastAction::BAN) - // return who->disconnect("SyncHeaders Response header banned"); - if (info.previous_block_hash != m_sync_headers_previous_block_hash) - return who->disconnect("SyncHeaders Response binary_headers do not form chain with requested start"); - m_sync_headers_previous_block_hash = info.previous_block_hash; - } - // TODO - increase probability if reply takes too long - const size_t barrier = crypto::rand() % (np::SyncHeaders::Request::GOOD_COUNT * 110 / 100); - const bool finished = m_sync_headers_previous_block_hash == m_sync_headers_client->get_other_top_block_desc().hash; - if (resp.binary_headers.size() <= barrier || finished) { - m_node->m_log(logging::INFO) << "DownloaderV3::on_msg_sync_headers probability switch " - << m_sync_headers_client->get_address() << " count=" << resp.binary_headers.size() - << " barrier=" << barrier << " finished=" << int(finished) - << " remote height=" << m_sync_headers_client->get_other_top_block_desc().height - << " our height=" << m_block_chain.get_tip_height() << std::endl; - m_sync_headers_client = nullptr; - advance_chain(); - return; - } - np::SyncHeaders::Request fd; - fd.previous_hash = m_sync_headers_previous_block_hash; - fd.max_count = np::SyncHeaders::Request::GOOD_COUNT; - BinaryArray msg = seria::to_binary_kv(fd); - m_find_diff_client->send(P2PProtocolBytecoinNew::create_header(np::SyncHeaders::Request::ID, msg.size())); - m_find_diff_client->send(std::move(msg)); - m_node->m_log(logging::INFO) << "DownloaderV3::advance_chain next SyncHeaders::Request from " - << m_sync_headers_client->get_address() - << " remote height=" << m_sync_headers_client->get_other_top_block_desc().height - << " our height=" << m_block_chain.get_tip_height() << std::endl; - m_chain_timer.once(SYNC_TIMEOUT); -} -void Node::DownloaderV3::on_msg_get_transactions(P2PProtocolBytecoinNew *who, np::GetTransactions::Response &&resp) {} - -/*void Node::DownloaderV3::on_msg_notify_request_chain(P2PProtocolBytecoin *who, - const NOTIFY_RESPONSE_CHAIN_ENTRY::request &req) { - if (m_chain_client != who || !m_chain_request_sent) - return; // TODO - who just sent us chain we did not ask, ban - m_chain_request_sent = false; - m_chain_timer.cancel(); - m_node->m_log(logging::INFO) << "Downloader received chain from " << who->get_address() - << " start_height=" << req.start_height << " length=" << req.m_block_ids.size() - << std::endl; - m_chain_start_height = req.start_height; - chain_source = m_chain_client->get_address(); - m_chain.assign(req.m_block_ids.begin(), req.m_block_ids.end()); - // Hash last_downloaded_block = m_chain.empty() ? Hash{} : m_chain.back(); - std::set downloading_bids; - for (auto &&dc : m_download_chain) - downloading_bids.insert(dc.bid); - while (!m_chain.empty() && - (m_node->m_block_chain.has_block(m_chain.front()) || downloading_bids.count(m_chain.front()) != 0)) { - m_chain.pop_front(); - m_chain_start_height += 1; - } // We stop removing as soon as we find new block, because wrong order might prevent us from applying blocks - if (req.m_block_ids.size() != m_chain.size() + 1) { - m_node->m_log(logging::INFO) << "Downloader truncated chain length=" << m_chain.size() << std::endl; - } - if (req.m_block_ids.empty()){ // Most likely peer is 3.2.0 - const auto now = m_node->m_p2p.get_local_time(); - m_node->m_log(logging::INFO) << "Downloader truncated chain to zero, delaying connect to " << who->get_address() -<< std::endl; - m_node->m_peer_db.delay_connection_attempt(who->get_address(), now); - who->disconnect(std::string()); // Will recursively call advance_chain again - } - advance_download(); -}*/ - -static const size_t GOOD_LAG = 5; // lagging by 5 blocks is ok for us - -void Node::DownloaderV3::advance_chain() { - if (m_find_diff_client || m_sync_headers_client) - return; // TODO - if number of headers we are preparing > some const - std::vector lagging_clients; - std::vector worth_clients; - const auto now = m_node->m_p2p.get_local_time(); - for (auto &&who : m_good_clients) { - if (who.first->get_other_top_block_desc().height + GOOD_LAG < m_node->m_block_chain.get_tip_height()) - lagging_clients.push_back(who.first); - api::BlockHeader info; - if (!m_node->m_block_chain.read_header(who.first->get_other_top_block_desc().hash, &info)) - worth_clients.push_back(who.first); - } - if (lagging_clients.size() > m_node->m_config.p2p_max_outgoing_connections / 4) { - auto who = lagging_clients.front(); - m_node->m_peer_db.delay_connection_attempt(who->get_address(), now); - m_node->m_log(logging::INFO) << "DownloaderV3 disconnecting lagging client " << who->get_address() << std::endl; - who->disconnect(std::string()); // Will recursively call advance_chain again - return; - } - if (worth_clients.empty()) - return; // We hope to get more connections soon - m_find_diff_client = worth_clients.at(crypto::rand() % worth_clients.size()); - m_find_diff_iteration = 0; - m_find_diff_bid = m_find_diff_client->get_other_top_block_desc().hash; - np::FindDiff::Request fd; - fd.gap_start = m_node->m_block_chain.get_sparse_chain(); - fd.desired_bid = m_find_diff_bid; - BinaryArray msg = seria::to_binary_kv(fd); - m_find_diff_client->send(P2PProtocolBytecoinNew::create_header(np::FindDiff::Request::ID, msg.size())); - m_find_diff_client->send(std::move(msg)); - m_node->m_log(logging::INFO) << "DownloaderV3::advance_chain FindDiff::Request from " - << m_find_diff_client->get_address() - << " remote height=" << m_find_diff_client->get_other_top_block_desc().height - << " our height=" << m_block_chain.get_tip_height() << std::endl; - m_chain_timer.once(SYNC_TIMEOUT); -} - -void Node::DownloaderV3::start_download(DownloadCell &dc, P2PProtocolBytecoinNew *who) { - auto idea_now = std::chrono::steady_clock::now(); - dc.downloading_client = who; - dc.block_source = who->get_address(); - dc.request_time = idea_now; - m_good_clients[dc.downloading_client] += 1; - total_downloading_blocks += 1; - NOTIFY_REQUEST_GET_OBJECTS::request msg; - msg.blocks.push_back(dc.bid); - if (std::chrono::duration_cast(idea_now - log_request_timestamp).count() > 1000) { - log_request_timestamp = idea_now; - std::cout << "Requesting block " << dc.expected_height << " from " << dc.downloading_client->get_address() - << std::endl; - } - m_node->m_log(logging::TRACE) << "DownloaderV3::advance_download requesting block " << dc.expected_height - << " hash=" << dc.bid << " from " << dc.downloading_client->get_address() - << std::endl; - BinaryArray raw_msg = - LevinProtocol::send_message(NOTIFY_REQUEST_GET_OBJECTS::ID, LevinProtocol::encode(msg), false); - dc.downloading_client->send(std::move(raw_msg)); -} - -void Node::DownloaderV3::stop_download(DownloadCell &dc, bool success) { - if (dc.status != DownloadCell::DOWNLOADING || !dc.downloading_client) - return; - auto git = m_good_clients.find(dc.downloading_client); - invariant(git != m_good_clients.end() && git->second != 0 && total_downloading_blocks != 0, - "DownloadCell reference to good client not found"); - git->second -= 1; - total_downloading_blocks -= 1; - if (success) { - dc.status = DownloadCell::DOWNLOADED; - m_who_downloaded_block.push_back(dc.downloading_client); - } - dc.downloading_client = nullptr; -} - -/*void Node::DownloaderV3::on_msg_notify_request_objects(P2PProtocolBytecoin *who, - const NOTIFY_RESPONSE_GET_OBJECTS::request &req) { - for (auto &&rb : req.blocks) { - Hash bid; - try { - BlockTemplate bheader; - seria::from_binary(bheader, rb.block); - bid = bytecoin::get_block_hash(bheader); - } catch (const std::exception &ex) { - m_node->m_log(logging::INFO) << "Exception " << common::what(ex) << " while parsing returned block, banning -" - << who->get_address() << std::endl; - who->disconnect(std::string()); - break; - } - bool cell_found = false; - for (auto &&dc : m_download_chain) { - if (dc.status != DownloadCell::DOWNLOADING || dc.downloading_client != who || dc.bid != bid) - continue; // downloaded or downloading - stop_download(dc, true); - dc.rb.block = rb.block; // TODO - std::move - dc.rb.transactions = rb.transactions; // TODO - std::move - auto now = std::chrono::steady_clock::now(); - if (std::chrono::duration_cast(now - log_response_timestamp).count() > 1000) { - log_response_timestamp = now; - std::cout << "Received block with height=" << dc.expected_height - << " (queue=" << total_downloading_blocks << ") from " << who->get_address() << std::endl; - } - m_node->m_log(logging::TRACE) - << "Downloader received block with height=" << dc.expected_height << " hash=" << dc.bid - << " (queue=" << total_downloading_blocks << ") from " << who->get_address() << std::endl; - cell_found = true; - if (multicore) { - dc.status = DownloadCell::PREPARING; - add_work(std::tuple(dc.bid, - !m_node->m_block_chain.get_currency().is_in_sw_checkpoint_zone(dc.expected_height), - std::move(dc.rb))); - } else { - dc.pb = PreparedBlock(std::move(dc.rb), m_node->m_block_chain.get_currency(), nullptr); - dc.status = DownloadCell::PREPARED; - } - break; - } - if (!cell_found) { - m_node->m_log(logging::INFO) << "Downloader received stray block from " << who->get_address() << std::endl; - // who->disconnect(std::string()); - // break; - } - } - for (auto &&bid : req.missed_ids) { - for (size_t dit_counter = 0; dit_counter != m_download_chain.size(); ++dit_counter) { - auto & dit = m_download_chain.at(dit_counter); - if (dit.status != DownloadCell::DOWNLOADING || dit.downloading_client != who || dit.bid != bid) - continue; // downloaded or downloading - stop_download(dit, false); - if (!m_chain_client || m_chain_client == who) { - m_node->m_log(logging::INFO) - << "Downloader cannot download block from any connected client, cleaning chain" << std::endl; - while (m_download_chain.size() > dit_counter) { - stop_download(m_download_chain.back(), false); - m_download_chain.pop_back(); - } - m_chain.clear(); - advance_download(); - return; - } - start_download(dit, m_chain_client); - } - } - advance_download(); -}*/ - -bool Node::DownloaderV3::on_idle() { - int added_counter = 0; - if (multicore) { - std::unique_lock lock(mu); - for (auto &&pb : prepared_blocks) { - for (auto &&dc : m_download_chain) - if (dc.status == DownloadCell::PREPARING && dc.bid == pb.first) { - dc.pb = std::move(pb.second); - dc.status = DownloadCell::PREPARED; - break; - } - } - prepared_blocks.clear(); - } - auto idea_start = std::chrono::high_resolution_clock::now(); - while (!m_download_chain.empty() && m_download_chain.front().status == DownloadCell::PREPARED) { - DownloadCell dc = std::move(m_download_chain.front()); - m_download_chain.pop_front(); - api::BlockHeader info; - auto action = m_block_chain.add_block( - dc.pb, &info, common::ip_address_and_port_to_string(dc.block_source.ip, dc.block_source.port)); - if (action == BroadcastAction::BAN) { - m_node->m_log(logging::INFO) << "Downloader DownloadCell BAN height=" << dc.expected_height - << " wb=" << dc.bid << std::endl; - // TODO - ban client who gave us chain - // continue; - } - // if (action == BroadcastAction::NOTHING) - // std::cout << "BroadcastAction::NOTHING height=" << info.height << " cd=" << - // info.cumulative_difficulty.lo - // << std::endl; - if (action == BroadcastAction::BROADCAST_ALL) { - // std::cout << "BroadcastAction::BROADCAST_ALL height=" << info.height - // << " cd=" << info.cumulative_difficulty.lo << std::endl; - if (m_download_chain.empty()) { - // We do not want to broadcast too often during download - m_node->m_log(logging::INFO) << "Added last (from batch) downloaded block height=" << info.height - << " bid=" << info.hash << std::endl; - COMMAND_TIMED_SYNC::request req; - req.payload_data = - CORE_SYNC_DATA{m_node->m_block_chain.get_tip_height(), m_node->m_block_chain.get_tip_bid()}; - BinaryArray raw_msg = - LevinProtocol::send_message(COMMAND_TIMED_SYNC::ID, LevinProtocol::encode(req), true); - m_node->broadcast( - nullptr, raw_msg); // nullptr - we can not always know which connection was block source - // m_node->broadcast_new(nullptr, raw_msg); // TODO nullptr - we can not always know which - // connection was block source - } - } - added_counter += 1; - auto idea_ms = std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - idea_start); - if (idea_ms.count() > 100) - break; - } - if (added_counter) { - m_node->advance_long_poll(); - advance_download(); - if (m_download_chain.empty()) - for (auto &&who : m_good_clients) { - if (who.first->get_other_top_block_desc().height == m_node->m_block_chain.get_tip_height()) { - m_node->m_log(logging::TRACE) - << "DownloaderV3::on_idle sync_transactions to " << who.first->get_address() - << " our pool size=" << m_node->m_block_chain.get_memory_state_transactions().size() - << std::endl; - m_node->sync_transactions(who.first); - break; // TODO - sync with all nodes - } - } - } - - return !m_download_chain.empty() && m_download_chain.front().status == DownloadCell::PREPARED; -} - -void Node::DownloaderV3::on_download_timer() { - m_download_timer.once(SYNC_TIMEOUT / 8); // just several ticks per SYNC_TIMEOUT - auto idea_now = std::chrono::steady_clock::now(); - if (!m_download_chain.empty() && m_download_chain.front().status == DownloadCell::DOWNLOADING && - m_download_chain.front().downloading_client && m_download_chain.front().protect_from_disconnect && - std::chrono::duration_cast(idea_now - m_download_chain.front().request_time).count() > - SYNC_TIMEOUT) { - auto who = m_download_chain.front().downloading_client; - m_node->m_peer_db.delay_connection_attempt(who->get_address(), m_node->m_p2p.get_local_time()); - m_node->m_log(logging::INFO) << "Downloader disconnecting protected slacker " << who->get_address() - << std::endl; - who->disconnect(std::string()); - } -} - -void Node::DownloaderV3::advance_download() { - /* if (m_node->m_block_chain_reader1 || m_node->m_block_chain_reader2 || - m_block_chain.get_tip_height() < m_block_chain.internal_import_known_height()) - return; - const size_t TOTAL_DOWNLOAD_BLOCKS = 400; // TODO - dynamic count - const size_t TOTAL_DOWNLOAD_WINDOW = 2000; // TODO - dynamic count - while (m_download_chain.size() < TOTAL_DOWNLOAD_WINDOW && !m_chain.empty()) { - m_download_chain.push_back(DownloadCell()); - m_download_chain.back().bid = m_chain.front(); - m_download_chain.back().expected_height = m_chain_start_height; - m_download_chain.back().bid_source = chain_source; - m_chain.pop_front(); - m_chain_start_height += 1; - } - advance_chain(); - - while (m_who_downloaded_block.size() > TOTAL_DOWNLOAD_BLOCKS) - m_who_downloaded_block.pop_front(); - std::map who_downloaded_counter; - for (auto lit = m_who_downloaded_block.begin(); lit != m_who_downloaded_block.end(); ++lit) - who_downloaded_counter[*lit] += 1; - auto idea_now = std::chrono::steady_clock::now(); - for (size_t dit_counter = 0; dit_counter != m_download_chain.size(); ++dit_counter) { - auto & dit = m_download_chain.at(dit_counter); - if (dit.status != DownloadCell::DOWNLOADING || dit.downloading_client) - continue; // downloaded or downloading - if (total_downloading_blocks >= TOTAL_DOWNLOAD_BLOCKS) - break; - P2PProtocolBytecoin *ready_client = nullptr; - size_t ready_counter = std::numeric_limits::max(); - size_t ready_speed = 1; - for (auto &&who : m_good_clients) { - size_t speed = - std::max(1, std::min(TOTAL_DOWNLOAD_BLOCKS / 4, - who_downloaded_counter[who.first])); - // We clamp speed so that if even 1 downloaded all blocks, we will give - // small % of blocks to other peers - if (who.second * ready_speed < ready_counter * speed && - who.first->get_last_received_sync_data().current_height >= dit.expected_height) { - ready_client = who.first; - ready_counter = who.second; - ready_speed = speed; - } - } - if (!ready_client && m_chain_client) - ready_client = m_chain_client; - if (!ready_client){ // Cannot download chain from any client - m_node->m_log(logging::INFO) << "DownloaderV3::advance_download cannot download blocks from any - connected client, cleaning chain" << std::endl; - while (m_download_chain.size() > dit_counter) { - stop_download(m_download_chain.back(), false); - m_download_chain.pop_back(); - } - m_chain.clear(); - advance_chain(); - return; - } - start_download(dit, ready_client); - } - const bool bad_timeout = - !m_download_chain.empty() && m_download_chain.front().status == DownloadCell::DOWNLOADING && - m_download_chain.front().downloading_client && !m_download_chain.front().protect_from_disconnect && - std::chrono::duration_cast(idea_now - m_download_chain.front().request_time).count() > - 2 * SYNC_TIMEOUT; - const bool bad_relatively_slow = - total_downloading_blocks < TOTAL_DOWNLOAD_BLOCKS && m_download_chain.size() >= TOTAL_DOWNLOAD_WINDOW && - m_good_clients.size() > 1 && m_download_chain.front().status == DownloadCell::DOWNLOADING && - m_download_chain.front().downloading_client && !m_download_chain.front().protect_from_disconnect; - if (bad_relatively_slow || bad_timeout) { - auto who = m_download_chain.front().downloading_client; - for (auto &&dc : m_download_chain) - if (dc.downloading_client == who) - dc.protect_from_disconnect = true; - m_node->m_peer_db.delay_connection_attempt(who->get_address(), m_node->m_p2p.get_local_time()); - m_node->m_log(logging::INFO) << "DownloaderV3::advance_download disconnecting slacker " << - who->get_address() - << std::endl; - who->disconnect(std::string()); - }*/ -} diff --git a/src/Core/NodeLegacyAPI.cpp b/src/Core/NodeLegacyAPI.cpp index 38b05f1b..c3677879 100644 --- a/src/Core/NodeLegacyAPI.cpp +++ b/src/Core/NodeLegacyAPI.cpp @@ -14,9 +14,9 @@ #include "seria/KVBinaryInputStream.hpp" #include "seria/KVBinaryOutputStream.hpp" -using namespace bytecoin; +using namespace cn; -bool Node::on_json_rpc(http::Client *who, http::RequestData &&request, http::ResponseData &response) { +bool Node::on_json_rpc(http::Client *who, http::RequestBody &&request, http::ResponseBody &response) { response.r.headers.push_back({"Content-Type", "application/json; charset=utf-8"}); common::JsonValue jid(nullptr); @@ -30,19 +30,14 @@ bool Node::on_json_rpc(http::Client *who, http::RequestData &&request, http::Res m_log(logging::INFO) << "jsonrpc request method not found - " << json_req.get_method() << std::endl; if (WalletNode::m_jsonrpc_handlers.count(json_req.get_method()) != 0) throw json_rpc::Error(json_rpc::METHOD_NOT_FOUND, - "Method not found " + json_req.get_method() + " (attempt to call walletd method on bytecoind)"); + "Method not found " + json_req.get_method() + + " (attempt to call walletd method on " CRYPTONOTE_NAME "d)"); throw json_rpc::Error(json_rpc::METHOD_NOT_FOUND, "Method not found " + json_req.get_method()); } - // m_log(logging::INFO) << "jsonrpc request method=" << - // json_req.get_method() << std::endl; std::string response_body; if (!it->second(this, who, std::move(request), std::move(json_req), response_body)) return false; response.set_body(std::move(response_body)); - // } catch (const api::bytecoind::SendTransaction::Error &err) { - // response.set_body(json_rpc::create_error_response_body(err, jid)); - // } catch (const api::bytecoind::GetArchive::Error &err) { - // response.set_body(json_rpc::create_error_response_body(err, jid)); } catch (const json_rpc::Error &err) { response.set_body(json_rpc::create_error_response_body(err, jid)); } catch (const std::exception &e) { @@ -53,7 +48,7 @@ bool Node::on_json_rpc(http::Client *who, http::RequestData &&request, http::Res return true; } -bool Node::on_binary_rpc(http::Client *who, http::RequestData &&request, http::ResponseData &response) { +bool Node::on_binary_rpc(http::Client *who, http::RequestBody &&request, http::ResponseBody &response) { response.r.headers.push_back({"Content-Type", "application/octet-stream"}); common::JsonValue jid(nullptr); @@ -71,16 +66,10 @@ bool Node::on_binary_rpc(http::Client *who, http::RequestData &&request, http::R m_log(logging::INFO) << "binaryrpc request method not found - " << binary_req.get_method() << std::endl; throw json_rpc::Error(json_rpc::METHOD_NOT_FOUND, "Method not found " + binary_req.get_method()); } - // m_log(logging::INFO) << "jsonrpc request method=" << - // json_req.get_method() << std::endl; std::string response_body; if (!it->second(this, who, body_stream, std::move(binary_req), response_body)) return false; response.set_body(std::move(response_body)); - // } catch (const api::bytecoind::SendTransaction::Error &err) { - // response.set_body(json_rpc::create_binary_response_error_body(err, jid)); - // } catch (const api::bytecoind::GetArchive::Error &err) { - // response.set_body(json_rpc::create_binary_response_error_body(err, jid)); } catch (const json_rpc::Error &err) { response.set_body(json_rpc::create_binary_response_error_body(err, jid)); } catch (const std::exception &e) { @@ -91,9 +80,9 @@ bool Node::on_binary_rpc(http::Client *who, http::RequestData &&request, http::R return true; } -bool Node::on_getblocktemplate(http::Client *who, http::RequestData &&raw_request, json_rpc::Request &&raw_js_request, - api::bytecoind::GetBlockTemplate::Request &&req, api::bytecoind::GetBlockTemplate::Response &res) { - api::bytecoind::GetStatus::Request sta; +bool Node::on_getblocktemplate(http::Client *who, http::RequestBody &&raw_request, json_rpc::Request &&raw_js_request, + api::cnd::GetBlockTemplate::Request &&req, api::cnd::GetBlockTemplate::Response &res) { + api::cnd::GetStatus::Request sta; sta.top_block_hash = req.top_block_hash; sta.transaction_pool_version = req.transaction_pool_version; m_log(logging::INFO) << "Node received getblocktemplate REQ transaction_pool_version=" @@ -107,10 +96,6 @@ bool Node::on_getblocktemplate(http::Client *who, http::RequestData &&raw_reques if ((!sta.top_block_hash || sta.top_block_hash.get() == m_block_chain.get_tip_bid()) && (!sta.transaction_pool_version || sta.transaction_pool_version.get() == m_block_chain.get_tx_pool_version()) && (sta.top_block_hash || sta.transaction_pool_version)) { - // m_log(logging::INFO) << "on_getblocktemplate will long poll, - // json=" - //<< - // raw_request.body << std::endl; LongPollClient lpc; lpc.original_who = who; lpc.original_request = raw_request; @@ -123,110 +108,105 @@ bool Node::on_getblocktemplate(http::Client *who, http::RequestData &&raw_reques return true; } -void Node::getblocktemplate(const api::bytecoind::GetBlockTemplate::Request &req, - api::bytecoind::GetBlockTemplate::Response &res) { +void Node::getblocktemplate(const api::cnd::GetBlockTemplate::Request &req, api::cnd::GetBlockTemplate::Response &res) { if (req.reserve_size > TransactionExtraNonce::MAX_COUNT) - throw json_rpc::Error{api::bytecoind::GetBlockTemplate::TOO_BIG_RESERVE_SIZE, + throw json_rpc::Error{api::cnd::GetBlockTemplate::TOO_BIG_RESERVE_SIZE, "To big reserved size, maximum " + common::to_string(TransactionExtraNonce::MAX_COUNT)}; - AccountPublicAddress acc{}; + AccountAddress acc{}; if (!m_block_chain.get_currency().parse_account_address_string(req.wallet_address, &acc)) throw api::ErrorAddress( api::ErrorAddress::ADDRESS_FAILED_TO_PARSE, "Failed to parse wallet address", req.wallet_address); BlockTemplate block_template{}; BinaryArray blob_reserve; - blob_reserve.resize(req.reserve_size, 0); + uint8_t reserve_magic = 0xbb; + blob_reserve.resize(req.reserve_size, reserve_magic); + size_t reserve_back_offset = 0; try { - m_block_chain.create_mining_block_template(acc, blob_reserve, &block_template, &res.difficulty, &res.height); + m_block_chain.create_mining_block_template(m_block_chain.get_tip_bid(), acc, blob_reserve, &block_template, + &res.difficulty, &res.height, &reserve_back_offset); } catch (const std::exception &ex) { m_log(logging::ERROR) << logging::BrightRed << "getblocktemplate exception " << ex.what() << std::endl; throw; } BinaryArray block_blob = seria::to_binary(block_template); - PublicKey tx_pub_key = extra_get_transaction_public_key(block_template.base_transaction.extra); - if (tx_pub_key == PublicKey{}) { - m_log(logging::ERROR) << "Failed to find tx pub key in coinbase extra"; - throw json_rpc::Error{json_rpc::INTERNAL_ERROR, "Internal error: failed to find tx pub key in coinbase extra"}; - } - if (req.reserve_size > 0) { - const unsigned char *found = - common::slow_memmem(block_blob.data(), block_blob.size(), tx_pub_key.data, sizeof(tx_pub_key)); - if (!found) { - m_log(logging::ERROR) << "Failed to find tx pub key in blockblob"; - throw json_rpc::Error{json_rpc::INTERNAL_ERROR, "Internal error: failed to create block template"}; - } - res.reserved_offset = static_cast(found - block_blob.data()); - res.reserved_offset += - sizeof(tx_pub_key) + 2; // tag for TX_EXTRA_NONCE(1 byte), counter in TX_EXTRA_NONCE(1 byte) - if (res.reserved_offset + req.reserve_size > block_blob.size()) { + if (reserve_back_offset + blob_reserve.size() > block_blob.size()) { m_log(logging::ERROR) << "Failed to calculate offset for reserved bytes"; throw json_rpc::Error{json_rpc::INTERNAL_ERROR, "Internal error: failed to create block template"}; } - for (size_t i = 0; i != req.reserve_size; ++i) - invariant(block_blob.at(res.reserved_offset + i) == 00, ""); + res.reserved_offset = block_blob.size() - reserve_back_offset - blob_reserve.size(); + for (size_t i = 0; i != req.reserve_size; ++i) { + invariant(block_blob.at(res.reserved_offset + i) == reserve_magic, ""); + block_blob.at(res.reserved_offset + i) = 0; + } } res.blocktemplate_blob = block_blob; res.top_block_hash = m_block_chain.get_tip_bid(); res.transaction_pool_version = m_block_chain.get_tx_pool_version(); res.previous_block_hash = m_block_chain.get_tip().previous_block_hash; + // Experimental, a bit hacky + if (block_template.major_version >= m_block_chain.get_currency().amethyst_block_version) { + try { + block_template.major_version += 1; + auto body_proxy = get_body_proxy_from_template(block_template); + res.cm_prehash = get_auxiliary_block_header_hash(block_template, body_proxy); + res.cm_path = m_block_chain.get_genesis_bid(); + } catch (const std::exception &) { + } + } } -bool Node::on_get_currency_id(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetCurrencyId::Request && /*req*/, api::bytecoind::GetCurrencyId::Response &res) { +bool Node::on_get_currency_id(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetCurrencyId::Request &&, api::cnd::GetCurrencyId::Response &res) { res.currency_id_blob = m_block_chain.get_genesis_bid(); return true; } -bool Node::on_submitblock_legacy(http::Client *who, http::RequestData &&rd, json_rpc::Request &&jr, - api::bytecoind::SubmitBlockLegacy::Request &&req, api::bytecoind::SubmitBlockLegacy::Response &res) { +bool Node::on_submitblock_legacy(http::Client *who, http::RequestBody &&rd, json_rpc::Request &&jr, + api::cnd::SubmitBlockLegacy::Request &&req, api::cnd::SubmitBlockLegacy::Response &res) { if (req.size() != 1) throw json_rpc::Error{json_rpc::INVALID_PARAMS, "Request params should be an array with exactly 1 element"}; BinaryArray blocktemplate_blob; - if (!common::from_hex(req[0], blocktemplate_blob)) { - throw json_rpc::Error{api::bytecoind::SubmitBlock::WRONG_BLOCKBLOB, "blocktemplate_blob should be in hex"}; + if (!common::from_hex(req[0], &blocktemplate_blob)) { + throw json_rpc::Error{api::cnd::SubmitBlock::WRONG_BLOCKBLOB, "blocktemplate_blob should be in hex"}; } api::BlockHeader info; submit_block(blocktemplate_blob, &info); return true; } -bool Node::on_get_last_block_header(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetLastBlockHeaderLegacy::Request &&, - api::bytecoind::GetLastBlockHeaderLegacy::Response &response) { +bool Node::on_get_last_block_header(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetLastBlockHeaderLegacy::Request &&, api::cnd::GetLastBlockHeaderLegacy::Response &response) { static_cast(response.block_header) = m_block_chain.get_tip(); - m_block_chain.fix_block_sizes(&response.block_header); - response.block_header.orphan_status = false; + response.block_header.orphan_status = false; response.block_header.depth = api::HeightOrDepth(m_block_chain.get_tip_height() - response.block_header.height); return true; } -bool Node::on_get_block_header_by_hash(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetBlockHeaderByHashLegacy::Request &&request, - api::bytecoind::GetBlockHeaderByHashLegacy::Response &response) { - if (!m_block_chain.read_header(request.hash, &response.block_header)) +bool Node::on_get_block_header_by_hash(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetBlockHeaderByHashLegacy::Request &&request, api::cnd::GetBlockHeaderByHashLegacy::Response &response) { + if (!m_block_chain.get_header(request.hash, &response.block_header)) throw api::ErrorHashNotFound("Block is neither in main nor in any side chain", request.hash); - m_block_chain.fix_block_sizes(&response.block_header); response.block_header.orphan_status = !m_block_chain.in_chain(response.block_header.height, response.block_header.hash); response.block_header.depth = api::HeightOrDepth(m_block_chain.get_tip_height() - response.block_header.height); return true; } -bool Node::on_get_block_header_by_height(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::GetBlockHeaderByHeightLegacy::Request &&request, - api::bytecoind::GetBlockHeaderByHeightLegacy::Response &response) { +bool Node::on_get_block_header_by_height(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::GetBlockHeaderByHeightLegacy::Request &&request, + api::cnd::GetBlockHeaderByHeightLegacy::Response &response) { Hash block_hash; // Freaking legacy, this call request counts blocks from 1, response counts from 0 - if (request.height == 0 || !m_block_chain.read_chain(request.height - 1, &block_hash)) { + if (request.height == 0 || !m_block_chain.get_chain(request.height - 1, &block_hash)) { throw api::ErrorWrongHeight( "Too big height. Note, this method request counts blocks from 1, not 0 as all other methods", request.height - 1, m_block_chain.get_tip_height()); } - invariant(m_block_chain.read_header(block_hash, &response.block_header), ""); - m_block_chain.fix_block_sizes(&response.block_header); + invariant(m_block_chain.get_header(block_hash, &response.block_header), ""); response.block_header.orphan_status = false; response.block_header.depth = api::HeightOrDepth(m_block_chain.get_tip_height() - response.block_header.height); return true; diff --git a/src/Core/Node_P2PProtocolBytecoin.cpp b/src/Core/Node_P2PProtocolBytecoin.cpp index b8b79340..53953a77 100644 --- a/src/Core/Node_P2PProtocolBytecoin.cpp +++ b/src/Core/Node_P2PProtocolBytecoin.cpp @@ -15,7 +15,15 @@ #include "seria/KVBinaryOutputStream.hpp" #include "version.hpp" -using namespace bytecoin; +using namespace cn; + +Node::P2PProtocolBytecoin::P2PProtocolBytecoin(Node *node, P2PClient *client) + : P2PProtocolBasic(node->m_config, node->m_p2p.get_unique_number(), client) + , m_node(node) + , m_chain_timer(std::bind(&P2PProtocolBytecoin::on_chain_timer, this)) + , m_download_timer(std::bind(&P2PProtocolBytecoin::on_download_timer, this)) + , m_syncpool_timer(std::bind(&P2PProtocolBytecoin::on_syncpool_timer, this)) + , m_download_transactions_timer(std::bind(&P2PProtocolBytecoin::on_download_transactions_timer, this)) {} Node::P2PProtocolBytecoin::~P2PProtocolBytecoin() { // std::cout << "~P2PProtocolBytecoin this=" << std::hex << (size_t)this << std::dec << std::endl; @@ -26,118 +34,571 @@ void Node::P2PProtocolBytecoin::on_msg_bytes(size_t, size_t) { // downloaded. u // node->p2p.get_local_time()); } -CORE_SYNC_DATA -Node::P2PProtocolBytecoin::get_sync_data() const { - CORE_SYNC_DATA sync_data; +CoreSyncData Node::P2PProtocolBytecoin::get_my_sync_data() const { + CoreSyncData sync_data; sync_data.current_height = m_node->m_block_chain.get_tip_height(); sync_data.top_id = m_node->m_block_chain.get_tip_bid(); return sync_data; } -std::vector Node::P2PProtocolBytecoin::get_peers_to_share() const { - auto result = m_node->m_peer_db.get_peerlist_to_p2p_legacy( - get_address(), m_node->m_p2p.get_local_time(), config.p2p_default_peers_in_handshake); +std::vector Node::P2PProtocolBytecoin::get_peers_to_share(bool lots) const { + auto result = m_node->m_peer_db.get_peerlist_to_p2p_legacy(get_address(), + m_node->m_p2p.get_local_time(), + lots ? config.p2p_default_peers_in_handshake : config.p2p_default_peers_in_handshake / 10); return result; } void Node::P2PProtocolBytecoin::on_first_message_after_handshake() { // if we set just seen on handshake, we will keep connecting to seed nodes // forever - m_node->m_peer_db.set_peer_just_seen( - get_last_received_unique_number(), get_address(), m_node->m_p2p.get_local_time()); + m_node->m_peer_db.set_peer_just_seen(get_peer_unique_number(), get_address(), m_node->m_p2p.get_local_time()); } -void Node::P2PProtocolBytecoin::on_immediate_protocol_switch(unsigned char first_byte) { -#if bytecoin_NEWP2P - // We ignore first_byte for now, because we have only 1 new protocol - get_client()->set_protocol(std::make_unique(m_node, get_client())); -#endif +void Node::P2PProtocolBytecoin::on_chain_timer() { + invariant(m_chain_request_sent, ""); + m_node->m_log(logging::TRACE) << "on_chain_timer, disconnecting " << get_address() << std::endl; + disconnect(std::string()); +} + +void Node::P2PProtocolBytecoin::on_download_timer() { + invariant(m_downloading_block_count != 0, ""); + m_node->m_log(logging::TRACE) << "on_download_timer, disconnecting " << get_address() << std::endl; + disconnect(std::string()); +} + +void Node::P2PProtocolBytecoin::on_syncpool_timer() { + invariant(m_syncpool_equest_sent, ""); + m_node->m_log(logging::TRACE) << "on_download_transactions_timer, disconnecting " << get_address() << std::endl; + disconnect(std::string()); +} +void Node::P2PProtocolBytecoin::on_download_transactions_timer() { + invariant(m_downloading_transaction_count != 0, ""); + m_node->m_log(logging::TRACE) << "on_download_transactions_timer, disconnecting " << get_address() << std::endl; + disconnect(std::string()); +} + +void Node::P2PProtocolBytecoin::advance_chain() { + if (is_incoming() || !m_chain.empty() || m_chain_request_sent) + return; + api::BlockHeader info; + if (m_node->m_block_chain.get_header(get_peer_sync_data().top_id, &info)) { + if (info.height + m_node->m_config.p2p_outgoing_peer_max_lag < m_node->m_block_chain.get_tip_height()) { + m_node->m_log(logging::INFO) << "Disconnecting and delay connecting lagging client " << get_address() + << std::endl; + const auto now = m_node->m_p2p.get_local_time(); + m_node->m_peer_db.delay_connection_attempt(get_address(), now); + disconnect(std::string()); + } + // We have peer's top block in our blockchain, nop. + return; + } + m_chain_request_sent = true; + p2p::GetChainRequest::Notify msg; + if (m_previous_chain_hash != Hash{}) { + // TODO - turn m_next_chain_hash into sparse chain + msg.block_ids.push_back(m_previous_chain_hash); + msg.block_ids.push_back(m_node->m_block_chain.get_genesis_bid()); + } else { + msg.block_ids = m_node->m_block_chain.get_sparse_chain(p2p::GetChainResponse::Notify::MAX_BLOCK_IDS * 4 / 5); + // Fix for very large blockchain with last checkpoint far in the past + if (msg.block_ids.size() > p2p::GetChainRequest::Notify::MAX_BLOCK_IDS) + msg.block_ids.erase( + msg.block_ids.begin() + p2p::GetChainRequest::Notify::MAX_BLOCK_IDS - 1, msg.block_ids.end() - 1); + } + + m_chain_timer.once(m_node->m_config.download_chain_timeout); + m_node->m_log(logging::INFO) << "advance_chain Requesting chain from " << get_address() + << " remote height=" << get_peer_sync_data().current_height + << " our height=" << m_node->m_block_chain.get_tip_height() << std::endl; + BinaryArray raw_msg = LevinProtocol::send(msg); + send(std::move(raw_msg)); +} + +void Node::P2PProtocolBytecoin::advance_blocks() { + // Remove already added to the block chain + while (!m_chain.empty() && m_node->m_block_chain.has_header(m_chain.front()->first) && + m_chain.front()->second.who_downloading != this) { + m_previous_chain_hash = m_chain.front()->first; + m_chain_start_height += 1; + m_node->remove_chain_block(m_chain.front()); + m_chain.pop_front(); + } + if (m_downloading_block_count == m_node->m_config.max_downloading_blocks_from_each_peer) + return; + size_t we_downloading = 0; + std::vector request_block_ids; + for (size_t i = 0; i < std::min(m_chain.size(), m_node->m_config.download_window) && + we_downloading < m_node->m_config.max_downloading_blocks_from_each_peer; + ++i) { + auto cit = m_chain.at(i); + if (cit->second.who_downloading || cit->second.preparing) { + we_downloading += (cit->second.who_downloading == this) ? 1 : 0; + continue; + } + we_downloading += 1; + cit->second.who_downloading = this; + cit->second.expected_height = static_cast(m_chain_start_height + i); + m_downloading_block_count += 1; + request_block_ids.push_back(cit->first); + const auto now = std::chrono::steady_clock::now(); + if (std::chrono::duration_cast(now - m_node->log_request_timestamp).count() > 1000) { + m_node->log_request_timestamp = now; + std::cout << "Requesting block " << m_chain_start_height + i << " from " << get_address() << std::endl; + } + m_node->m_log(logging::TRACE) << "advance_download requesting block " << m_chain_start_height + i + << " hash=" << cit->first << " from " << get_address() << std::endl; + } + if (!request_block_ids.empty()) + m_download_timer.once(m_node->m_config.download_block_timeout); + for (const auto &bid : request_block_ids) { + p2p::GetObjectsRequest::Notify msg; + msg.blocks.push_back(bid); + send(LevinProtocol::send(msg)); + } +} + +void Node::P2PProtocolBytecoin::advance_transactions() { + if (get_peer_sync_data().top_id != m_node->m_block_chain.get_tip_bid()) + return; + if (get_peer_version() < P2PProtocolVersion::AMETHYST) { // Single sync pool per connection, no timer + if (m_syncpool_equest_sent) + return; // We will never reset m_syncpool_equest_sent on V1 connection + p2p::SyncPool::Notify msg; + auto mytxs = m_node->m_block_chain.get_memory_state_transactions(); + msg.txs.reserve(mytxs.size()); + for (auto &&tx : mytxs) + msg.txs.push_back(tx.first); + m_syncpool_equest_sent = true; + send(LevinProtocol::send(msg)); + return; + } + if (!m_syncpool_equest_sent) { + p2p::SyncPool::Request msg; + msg.from = syncpool_start; + msg.to.first = m_node->m_block_chain.minimum_pool_fee_per_byte(true, &msg.to.second); + if (msg.from <= msg.to) + return; + m_syncpool_equest_sent = true; + m_syncpool_timer.once(m_node->m_config.sync_pool_timeout); + m_node->m_log(logging::TRACE) << "Sending SyncPool to " << get_address() + << " with fee_per_byte=" << msg.from.first << " hash=" << msg.from.second + << std::endl; + send(LevinProtocol::send(msg)); + return; + } +} + +bool Node::P2PProtocolBytecoin::on_transaction_descs(const std::vector &descs) { + const auto &pool = m_node->m_block_chain.get_memory_state_transactions(); + Amount minimum_fee = m_node->m_block_chain.minimum_pool_fee_per_byte(true); + std::vector request_transaction_ids; + Amount previous_fee_per_byte = std::numeric_limits::max(); + Hash previous_hash; + // TODO - check that descs are in limits set at request + for (const auto &desc : descs) { + if (desc.size == 0) { + disconnect("SyncPool desc size == 0"); + return false; + } + Amount fee_per_byte = desc.fee / desc.size; + if (fee_per_byte > previous_fee_per_byte || + (fee_per_byte == previous_fee_per_byte && desc.hash >= previous_hash)) { + disconnect("SyncPool descs not sorted"); + return false; + } + previous_hash = desc.hash; + previous_fee_per_byte = fee_per_byte; + if (fee_per_byte < minimum_fee) + continue; + if (!m_node->m_block_chain.in_chain(desc.newest_referenced_block)) + continue; + if (pool.count(desc.hash) != 0 || m_node->m_block_chain.has_transaction(desc.hash)) + continue; + if (!m_transaction_descs.insert(std::make_pair(desc.hash, desc)).second) + continue; // Already have + if (!m_node->downloading_transactions.insert(std::make_pair(desc.hash, this)).second) + continue; + request_transaction_ids.push_back(desc.hash); + } + if (!request_transaction_ids.empty()) + m_download_transactions_timer.once(m_node->m_config.download_transaction_timeout); + m_downloading_transaction_count += request_transaction_ids.size(); + for (const auto &tid : request_transaction_ids) { + p2p::GetObjectsRequest::Notify msg; + msg.txs.push_back(tid); + send(LevinProtocol::send(msg)); + } + return true; +} + +void Node::P2PProtocolBytecoin::transaction_download_finished(const Hash &tid, bool success) { + auto tit = m_transaction_descs.find(tid); + if (tit == m_transaction_descs.end()) + return; + if (success) { + tit = m_transaction_descs.erase(tit); + return; + } + if (!m_node->downloading_transactions.insert(std::make_pair(tid, this)).second) + return; // Someone already started downloading it + m_downloading_transaction_count += 1; + m_download_transactions_timer.once(m_node->m_config.download_transaction_timeout); + p2p::GetObjectsRequest::Notify msg; + msg.txs.push_back(tid); + send(LevinProtocol::send(msg)); +} + +bool Node::P2PProtocolBytecoin::on_idle(std::chrono::steady_clock::time_point idle_start) { + size_t added_counter = 0; + PreparedBlock pb; + while (!m_chain.empty() && m_node->m_pow_checker.get_prepared_block(m_chain.front()->first, &pb)) { + auto cit = m_chain.front(); + m_node->m_log(logging::TRACE) << "on_idle prepared block " << cit->second.expected_height + << " hash=" << cit->first << " from " << get_address() << std::endl; + invariant(pb.bid == cit->first, ""); + invariant(cit->second.who_downloading == nullptr, ""); + m_previous_chain_hash = cit->first; + m_chain_start_height += 1; + m_chain.pop_front(); + const Height expected_height = cit->second.expected_height; + cit->second.preparing = false; + m_node->remove_chain_block(cit); + + api::BlockHeader info; + bool add_block_result = false; + try { + add_block_result = m_node->m_block_chain.add_block(pb, &info, get_address().to_string()); + } catch (const std::exception &ex) { + auto what = common::what(ex); + m_node->m_log(logging::INFO) << "on_idle add_block BAN expected height=" << expected_height + << " actual height=" << info.height << " wb=" << pb.bid << " what=" << what + << std::endl; + disconnect("on_idle add_block BAN what=" + what); + return false; + } + for (auto who : m_node->m_broadcast_protocols) + who->advance_transactions(); + if (expected_height != info.height) { + m_node->m_log(logging::INFO) << "on_idle add_block lied about height, expected height " << expected_height + << " actual height=" << info.height << " wb=" << pb.bid << std::endl; + } + if (add_block_result) { + if (m_chain.empty() || + m_node->m_block_chain.get_tip_height() % m_node->m_config.download_broadcast_every_n_blocks == 0) { + // We do not want to broadcast too often during download + m_node->m_log(logging::INFO) << "Added last (from batch) downloaded block height=" << info.height + << " bid=" << info.hash << std::endl; + p2p::TimedSync::Request req; + req.payload_data = + CoreSyncData{m_node->m_block_chain.get_tip_height(), m_node->m_block_chain.get_tip_bid()}; + BinaryArray raw_msg = LevinProtocol::send(req); + m_node->broadcast( + nullptr, raw_msg); // nullptr - we can not always know which connection was block source + } + } + added_counter += 1; + auto idea_ms = + std::chrono::duration_cast(std::chrono::steady_clock::now() - idle_start); + if (idea_ms.count() > int(1000 * m_node->m_config.max_on_idle_time)) + break; + } + return !m_chain.empty() && m_node->m_pow_checker.has_prepared_block(m_chain.front()->first); } void Node::P2PProtocolBytecoin::after_handshake() { m_node->m_p2p.peers_updated(); - m_node->broadcast_protocols.insert(this); + m_node->m_broadcast_protocols.insert(this); m_node->advance_long_poll(); auto signed_checkpoints = m_node->m_block_chain.get_latest_checkpoints(); for (const auto &sck : signed_checkpoints) { - BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_CHECKPOINT::ID, LevinProtocol::encode(sck), false); + p2p::Checkpoint::Notify msg(sck); + BinaryArray raw_msg = LevinProtocol::send(msg); send(std::move(raw_msg)); } - m_node->m_downloader.on_connect(this); // Can destroy this... + advance_transactions(); // We can be on the same height already, will sync pools then + advance_chain(); } -void Node::P2PProtocolBytecoin::on_msg_handshake(COMMAND_HANDSHAKE::request &&req) { -#if bytecoin_NEWP2P - if (get_version() == P2PProtocolVersion::V3_NEW) { - get_client()->set_protocol(std::make_unique(m_node, get_client())); - return; - } -#endif +void Node::P2PProtocolBytecoin::on_msg_handshake(p2p::Handshake::Request &&req) { m_node->m_peer_db.add_incoming_peer(get_address(), m_node->m_p2p.get_local_time()); after_handshake(); } -void Node::P2PProtocolBytecoin::on_msg_handshake(COMMAND_HANDSHAKE::response &&req) { -#if bytecoin_NEWP2P - if (get_version() == P2PProtocolVersion::V3_NEW) { - get_client()->set_protocol(std::make_unique(m_node, get_client())); - return; - } -#endif +void Node::P2PProtocolBytecoin::on_msg_handshake(p2p::Handshake::Response &&req) { m_node->m_peer_db.merge_peerlist_from_p2p(get_address(), req.local_peerlist, m_node->m_p2p.get_local_time()); after_handshake(); } -void Node::P2PProtocolBytecoin::on_msg_notify_request_chain(NOTIFY_REQUEST_CHAIN::request &&req) { - NOTIFY_RESPONSE_CHAIN_ENTRY::request msg; +void Node::P2PProtocolBytecoin::on_msg_notify_request_chain(p2p::GetChainRequest::Notify &&req) { + if (get_peer_version() >= P2PProtocolVersion::AMETHYST && + req.block_ids.size() > p2p::GetChainRequest::Notify::MAX_BLOCK_IDS) + return disconnect("GetChainRequest too many block_ids"); + p2p::GetChainResponse::Notify msg; msg.m_block_ids = m_node->m_block_chain.get_sync_headers_chain( - req.block_ids, &msg.start_height, config.p2p_block_ids_sync_default_count); - msg.total_height = m_node->m_block_chain.get_tip_height() + 1; + req.block_ids, &msg.start_height, p2p::GetChainResponse::Notify::MAX_BLOCK_IDS); + msg.total_height = m_node->m_block_chain.get_tip_height(); - BinaryArray raw_msg = - LevinProtocol::send_message(NOTIFY_RESPONSE_CHAIN_ENTRY::ID, LevinProtocol::encode(msg), false); + BinaryArray raw_msg = LevinProtocol::send(msg); send(std::move(raw_msg)); } -void Node::P2PProtocolBytecoin::on_msg_notify_request_chain(NOTIFY_RESPONSE_CHAIN_ENTRY::request &&req) { - m_node->m_downloader.on_msg_notify_request_chain(this, req); +void Node::P2PProtocolBytecoin::on_msg_notify_request_chain(p2p::GetChainResponse::Notify &&req) { + if (get_peer_version() >= P2PProtocolVersion::AMETHYST) { + if (req.m_block_ids.size() > p2p::GetChainResponse::Notify::MAX_BLOCK_IDS) + return disconnect("GetChainResponse too many block_ids"); + if (!m_chain_request_sent) + return disconnect("GetChainResponse stray chain"); + } + if (req.m_block_ids.empty()) + return disconnect("GetChainResponse chain"); + if (!m_chain_request_sent) + return; + invariant(m_chain.empty(), ""); + m_chain_request_sent = false; + m_chain_timer.cancel(); + m_node->m_log(logging::INFO) << "received chain from " << get_address() << " start_height=" << req.start_height + << " length=" << req.m_block_ids.size() << std::endl; + api::BlockHeader info; + if (!m_node->m_block_chain.get_header(req.m_block_ids.front(), &info) || info.height != req.start_height) + return disconnect("Chain does not start with hash we have or wrong start height"); + m_chain_start_height = req.start_height; + // TODO - prevent wrong order + for (const auto &bid : req.m_block_ids) { + if (m_node->m_block_chain.has_header(bid)) { + m_chain_start_height += 1; + m_previous_chain_hash = bid; + continue; + } + m_chain.push_back(m_node->chain_blocks.insert(std::make_pair(bid, DownloadInfo{})).first); + m_chain.back()->second.chain_counter += 1; + } + if (req.m_block_ids.size() != m_chain.size() + 1) { + m_node->m_log(logging::INFO) << "truncated chain length=" << m_chain.size() << std::endl; + } + if (m_chain.empty()) { + // TODO - add delay to advance_chain + // const auto now = m_node->m_p2p.get_local_time(); + m_node->m_log(logging::INFO) << "truncated chain to zero from" << get_address() << std::endl; + advance_chain(); + return; + } + advance_blocks(); } -void Node::P2PProtocolBytecoin::on_msg_notify_request_objects(NOTIFY_REQUEST_GET_OBJECTS::request &&req) { - NOTIFY_RESPONSE_GET_OBJECTS::request msg; - msg.current_blockchain_height = m_node->m_block_chain.get_tip_height() + 1; - for (auto &&bh : req.blocks) { +void Node::P2PProtocolBytecoin::on_msg_notify_request_objects(p2p::GetObjectsRequest::Notify &&req) { + if (get_peer_version() >= P2PProtocolVersion::AMETHYST && !req.blocks.empty() && !req.txs.empty()) + return disconnect("Both blocks and txs in GetObjectsRequest"); + if (get_peer_version() >= P2PProtocolVersion::AMETHYST && req.blocks.empty() && req.txs.empty()) + return disconnect("No blocks or txs in GetObjectsRequest"); + if (get_peer_version() >= P2PProtocolVersion::AMETHYST && req.blocks.size() > 1) + return disconnect("Too much blocks in GetObjectsRequest"); + if (get_peer_version() >= P2PProtocolVersion::AMETHYST && req.txs.size() > 1) + return disconnect("Too much transactions in GetObjectsRequest"); + p2p::GetObjectsResponse::Notify msg; + msg.current_blockchain_height = m_node->m_block_chain.get_tip_height(); + for (const auto &bid : req.blocks) { RawBlock raw_block; - if (m_node->m_block_chain.read_block(bh, &raw_block)) { - msg.blocks.push_back(RawBlockLegacy{raw_block.block, raw_block.transactions}); - } else - msg.missed_ids.push_back(bh); + if (m_node->m_block_chain.get_block(bid, &raw_block)) { + msg.blocks.push_back(std::move(raw_block)); + continue; + } + msg.missed_ids.push_back(bid); } - if (!req.txs.empty()) { - // TODO - remove after we are sure transactions are never asked - throw std::runtime_error( - "Transactions asked in NOTIFY_REQUEST_GET_OBJECTS by " + common::ip_address_to_string(get_address().ip)); + for (const auto &tid : req.txs) { + const auto &pool = m_node->m_block_chain.get_memory_state_transactions(); + auto tit = pool.find(tid); + if (tit != pool.end()) { + msg.txs.push_back(tit->second.binary_tx); + continue; + } + BinaryArray binary_tx; + size_t index_in_block = 0; + Height block_height = 0; + Hash block_hash; + if (m_node->m_block_chain.get_transaction(tid, &binary_tx, &block_height, &block_hash, &index_in_block)) { + msg.txs.push_back(std::move(binary_tx)); + continue; + } + msg.missed_ids.push_back(tid); } - BinaryArray raw_msg = - LevinProtocol::send_message(NOTIFY_RESPONSE_GET_OBJECTS::ID, LevinProtocol::encode(msg), false); - send(std::move(raw_msg)); + send(LevinProtocol::send(msg)); } -void Node::P2PProtocolBytecoin::on_msg_notify_request_objects(NOTIFY_RESPONSE_GET_OBJECTS::request &&req) { - m_node->m_downloader.on_msg_notify_request_objects(this, req); +void Node::P2PProtocolBytecoin::on_msg_notify_request_objects(p2p::GetObjectsResponse::Notify &&req) { + if (get_peer_version() >= P2PProtocolVersion::AMETHYST && req.blocks.size() > 1) + return disconnect("Too much blocks in GetObjectsResponse"); + if (get_peer_version() >= P2PProtocolVersion::AMETHYST && req.txs.size() > 1) + return disconnect("Too much transactions in GetObjectsResponse"); + for (auto &&rb : req.blocks) { + Hash bid; + try { + BlockTemplate bheader; + seria::from_binary(bheader, rb.block); + auto body_proxy = get_body_proxy_from_template(bheader); + bid = cn::get_block_hash(bheader, body_proxy); + } catch (const std::exception &ex) { + m_node->m_log(logging::INFO) << "Exception " << common::what(ex) + << " while parsing returned block, banning " << get_address() << std::endl; + disconnect("Bad Block Returned"); + return; + } + auto cit = m_node->chain_blocks.find(bid); + if (cit == m_node->chain_blocks.end() || cit->second.who_downloading != this) { + m_node->m_log(logging::INFO) << "GetObjectsResponse received stray block from " << get_address() + << std::endl; + disconnect("Stray Block Returned"); + return; + } + cit->second.who_downloading = nullptr; + cit->second.preparing = true; + invariant(m_downloading_block_count > 0, ""); + m_downloading_block_count -= 1; + m_node->m_log(logging::TRACE) << "GetObjectsResponse received block " << cit->second.expected_height + << " hash=" << cit->first << " from " << get_address() << std::endl; + bool check_pow = m_node->m_config.paranoid_checks || + !m_node->m_block_chain.get_currency().is_in_hard_checkpoint_zone(cit->second.expected_height); + m_node->m_pow_checker.add_block(bid, check_pow, std::move(rb)); + } + p2p::RelayTransactions::Notify msg; + p2p::RelayTransactions::Notify msg_v4; + for (const auto &btx : req.txs) { + Transaction tx; + try { + seria::from_binary(tx, btx); + } catch (const std::exception &ex) { + return disconnect("Invalid transaction binary format " + common::what(ex)); + } + const Hash tid = get_transaction_hash(tx); + auto cit = m_node->downloading_transactions.find(tid); + if (cit == m_node->downloading_transactions.end() || cit->second != this) { + m_node->m_log(logging::INFO) << "GetObjectsResponse received stray transaction from " << get_address() + << std::endl; + return disconnect("Stray Transaction Returned"); + } + auto tit = m_transaction_descs.find(tid); + invariant(tit != m_transaction_descs.end(), ""); + if (tit->second.size != btx.size()) + return disconnect("Lied about transcation size"); + Amount my_fee = get_tx_fee(tx); + if (tit->second.fee != my_fee) + return disconnect("Lied about transcation fee"); + if (m_node->m_block_chain.in_chain(tit->second.newest_referenced_block)) { + Height newest_referenced_height = 0; + if (!m_node->m_block_chain.get_largest_referenced_height(tx, &newest_referenced_height) || + !m_node->m_block_chain.in_chain(newest_referenced_height, tit->second.newest_referenced_block)) + return disconnect("Lied about newest_referenced_block"); + try { + if (m_node->m_block_chain.add_transaction( + tid, tx, btx, m_node->m_p2p.get_local_time(), get_address().to_string())) { + msg.txs.push_back(btx); + TransactionDesc desc; + desc.hash = tid; + desc.size = btx.size(); + desc.fee = my_fee; + desc.newest_referenced_block = tit->second.newest_referenced_block; + msg_v4.transaction_descs.push_back(desc); + } + } catch (const ConsensusErrorOutputDoesNotExist &ex) { + // We are safe to ban for bad output reference, because we have newest referenced block + return disconnect("NOTIFY_NEW_TRANSACTIONS add_transaction BAN what=" + common::what(ex)); + } catch (const ConsensusErrorBadOutputOrSignature &ex) { + // We are safe to ban for bad signatures, because we have newest referenced block + return disconnect("NOTIFY_NEW_TRANSACTIONS add_transaction BAN what=" + common::what(ex)); + } catch (const ConsensusErrorOutputSpent &) { + // Not a ban reason + } catch (const std::exception &ex) { + return disconnect("NOTIFY_NEW_TRANSACTIONS add_transaction BAN what=" + common::what(ex)); + } + } + cit = m_node->downloading_transactions.erase(cit); + tit = m_transaction_descs.erase(tit); + invariant(m_downloading_transaction_count > 0, ""); + m_downloading_transaction_count -= 1; + for (auto who : m_node->m_broadcast_protocols) + if (who != this) + who->transaction_download_finished(tid, true); + } + for (auto &&tid : req.missed_ids) { // Here should be only transactions, we ask only block peer alwasy has + auto cit = m_node->downloading_transactions.find(tid); + if (cit == m_node->downloading_transactions.end() || cit->second != this) { + m_node->m_log(logging::INFO) << "GetObjectsResponse received stray missed_id from " << get_address() + << std::endl; + return disconnect("Stray Transaction Returned"); + } + auto tit = m_transaction_descs.find(tid); + invariant(tit != m_transaction_descs.end(), ""); + cit = m_node->downloading_transactions.erase(cit); + tit = m_transaction_descs.erase(tit); + invariant(m_downloading_transaction_count > 0, ""); + m_downloading_transaction_count -= 1; + for (auto who : m_node->m_broadcast_protocols) + if (who != this) + who->transaction_download_finished(tid, false); + } + if (m_downloading_block_count != 0) + m_download_timer.once(m_node->m_config.download_block_timeout); + else + m_download_timer.cancel(); + if (m_downloading_transaction_count != 0) + m_download_transactions_timer.once(m_node->m_config.download_transaction_timeout); + else + m_download_transactions_timer.cancel(); + if (!msg.txs.empty()) { + BinaryArray raw_msg = LevinProtocol::send(msg); + // Too much descs can happen only transaitional period when relaying transactions got from + // V1 client to V4 client. We are ok with very unlinkely transactions loss here + if (msg_v4.transaction_descs.size() > p2p::RelayTransactions::Notify::MAX_DESC_COUNT) + msg_v4.transaction_descs.resize(p2p::RelayTransactions::Notify::MAX_DESC_COUNT); + BinaryArray raw_msg_v4 = LevinProtocol::send(msg_v4); + + m_node->broadcast(this, raw_msg, raw_msg_v4); + m_node->advance_long_poll(); + } + if (!req.blocks.empty()) + advance_blocks(); } void Node::P2PProtocolBytecoin::on_disconnect(const std::string &ban_reason) { - m_node->broadcast_protocols.erase(this); - m_node->m_downloader.on_disconnect(this); + m_node->m_broadcast_protocols.erase(this); + + m_chain_request_sent = false; + m_chain_timer.cancel(); + + for (const auto &cit : m_chain) { + if (cit->second.who_downloading == this) { + m_downloading_block_count -= 1; + cit->second.who_downloading = nullptr; + } + m_node->remove_chain_block(cit); + } + m_chain.clear(); + m_download_timer.cancel(); + invariant(m_downloading_block_count == 0, ""); + + m_syncpool_equest_sent = false; + m_syncpool_timer.cancel(); + for (auto const &cit : m_transaction_descs) { + auto tit = m_node->downloading_transactions.find(cit.first); + if (tit->second == this) { + tit = m_node->downloading_transactions.erase(tit); + m_downloading_transaction_count -= 1; + } + for (auto who : m_node->m_broadcast_protocols) + who->transaction_download_finished(cit.first, false); + } + m_transaction_descs.clear(); + m_download_transactions_timer.cancel(); + invariant(m_downloading_transaction_count == 0, ""); P2PProtocolBasic::on_disconnect(ban_reason); m_node->advance_long_poll(); } -void Node::P2PProtocolBytecoin::on_msg_notify_request_tx_pool(NOTIFY_REQUEST_TX_POOL::request &&req) { - NOTIFY_NEW_TRANSACTIONS::request msg; +void Node::P2PProtocolBytecoin::on_msg_notify_request_tx_pool(p2p::SyncPool::Notify &&req) { + if (get_peer_version() >= P2PProtocolVersion::AMETHYST) + return disconnect("SyncPool notify not allowed in V4"); + p2p::RelayTransactions::Notify msg; auto mytxs = m_node->m_block_chain.get_memory_state_transactions(); msg.txs.reserve(mytxs.size()); std::sort(req.txs.begin(), req.txs.end()); // Should have been sorted on wire, @@ -153,73 +614,135 @@ void Node::P2PProtocolBytecoin::on_msg_notify_request_tx_pool(NOTIFY_REQUEST_TX_ << std::endl; if (msg.txs.empty()) return; - BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_NEW_TRANSACTIONS::ID, LevinProtocol::encode(msg), false); - send(std::move(raw_msg)); + send(LevinProtocol::send(msg)); } - -void Node::P2PProtocolBytecoin::on_msg_timed_sync(COMMAND_TIMED_SYNC::request &&req) { - m_node->m_downloader.advance_download(); +void Node::P2PProtocolBytecoin::on_msg_notify_request_tx_pool(p2p::SyncPool::Request &&req) { + if (req.from <= req.to) + return disconnect("SyncPool request from <= to"); + p2p::SyncPool::Response msg; + msg.transaction_descs = m_node->m_block_chain.sync_pool(req.from, req.to, p2p::SyncPool::Response::MAX_DESC_COUNT); + send(LevinProtocol::send(msg)); } -void Node::P2PProtocolBytecoin::on_msg_timed_sync(COMMAND_TIMED_SYNC::response &&req) { - m_node->m_downloader.advance_download(); + +void Node::P2PProtocolBytecoin::on_msg_notify_request_tx_pool(p2p::SyncPool::Response &&req) { + m_syncpool_timer.cancel(); + m_syncpool_equest_sent = false; + if (req.transaction_descs.size() > p2p::SyncPool::Response::MAX_DESC_COUNT) + return disconnect("SyncPool too much descs"); + if (!on_transaction_descs(req.transaction_descs)) + return; // Disconnected + if (req.transaction_descs.empty()) + syncpool_start = {0, Hash{}}; + else { + // Divisions by zero here is impossible due to checks in on_transaction_descs + Amount first_fee_per_byte = req.transaction_descs.front().fee / req.transaction_descs.front().size; + if (std::make_pair(first_fee_per_byte, req.transaction_descs.front().hash) >= syncpool_start) + return disconnect("SyncPool wrong chunk"); + Amount last_fee_per_byte = req.transaction_descs.back().fee / req.transaction_descs.back().size; + syncpool_start = std::make_pair(last_fee_per_byte, req.transaction_descs.back().hash); + } + advance_transactions(); } -void Node::P2PProtocolBytecoin::on_msg_notify_new_block(NOTIFY_NEW_BLOCK::request &&req) { - RawBlock raw_block{req.b.block, req.b.transactions}; - PreparedBlock pb(std::move(raw_block), m_node->m_block_chain.get_currency(), nullptr); +void Node::P2PProtocolBytecoin::on_msg_timed_sync(p2p::TimedSync::Request &&req) { advance_chain(); } +void Node::P2PProtocolBytecoin::on_msg_timed_sync(p2p::TimedSync::Response &&req) { advance_chain(); } + +void Node::P2PProtocolBytecoin::on_msg_notify_new_block(p2p::RelayBlock::Notify &&req) { + if (get_peer_version() >= P2PProtocolVersion::AMETHYST) { + if (!req.b.transactions.empty()) + return disconnect("RelayBlock only header realy allowed"); + if (m_node->m_block_chain.has_header(req.top_id)) + return; + BlockTemplate header; + seria::from_binary(header, req.b.block); + const auto &pool = m_node->m_block_chain.get_memory_state_transactions(); + for (const auto &tid : header.transaction_hashes) { + auto tit = pool.find(tid); + if (tit != pool.end()) { + req.b.transactions.push_back(tit->second.binary_tx); + continue; + } + BinaryArray binary_tx; + size_t index_in_block = 0; + Height block_height = 0; + Hash block_hash; + if (m_node->m_block_chain.get_transaction(tid, &binary_tx, &block_height, &block_hash, &index_in_block)) { + req.b.transactions.push_back(std::move(binary_tx)); + continue; + } + // We cannot reassemble block from transactions, will download it normally + auto body_proxy = get_body_proxy_from_template(header); + Hash bid = cn::get_block_hash(header, body_proxy); + set_peer_sync_data(CoreSyncData{req.current_blockchain_height, bid}); + advance_chain(); + return; + } + // We reassembled full block, can now broadcast it to V1 or V4 clients + } + PreparedBlock pb{RawBlock{req.b}, m_node->m_block_chain.get_currency(), nullptr}; + if (get_peer_version() >= P2PProtocolVersion::AMETHYST && req.top_id != pb.bid) + return disconnect("RelayBlock lied about top_id"); api::BlockHeader info; - auto action = m_node->m_block_chain.add_block(pb, &info, get_address().to_string()); - switch (action) { - case BroadcastAction::BAN: - disconnect("NOTIFY_NEW_BLOCK add_block BAN"); - return; - case BroadcastAction::BROADCAST_ALL: { - req.hop += 1; - BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_NEW_BLOCK::ID, LevinProtocol::encode(req), false); - m_node->broadcast(this, raw_msg); - m_node->broadcast_new(nullptr, pb.raw_block.block); + // We'll catch consensus error automatically in common handler + if (m_node->m_block_chain.add_block(pb, &info, get_address().to_string())) { + if (get_peer_version() >= P2PProtocolVersion::AMETHYST && req.current_blockchain_height != info.height) + return disconnect("RelayBlock lied about current_blockchain_height"); + set_peer_sync_data(CoreSyncData{info.height, pb.bid}); + p2p::RelayBlock::Notify req_v4; + req_v4.b.block = req.b.block; + req_v4.top_id = req.top_id = info.hash; + req_v4.current_blockchain_height = req.current_blockchain_height = info.height; + req_v4.hop = req.hop = 1; // req.hop + 1; Do not do that, hop count allows tracking block origin + + req.top_id = Hash{}; // TODO - uncomment after 3.4 fork. Workaround for 3.2 bug + + BinaryArray raw_msg = LevinProtocol::send(req); + BinaryArray raw_msg_v4 = LevinProtocol::send(req_v4); + m_node->broadcast(this, raw_msg, raw_msg_v4); m_node->advance_long_poll(); - break; - } - case BroadcastAction::NOTHING: - break; + } else { + set_peer_sync_data(CoreSyncData{req.current_blockchain_height, pb.bid}); } - set_last_received_sync_data(CORE_SYNC_DATA{req.current_blockchain_height - 1, pb.bid}); - // -1 is in legacy protocol - m_node->m_downloader.advance_download(); } -void Node::P2PProtocolBytecoin::on_msg_notify_new_transactions(NOTIFY_NEW_TRANSACTIONS::request &&req) { - if (m_node->m_block_chain_reader1 || m_node->m_block_chain_reader2 || - m_node->m_block_chain.get_tip_height() < m_node->m_block_chain.internal_import_known_height()) - return; // We cannot check tx while downloading anyway - NOTIFY_NEW_TRANSACTIONS::request msg; +void Node::P2PProtocolBytecoin::on_msg_notify_new_transactions(p2p::RelayTransactions::Notify &&req) { + if (get_peer_version() >= P2PProtocolVersion::AMETHYST) { + if (!req.txs.empty()) + return disconnect("RelayTransactions relaying transaction bodies not allowed in V4"); + if (req.transaction_descs.size() > p2p::RelayTransactions::Notify::MAX_DESC_COUNT) + return disconnect("RelayTransactions too much descs"); + on_transaction_descs(req.transaction_descs); + return; + } + p2p::RelayTransactions::Notify msg; + p2p::RelayTransactions::Notify msg_v4; Hash any_tid; for (auto &&raw_tx : req.txs) { Transaction tx; try { seria::from_binary(tx, raw_tx); + const Hash tid = get_transaction_hash(tx); + any_tid = tid; + if (m_node->m_block_chain.add_transaction( + tid, tx, raw_tx, m_node->m_p2p.get_local_time(), get_address().to_string())) { + msg.txs.push_back(raw_tx); + TransactionDesc desc; + desc.hash = tid; + desc.size = raw_tx.size(); + desc.fee = get_tx_fee(tx); + Height newest_referenced_height = 0; + invariant(m_node->m_block_chain.get_largest_referenced_height(tx, &newest_referenced_height), ""); + invariant(m_node->m_block_chain.get_chain(newest_referenced_height, &desc.newest_referenced_block), ""); + msg_v4.transaction_descs.push_back(desc); + } + } catch (const ConsensusErrorOutputDoesNotExist &) { + // Not a ban reason in V4 + } catch (const ConsensusErrorBadOutputOrSignature &) { + // Not a ban reason in V4 + } catch (const ConsensusErrorOutputSpent &) { + // Not a ban reason } catch (const std::exception &ex) { - disconnect("NOTIFY_NEW_TRANSACTIONS add_transaction BAN from_binary failed " + common::what(ex)); - return; - } - const Hash tid = get_transaction_hash(tx); - any_tid = tid; - Height conflict_height = 0; - auto action = m_node->m_block_chain.add_transaction( - tid, tx, raw_tx, m_node->m_p2p.get_local_time(), &conflict_height, get_address().to_string()); - switch (action) { - case AddTransactionResult::BAN: - disconnect("NOTIFY_NEW_TRANSACTIONS add_transaction BAN"); - return; - case AddTransactionResult::BROADCAST_ALL: - msg.txs.push_back(raw_tx); - break; - case AddTransactionResult::ALREADY_IN_POOL: - case AddTransactionResult::INCREASE_FEE: - case AddTransactionResult::FAILED_TO_REDO: - case AddTransactionResult::OUTPUT_ALREADY_SPENT: - break; + return disconnect("NOTIFY_NEW_TRANSACTIONS add_transaction BAN what=" + common::what(ex)); } } m_node->m_log(logging::TRACE) << "on_msg_notify_new_transactions from " << get_address() @@ -228,70 +751,37 @@ void Node::P2PProtocolBytecoin::on_msg_notify_new_transactions(NOTIFY_NEW_TRANSA << (any_tid == Hash{} ? "" : common::pod_to_hex(any_tid)) << std::endl; if (msg.txs.empty()) return; - BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_NEW_TRANSACTIONS::ID, LevinProtocol::encode(msg), false); - m_node->broadcast(this, raw_msg); - // m_node->broadcast_new(nullptr, ); // TODO - broadcast + BinaryArray raw_msg = LevinProtocol::send(msg); + // Too much descs can happen only transaitional period when relaying transactions got from + // V1 client to V4 client. We are ok with very unlinkely transactions loss here + if (msg_v4.transaction_descs.size() > p2p::RelayTransactions::Notify::MAX_DESC_COUNT) + msg_v4.transaction_descs.resize(p2p::RelayTransactions::Notify::MAX_DESC_COUNT); + BinaryArray raw_msg_v4 = LevinProtocol::send(msg_v4); + m_node->broadcast(this, raw_msg, raw_msg_v4); m_node->advance_long_poll(); } -void Node::P2PProtocolBytecoin::on_msg_notify_checkpoint(NOTIFY_CHECKPOINT::request &&req) { +void Node::P2PProtocolBytecoin::on_msg_notify_checkpoint(p2p::Checkpoint::Notify &&req) { if (!m_node->m_block_chain.add_checkpoint(req, get_address().to_string())) return; - m_node->m_log(logging::INFO) << "NOTIFY_CHECKPOINT::request height=" << req.height << " hash=" << req.hash + m_node->m_log(logging::INFO) << "p2p::Checkpoint::Notify height=" << req.height << " hash=" << req.hash << " key_id=" << req.key_id << " counter=" << req.counter << std::endl; - BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_CHECKPOINT::ID, LevinProtocol::encode(req), false); + BinaryArray raw_msg = LevinProtocol::send(req); m_node->broadcast(nullptr, raw_msg); // nullptr, not this - so a sender sees "reflection" of message - // m_node->broadcast_new(nullptr, raw_msg); // nullptr, not this - so a sender sees "reflection" of message - COMMAND_TIMED_SYNC::request ts_req; - ts_req.payload_data = CORE_SYNC_DATA{m_node->m_block_chain.get_tip_height(), m_node->m_block_chain.get_tip_bid()}; - raw_msg = LevinProtocol::send_message(COMMAND_TIMED_SYNC::ID, LevinProtocol::encode(ts_req), true); + p2p::TimedSync::Request ts_req; + ts_req.payload_data = CoreSyncData{m_node->m_block_chain.get_tip_height(), m_node->m_block_chain.get_tip_bid()}; + raw_msg = LevinProtocol::send(ts_req); m_node->broadcast(nullptr, raw_msg); - // m_node->broadcast_new(nullptr, raw_msg); // ? m_node->advance_long_poll(); } #if bytecoin_ALLOW_DEBUG_COMMANDS -void Node::P2PProtocolBytecoin::on_msg_network_state(COMMAND_REQUEST_NETWORK_STATE::request &&req) { - if (!m_node->check_trust(req.tr)) { - disconnect(std::string()); - return; - } - COMMAND_REQUEST_NETWORK_STATE::response msg; - msg.local_time = m_node->m_p2p.get_local_time(); - msg.my_id = get_unique_number(); - for (auto &&cc : m_node->m_downloader.get_good_clients()) { - connection_entry item; - item.is_income = cc.first->is_incoming(); - item.id = cc.first->get_unique_number(); - item.adr.port = cc.first->get_address().port; - item.adr.ip = ip_address_to_legacy(cc.first->get_address().ip); - msg.connections_list.push_back(item); - } - BinaryArray raw_msg = LevinProtocol::send_reply(COMMAND_REQUEST_NETWORK_STATE::ID, LevinProtocol::encode(msg), 0); - send(std::move(raw_msg)); -} -void Node::P2PProtocolBytecoin::on_msg_stat_info(COMMAND_REQUEST_STAT_INFO::request &&req) { - if (!m_node->check_trust(req.tr)) { - disconnect(std::string()); - return; - } - COMMAND_REQUEST_STAT_INFO::response msg; - for (auto &&pb : m_node->broadcast_protocols) - if (pb->is_incoming()) - msg.incoming_connections_count += 1; - else - msg.connections_count += 1; - for (auto &&pb : m_node->broadcast_protocols_new) - if (pb->is_incoming()) - msg.incoming_connections_count += 1; - else - msg.connections_count += 1; - msg.connections_count += msg.incoming_connections_count; - msg.version = app_version(); - msg.os_version = platform::get_os_version_string(); - msg.payload_info = CoreStatistics{}; - BinaryArray raw_msg = LevinProtocol::send_reply(COMMAND_REQUEST_STAT_INFO::ID, LevinProtocol::encode(msg), 0); +void Node::P2PProtocolBytecoin::on_msg_stat_info(p2p::GetStatInfo::Request &&req) { + if (!m_node->check_trust(req.tr)) + return disconnect(std::string()); + p2p::GetStatInfo::Response msg = m_node->create_statistics_response(api::cnd::GetStatistics::Request{true, true}); + BinaryArray raw_msg = LevinProtocol::send(msg); send(std::move(raw_msg)); } diff --git a/src/Core/Node_P2PProtocolBytecoinNew.cpp b/src/Core/Node_P2PProtocolBytecoinNew.cpp deleted file mode 100644 index be603769..00000000 --- a/src/Core/Node_P2PProtocolBytecoinNew.cpp +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) 2012-2018, The CryptoNote developers, The Bytecoin developers. -// Licensed under the GNU Lesser General Public License. See LICENSE for details. - -#include -#include -#include "Config.hpp" -#include "CryptoNoteTools.hpp" -#include "Node.hpp" -#include "TransactionExtra.hpp" -#include "common/JsonValue.hpp" -#include "platform/PathTools.hpp" -#include "seria/BinaryInputStream.hpp" -#include "seria/BinaryOutputStream.hpp" -#include "seria/KVBinaryInputStream.hpp" -#include "seria/KVBinaryOutputStream.hpp" -#include "version.hpp" - -using namespace bytecoin; - -void Node::P2PProtocolBytecoinNew::on_download_timer() { disconnect(std::string()); } - -np::TopBlockDesc Node::P2PProtocolBytecoinNew::get_top_block_desc() const { - np::TopBlockDesc result; - result.cd = m_node->m_block_chain.get_tip_cumulative_difficulty(); - result.height = m_node->m_block_chain.get_tip_height(); - result.hash = m_node->m_block_chain.get_tip_bid(); - return result; -} - -std::vector Node::P2PProtocolBytecoinNew::get_peers_to_share() const { - auto result = m_node->m_peer_db.get_peerlist_to_p2p( - get_address(), m_node->m_p2p.get_local_time(), m_node->m_config.p2p_default_peers_in_handshake); - return result; -} - -void Node::P2PProtocolBytecoinNew::on_first_message_after_handshake() { - // if we set just seen on handshake, we will keep connecting to seed nodes forever - m_node->m_peer_db.set_peer_just_seen(get_other_peer_desc().peer_id, get_address(), m_node->m_p2p.get_local_time()); -} - -void Node::P2PProtocolBytecoinNew::on_disconnect(const std::string &ban_reason) { - m_node->broadcast_protocols_new.erase(this); - m_node->m_downloader_v3.on_disconnect(this); - - P2PProtocolNew::on_disconnect(ban_reason); - m_node->advance_long_poll(); -} - -void Node::P2PProtocolBytecoinNew::on_msg_bytes(size_t, size_t) { // downloaded. uploaded - // P2PProtocolNew::on_msg_bytes(, ); -} -void Node::P2PProtocolBytecoinNew::after_handshake() { - m_node->m_p2p.peers_updated(); - m_node->broadcast_protocols_new.insert(this); - m_node->advance_long_poll(); - - auto signed_checkpoints = m_node->m_block_chain.get_latest_checkpoints(); - for (const auto &sck : signed_checkpoints) { - BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_CHECKPOINT::ID, LevinProtocol::encode(sck), false); - send(std::move(raw_msg)); - } - m_node->m_downloader_v3.on_connect(this); // Can destroy this... -} - -void Node::P2PProtocolBytecoinNew::on_msg_handshake(np::Handshake::Request &&req) { - m_node->m_peer_db.add_incoming_peer(get_address(), m_node->m_p2p.get_local_time()); - after_handshake(); -} -void Node::P2PProtocolBytecoinNew::on_msg_handshake(np::Handshake::Response &&req) { - m_node->m_peer_db.merge_peerlist_from_p2p(get_address(), req.peerlist, m_node->m_p2p.get_local_time()); - after_handshake(); -} -void Node::P2PProtocolBytecoinNew::on_msg_find_diff(np::FindDiff::Request &&req) { - np::FindDiff::Response fd; - if (req.gap_start.size() > np::FindDiff::Request::MAX_GAP_START_LENGTH || req.gap_start.size() == 0) - return disconnect("FindDiff.Request.gap_start.size violation"); - for (Hash gap : req.gap_start) { - fd.sparse_chain = m_node->m_block_chain.get_sparse_chain(gap, req.desired_bid); - if (fd.sparse_chain.empty()) - continue; - BinaryArray msg = seria::to_binary_kv(fd); - send(create_header(np::FindDiff::Response::ID, msg.size())); - send(std::move(msg)); - } - disconnect("FindDiff Request protocol violation - no hash from req.gap_start found in blockchain"); -} -void Node::P2PProtocolBytecoinNew::on_msg_find_diff(np::FindDiff::Response &&resp) { - m_node->m_downloader_v3.on_msg_find_diff(this, std::move(resp)); -} -void Node::P2PProtocolBytecoinNew::on_msg_sync_headers(np::SyncHeaders::Request &&req) { - if (req.max_count > np::SyncHeaders::Request::MAX_COUNT) - return disconnect("SyncHeaders.Request.max_count violation"); - api::BlockHeader previous_header; - if (!m_node->m_block_chain.read_header(req.previous_hash, &previous_header)) - return disconnect("SyncHeaders.Request.previous_hash block not found"); - np::SyncHeaders::Response res; - if (m_node->m_block_chain.in_chain(previous_header.height, previous_header.hash)) { - for (Height ha = previous_header.height + 1; ha < m_node->m_block_chain.get_tip_height(); ++ha) { - if (res.binary_headers.size() >= req.max_count) - break; - Hash bid; - invariant(m_node->m_block_chain.read_chain(ha, &bid), ""); - RawBlock rb; - invariant(m_node->m_block_chain.read_block(bid, &rb), "SyncHeaders.Request block not found"); - res.binary_headers.push_back(rb.block); - } - } - BinaryArray msg = seria::to_binary_kv(res); - send(create_header(np::SyncHeaders::Response::ID, msg.size())); - send(std::move(msg)); -} -void Node::P2PProtocolBytecoinNew::on_msg_sync_headers(np::SyncHeaders::Response &&resp) { - m_node->m_downloader_v3.on_msg_sync_headers(this, std::move(resp)); -} -void Node::P2PProtocolBytecoinNew::on_msg_get_transactions(np::GetTransactions::Request &&req) { - if (req.transaction_hashes.size() > np::GetTransactions::Request::MAX_TRANSACTION_HASHES) - return disconnect("MAX_TRANSACTION_HASHES"); - np::GetTransactions::Response fd; - fd.top_block_desc = get_top_block_desc(); - if (req.transaction_hashes.empty()) { - RawBlock rb; - if (m_node->m_block_chain.read_block(req.block_hash, &rb)) { - fd.transactions = std::move(rb.transactions); - } - } else { - fd.transactions.reserve(req.transaction_hashes.size()); - for (auto &&tid : req.transaction_hashes) { - BinaryArray binary_tx; - Height block_height = 0; - Hash block_hash; - size_t index_in_block = 0; - if (m_node->m_block_chain.read_transaction(tid, &binary_tx, &block_height, &block_hash, &index_in_block)) { - fd.transactions.push_back(std::move(binary_tx)); - } - } - } - BinaryArray msg = seria::to_binary_kv(fd); - send(create_header(np::GetTransactions::Response::ID, msg.size())); - send(std::move(msg)); -} -void Node::P2PProtocolBytecoinNew::on_msg_get_transactions(np::GetTransactions::Response &&resp) { - m_node->m_downloader_v3.on_msg_get_transactions(this, std::move(resp)); -} -void Node::P2PProtocolBytecoinNew::on_msg_get_pool_hashes(np::GetPoolHashes::Request &&req) {} -void Node::P2PProtocolBytecoinNew::on_msg_get_pool_hashes(np::GetPoolHashes::Response &&resp) {} -void Node::P2PProtocolBytecoinNew::on_msg_relay_block_header(np::RelayBlockHeader &&req) {} -void Node::P2PProtocolBytecoinNew::on_msg_relay_transaction_desc(np::RelayTransactionDescs &&req) {} - -#if bytecoin_ALLOW_DEBUG_COMMANDS -void Node::P2PProtocolBytecoinNew::on_msg_get_peer_statistics(np::GetPeerStatistics::Request &&req) { - if (!m_node->check_trust(req.tr)) - return disconnect(std::string()); - np::GetPeerStatistics::Response res; - res = m_node->create_statistics_response(); - BinaryArray msg = seria::to_binary_kv(res); - send(create_header(np::GetPeerStatistics::Response::ID, msg.size())); - send(std::move(msg)); -} -#endif diff --git a/src/Core/TransactionBuilder.cpp b/src/Core/TransactionBuilder.cpp index 492e9a2c..294e5914 100644 --- a/src/Core/TransactionBuilder.cpp +++ b/src/Core/TransactionBuilder.cpp @@ -6,156 +6,174 @@ #include "BlockChain.hpp" #include "CryptoNoteTools.hpp" #include "Currency.hpp" +#include "TransactionExtra.hpp" #include "Wallet.hpp" +#include "WalletStateBasic.hpp" +#include "common/Varint.hpp" #include "common/string.hpp" #include "crypto/crypto.hpp" -#include "crypto/random.h" #include "http/JsonRpc.hpp" #include "seria/BinaryOutputStream.hpp" -using namespace bytecoin; - -bool TransactionBuilder::derive_public_key(const AccountPublicAddress &to, - const SecretKey &tx_key, - size_t output_index, - PublicKey &ephemeral_key) { - KeyDerivation derivation; - if (!generate_key_derivation(to.view_public_key, tx_key, derivation)) - return false; - return crypto::derive_public_key(derivation, output_index, to.spend_public_key, ephemeral_key); -} - -TransactionBuilder::TransactionBuilder(const Currency ¤cy, BlockOrTimestamp unlock_time) { - m_transaction.version = currency.current_transaction_version; - m_transaction.unlock_block_or_timestamp = unlock_time; +using namespace cn; + +OutputKey TransactionBuilder::create_output(const AccountAddress &to, const SecretKey &tx_secret_key, + const Hash &tx_inputs_hash, size_t output_index, const Hash &output_secret) { + OutputKey out_key; + if (to.type() == typeid(AccountAddressSimple)) { + auto &addr = boost::get(to); + const KeyDerivation derivation = crypto::generate_key_derivation(addr.view_public_key, tx_secret_key); + out_key.public_key = crypto::derive_public_key(derivation, output_index, addr.spend_public_key); + for (size_t i = 0; i != sizeof(output_secret); ++i) + out_key.encrypted_secret.data[i] = output_secret.data[i] ^ addr.view_public_key.data[i]; + return out_key; + } + if (to.type() == typeid(AccountAddressUnlinkable)) { + auto &addr = boost::get(to); + out_key.public_key = crypto::unlinkable_derive_public_key( + output_secret, tx_inputs_hash, output_index, addr.s, addr.sv, &out_key.encrypted_secret); + out_key.is_auditable = addr.is_auditable; + return out_key; + } + throw std::runtime_error("TransactionBuilder::create_output unknown address type"); } -void TransactionBuilder::set_payment_id(const Hash &hash) { extra_add_payment_id(m_transaction.extra, hash); } - -size_t TransactionBuilder::add_output(uint64_t amount, const AccountPublicAddress &to) { - m_outputs_amount += amount; - - OutputDesc desc; - desc.amount = amount; - desc.addr = to; - m_output_descs.push_back(std::move(desc)); - return m_output_descs.size() - 1; +void TransactionBuilder::add_output(uint64_t amount, const AccountAddress &to) { + m_output_descs.push_back(OutputDesc{amount, to}); } static bool APIOutputLessGlobalIndex(const api::Output &a, const api::Output &b) { return a.index < b.index; } static bool APIOutputEqualGlobalIndex(const api::Output &a, const api::Output &b) { return a.index == b.index; } -bool TransactionBuilder::generate_key_image_helper(const AccountKeys &ack, const PublicKey &tx_public_key, - size_t real_output_index, KeyPair &in_ephemeral, KeyImage &ki) { - KeyDerivation recv_derivation; - bool r = generate_key_derivation(tx_public_key, ack.view_secret_key, recv_derivation); - if (!r) - return false; - r = crypto::derive_public_key( - recv_derivation, real_output_index, ack.address.spend_public_key, in_ephemeral.public_key); - if (!r) - return false; - crypto::derive_secret_key(recv_derivation, real_output_index, ack.spend_secret_key, in_ephemeral.secret_key); - crypto::generate_key_image(in_ephemeral.public_key, in_ephemeral.secret_key, ki); - return true; -} -std::vector TransactionBuilder::absolute_output_offsets_to_relative(const std::vector &off) { - auto copy = off; - for (size_t i = 1; i < copy.size(); ++i) { - copy[i] = off[i] - off[i - 1]; - } - return copy; +void TransactionBuilder::add_input(const std::vector &mix_outputs, size_t real_output_index) { + m_input_descs.push_back(InputDesc{mix_outputs, real_output_index}); } -size_t TransactionBuilder::add_input(const AccountKeys &sender_keys, - api::Output real_output, - const std::vector &mix_outputs) { - m_inputs_amount += real_output.amount; - - InputDesc desc; - desc.input.amount = real_output.amount; - desc.outputs = mix_outputs; - std::sort(desc.outputs.begin(), desc.outputs.end(), APIOutputLessGlobalIndex); - desc.real_output_index = - std::lower_bound(desc.outputs.begin(), desc.outputs.end(), real_output, APIOutputLessGlobalIndex) - - desc.outputs.begin(); - desc.outputs.insert(desc.outputs.begin() + desc.real_output_index, real_output); - - if (!generate_key_image_helper(sender_keys, real_output.transaction_public_key, real_output.index_in_transaction, - desc.eph_keys, desc.input.key_image)) - throw std::runtime_error("generating key_image failed"); - if (desc.input.key_image != real_output.key_image) - throw std::runtime_error("generated key_image does not match input"); - - // fill outputs array and use relative offsets - for (const auto &out : desc.outputs) { - if (out.amount != real_output.amount) // they are all zero as sent from node - throw std::runtime_error("Mixin outputs with different amounts is not allowed"); - desc.input.output_indexes.push_back(out.index); - } +KeyPair TransactionBuilder::transaction_keys_from_seed(const Hash &tx_inputs_hash, const Hash &tx_derivation_seed) { + BinaryArray ba; + common::append(ba, std::begin(tx_inputs_hash.data), std::end(tx_inputs_hash.data)); + common::append(ba, std::begin(tx_derivation_seed.data), std::end(tx_derivation_seed.data)); - desc.input.output_indexes = absolute_output_offsets_to_relative(desc.input.output_indexes); - m_input_descs.push_back(std::move(desc)); - return m_input_descs.size() - 1; + KeyPair tx_keys{}; + tx_keys.secret_key = crypto::hash_to_scalar(ba.data(), ba.size()); + crypto::secret_key_to_public_key(tx_keys.secret_key, &tx_keys.public_key); + return tx_keys; } -KeyPair TransactionBuilder::deterministic_keys_from_seed(const Hash &tx_inputs_hash, const Hash &tx_derivation_seed) { +KeyPair TransactionBuilder::deterministic_keys_from_seed( + const Hash &tx_inputs_hash, const Hash &tx_derivation_seed, const BinaryArray &add) { BinaryArray ba; - common::append(ba, std::begin(tx_inputs_hash.data), std::end(tx_inputs_hash.data)); common::append(ba, std::begin(tx_derivation_seed.data), std::end(tx_derivation_seed.data)); + common::append(ba, std::begin(tx_inputs_hash.data), std::end(tx_inputs_hash.data)); + common::append(ba, add); KeyPair tx_keys{}; - crypto::hash_to_scalar(ba.data(), ba.size(), tx_keys.secret_key); - crypto::secret_key_to_public_key(tx_keys.secret_key, tx_keys.public_key); + tx_keys.secret_key = crypto::hash_to_scalar(ba.data(), ba.size()); + crypto::secret_key_to_public_key(tx_keys.secret_key, &tx_keys.public_key); return tx_keys; } -KeyPair TransactionBuilder::deterministic_keys_from_seed(const TransactionPrefix &tx, const Hash &tx_derivation_seed) { +KeyPair TransactionBuilder::deterministic_keys_from_seed( + const TransactionPrefix &tx, const Hash &tx_derivation_seed, const BinaryArray &add) { Hash tx_inputs_hash = get_transaction_inputs_hash(tx); - return deterministic_keys_from_seed(tx_inputs_hash, tx_derivation_seed); + return deterministic_keys_from_seed(tx_inputs_hash, tx_derivation_seed, add); } -Transaction TransactionBuilder::sign(const Hash &tx_derivation_seed) { +Transaction TransactionBuilder::sign( + const WalletStateBasic &wallet_state, Wallet *wallet, const std::set *only_records) { std::shuffle(m_output_descs.begin(), m_output_descs.end(), crypto::random_engine{}); std::shuffle(m_input_descs.begin(), m_input_descs.end(), crypto::random_engine{}); std::stable_sort(m_output_descs.begin(), m_output_descs.end(), OutputDesc::less_amount); std::stable_sort(m_input_descs.begin(), m_input_descs.end(), InputDesc::less_amount); + const bool is_tx_amethyst = m_transaction.version >= wallet_state.get_currency().amethyst_transaction_version; + // First we create inputs, because we need tx_inputs_hash + m_transaction.inputs.reserve(m_input_descs.size()); + for (size_t i = 0; i != m_input_descs.size(); ++i) { + const InputDesc &desc = m_input_descs[i]; + const api::Output &our_output = desc.outputs.at(desc.real_output_index); + InputKey input_key; + input_key.key_image = our_output.key_image; + input_key.amount = our_output.amount; + for (const auto &o : desc.outputs) + input_key.output_indexes.push_back(o.index); + input_key.output_indexes = absolute_output_offsets_to_relative(input_key.output_indexes); + m_transaction.inputs.push_back(input_key); + } // Deterministic generation of tx private key. - m_transaction.inputs.resize(m_input_descs.size()); - for (size_t i = 0; i != m_input_descs.size(); ++i) - m_transaction.inputs.at(i) = std::move(m_input_descs[i].input); - KeyPair tx_keys = deterministic_keys_from_seed(m_transaction, tx_derivation_seed); + const Hash tx_inputs_hash = get_transaction_inputs_hash(m_transaction); + const KeyPair tx_keys = transaction_keys_from_seed(tx_inputs_hash, wallet->get_tx_derivation_seed()); extra_add_transaction_public_key(m_transaction.extra, tx_keys.public_key); // Now when we set tx keys we can derive output keys m_transaction.outputs.resize(m_output_descs.size()); - for (size_t i = 0; i != m_output_descs.size(); ++i) { - KeyOutput out_key; - if (!derive_public_key(m_output_descs[i].addr, tx_keys.secret_key, i, out_key.public_key)) - throw std::runtime_error("output keys detected as corrupted during output key derivation"); - TransactionOutput out; // TODO - return {} initializer after NDK compiler upgrade - out.amount = m_output_descs[i].amount; - out.target = out_key; - m_transaction.outputs.at(i) = out; + for (size_t out_index = 0; out_index != m_output_descs.size(); ++out_index) { + KeyPair output_secret_keys = deterministic_keys_from_seed( + tx_inputs_hash, wallet->get_tx_derivation_seed(), common::get_varint_data(out_index)); + Hash output_secret = + crypto::cn_fast_hash(output_secret_keys.public_key.data, sizeof(output_secret_keys.public_key.data)); + OutputKey out_key = TransactionBuilder::create_output( + m_output_descs.at(out_index).addr, tx_keys.secret_key, tx_inputs_hash, out_index, output_secret); + out_key.amount = m_output_descs.at(out_index).amount; + m_transaction.outputs.at(out_index) = out_key; } - - Hash hash = get_transaction_prefix_hash(m_transaction); - m_transaction.signatures.resize(m_input_descs.size()); + // Now we can sign + const Hash hash = get_transaction_prefix_hash(m_transaction); + std::vector all_secret_keys; + std::vector all_sec_indexes; + std::vector all_keyimages; + std::vector> all_output_keys; for (size_t i = 0; i != m_input_descs.size(); ++i) { - const KeyInput &input = boost::get(m_transaction.inputs.at(i)); - const InputDesc &desc = m_input_descs[i]; - std::vector signatures; - std::vector keys_ptrs; - for (const auto &o : desc.outputs) { - keys_ptrs.push_back(&o.public_key); - } - signatures.resize(keys_ptrs.size(), Signature{}); - if (!generate_ring_signature(hash, input.key_image, keys_ptrs, desc.eph_keys.secret_key, desc.real_output_index, - signatures.data())) { - throw std::runtime_error("output keys detected as corrupted during ring signing"); + const InputDesc &desc = m_input_descs[i]; + const api::Output &our_output = desc.outputs.at(desc.real_output_index); + std::vector output_keys; + for (const auto &o : desc.outputs) + output_keys.push_back(o.public_key); + TransactionPrefix ptx; + api::Transaction atx; + invariant(wallet_state.get_transaction(our_output.transaction_hash, &ptx, &atx) && + our_output.index_in_transaction < ptx.outputs.size(), + "Originating transaction for output not found"); + const auto &key_output = boost::get(ptx.outputs.at(our_output.index_in_transaction)); + Hash other_inputs_hash = get_transaction_inputs_hash(ptx); + AccountAddress address; + if (!wallet_state.get_currency().parse_account_address_string(our_output.address, &address)) + throw json_rpc::Error(json_rpc::INTERNAL_ERROR, "Could not parse address " + our_output.address); + invariant(!only_records || only_records->count(address) != 0, "Output with wrong address selected by selector"); + WalletRecord record; + if (!wallet->get_record(record, address)) + throw json_rpc::Error(json_rpc::INTERNAL_ERROR, "No keys in wallet for address " + our_output.address); + KeyPair output_keypair; + boost::optional kd; + PublicKey spend_public_key; + SecretKey spend_secret; + wallet->get_output_handler()(atx.public_key, &kd, other_inputs_hash, our_output.index_in_transaction, + key_output, &spend_public_key, &spend_secret); + Amount other_amount = 0; + AccountAddress other_address; + if (!wallet->detect_our_output(atx.hash, other_inputs_hash, kd, our_output.index_in_transaction, + spend_public_key, spend_secret, key_output, &other_amount, &output_keypair, &other_address)) + throw json_rpc::Error(json_rpc::INTERNAL_ERROR, "No keys in wallet for address " + our_output.address); + const KeyImage other_key_image = generate_key_image(output_keypair.public_key, output_keypair.secret_key); + invariant(other_key_image == our_output.key_image, "generated key_image does not match input"); + all_keyimages.push_back(our_output.key_image); + all_output_keys.push_back(std::move(output_keys)); + all_secret_keys.push_back(output_keypair.secret_key); + all_sec_indexes.push_back(desc.real_output_index); + } + if (is_tx_amethyst) { + const RingSignature3 ring_signatures3 = generate_ring_signature3( + hash, all_keyimages, all_output_keys, all_secret_keys, all_sec_indexes, wallet->get_view_secret_key()); + m_transaction.signatures = std::move(ring_signatures3); + } else { + RingSignatures ring_signatures; + for (size_t i = 0; i != m_input_descs.size(); ++i) { + const RingSignature signature = + generate_ring_signature(hash, all_keyimages.at(i), all_output_keys.at(i).data(), + all_output_keys.at(i).size(), all_secret_keys.at(i), all_sec_indexes.at(i)); + ring_signatures.signatures.push_back(signature); } - m_transaction.signatures.at(i) = signatures; + m_transaction.signatures = std::move(ring_signatures); } return m_transaction; } @@ -172,73 +190,59 @@ void UnspentSelector::reset(Unspents &&unspents) { m_ra_amounts.clear(); } -void UnspentSelector::add_mixed_inputs(const SecretKey &view_secret_key, const Wallet *wallet, - const std::unordered_map &wallet_records, TransactionBuilder *builder, uint32_t anonymity, - api::bytecoind::GetRandomOutputs::Response &&ra_response) { +size_t UnspentSelector::add_mixed_inputs( + TransactionBuilder *builder, size_t anonymity, api::cnd::GetRandomOutputs::Response &&ra_response) { + size_t actual_anonymity = anonymity; for (const auto &uu : m_used_unspents) { std::vector mix_outputs; - if(anonymity != 0){ - auto &our_ra_outputs = ra_response.outputs[uu.amount]; - while (mix_outputs.size() < anonymity + 1) { - if (our_ra_outputs.empty()) - throw json_rpc::Error(api::walletd::CreateTransaction::NOT_ENOUGH_ANONYMITY, - "Requested anonymity too high for amount " + common::to_string(uu.amount)); - mix_outputs.push_back(std::move(our_ra_outputs.back())); - our_ra_outputs.pop_back(); - } - std::sort(mix_outputs.begin(), mix_outputs.end(), APIOutputLessGlobalIndex); - mix_outputs.erase( - std::unique(mix_outputs.begin(), mix_outputs.end(), APIOutputEqualGlobalIndex), mix_outputs.end()); - int best_distance = 0; - size_t best_index = mix_outputs.size(); - for (size_t i = 0; i != mix_outputs.size(); ++i) { - int distance = abs(int(uu.index) - int(mix_outputs[i].index)); - if (best_index == mix_outputs.size() || distance < best_distance) { - best_index = i; - best_distance = distance; - } - } - invariant(best_index != mix_outputs.size(), ""); - mix_outputs.erase(mix_outputs.begin() + best_index); + auto &our_ra_outputs = ra_response.outputs[uu.amount]; + while (mix_outputs.size() < anonymity + 1 && !our_ra_outputs.empty()) { + if (uu.amount != our_ra_outputs.back().amount) + throw std::runtime_error("Got outputs with wrong amounts from GetRandomOutputs call"); + mix_outputs.push_back(std::move(our_ra_outputs.back())); + our_ra_outputs.pop_back(); } - AccountKeys sender_keys; - sender_keys.view_secret_key = view_secret_key; - if (!m_currency.parse_account_address_string(uu.address, &sender_keys.address)) - throw json_rpc::Error(json_rpc::INTERNAL_ERROR, "Could not parse address " + uu.address); - if (wallet) { - WalletRecord record; - if (!wallet->get_record(record, sender_keys.address)) - throw json_rpc::Error(json_rpc::INTERNAL_ERROR, "No keys in wallet for address " + uu.address); - sender_keys.spend_secret_key = record.spend_secret_key; - } else { - auto rit = wallet_records.find(sender_keys.address.spend_public_key); - if (rit == wallet_records.end() || rit->second.spend_public_key != sender_keys.address.spend_public_key) - throw json_rpc::Error(json_rpc::INTERNAL_ERROR, "No keys in wallet for address " + uu.address); - sender_keys.spend_secret_key = rit->second.spend_secret_key; + std::sort(mix_outputs.begin(), mix_outputs.end(), APIOutputLessGlobalIndex); + mix_outputs.erase( + std::unique(mix_outputs.begin(), mix_outputs.end(), APIOutputEqualGlobalIndex), mix_outputs.end()); + int best_distance = 0; + size_t best_index = mix_outputs.size(); + for (size_t i = 0; i != mix_outputs.size(); ++i) { + int distance = abs(int(uu.index) - int(mix_outputs[i].index)); + if (best_index == mix_outputs.size() || distance < best_distance) { + best_index = i; + best_distance = distance; + } } - builder->add_input(sender_keys, uu, mix_outputs); + if (best_index != mix_outputs.size()) // mix_outputs not empty + mix_outputs.erase(mix_outputs.begin() + best_index); + actual_anonymity = std::min(actual_anonymity, mix_outputs.size()); + mix_outputs.insert(mix_outputs.begin() + best_index, uu); + invariant(std::is_sorted(mix_outputs.begin(), mix_outputs.end(), APIOutputLessGlobalIndex), ""); + builder->add_input(mix_outputs, best_index); } + return actual_anonymity; } -constexpr Amount fake_large = 1000000000000000000; // optimize negative amounts - // by adding this large - // number to them +const bool detailed_output = false; +constexpr Amount fake_large = 10000000000000000000ULL; +// optimize negative amounts by adding this large number to them constexpr size_t OPTIMIZATIONS_PER_TX = 50; constexpr size_t OPTIMIZATIONS_PER_TX_AGGRESSIVE = 200; constexpr size_t MEDIAN_PERCENT = 25; // make tx up to X% of block constexpr size_t MEDIAN_PERCENT_AGGRESSIVE = 50; // make tx up to X% of block constexpr size_t STACK_OPTIMIZATION_THRESHOLD = 20; // If any coin stack is larger, we will spend 10 coins. -constexpr size_t TWO_THRESHOLD = 10; // if any of 2 coin stacks is larger, we - // will use 2 coins to cover single digit - // (e.g. 7 + 9 for 6) +constexpr size_t TWO_THRESHOLD = 10; +// if any of 2 coin stacks is larger, we will use 2 coins to cover single digit (e.g. 7 + 9 for 6) -void UnspentSelector::select_optimal_outputs(Height block_height, Timestamp block_time, Height confirmed_height, - size_t effective_median_size, size_t anonymity, Amount total_amount, size_t total_outputs, Amount fee_per_byte, - std::string optimization_level, Amount *change, Amount *receiver_fee) { - HaveCoins have_coins; +void UnspentSelector::select_optimal_outputs(size_t max_transaction_size, size_t anonymity, size_t min_anonymity, + Amount total_amount, size_t total_outputs, Amount fee_per_byte, std::string optimization_level, Amount *change, + Amount *receiver_fee) { + PrettyCoins pretty_coins; size_t max_digits; - DustCoins dust_coins; - create_have_coins(block_height, block_time, confirmed_height, &have_coins, &dust_coins, &max_digits); + NonPrettyCoins non_pretty_coins; + NonPrettyCoins dust_coins; + create_coin_index(&pretty_coins, &non_pretty_coins, &dust_coins, &max_digits); Amount fee = 0; size_t optimizations = (optimization_level == "aggressive") ? OPTIMIZATIONS_PER_TX_AGGRESSIVE @@ -246,19 +250,19 @@ void UnspentSelector::select_optimal_outputs(Height block_height, Timestamp bloc bool small_optimizations = true; // 9 allows some dust optimization, but never "stack of coins" optimization. // "Minimal" optimization does not mean no optimization - size_t optimization_median_percent = + const size_t optimization_median_percent = (optimization_level == "aggressive") ? MEDIAN_PERCENT_AGGRESSIVE : MEDIAN_PERCENT; - const size_t optimization_median = effective_median_size * optimization_median_percent / 100; + const size_t optimization_median = max_transaction_size * optimization_median_percent / 100; const Amount dust_threshold = m_currency.self_dust_threshold; while (true) { - if (!select_optimal_outputs(&have_coins, &dust_coins, max_digits, total_amount + (receiver_fee ? 0 : fee), - anonymity, optimizations, small_optimizations)) + if (!select_optimal_outputs(&pretty_coins, &non_pretty_coins, &dust_coins, max_digits, + total_amount + (receiver_fee ? 0 : fee), anonymity, optimizations, small_optimizations)) throw json_rpc::Error(api::walletd::CreateTransaction::NOT_ENOUGH_FUNDS, "Not enough spendable funds"); Amount change_dust_fee = (m_used_total - total_amount - (receiver_fee ? 0 : fee)) % dust_threshold; size_t tx_size = get_maximum_tx_size(m_inputs_count, total_outputs + m_currency.get_max_amount_outputs(), anonymity); // Expected max change outputs if (tx_size > optimization_median && (optimizations > 0 || small_optimizations)) { - unoptimize_amounts(&have_coins, &dust_coins); + return_coins_to_index(&pretty_coins, &non_pretty_coins, &dust_coins); if (optimizations == 0) small_optimizations = false; optimizations /= 2; @@ -271,22 +275,23 @@ void UnspentSelector::select_optimal_outputs(Height block_height, Timestamp bloc throw json_rpc::Error(json_rpc::INVALID_PARAMS, "'fee_per_byte' is too large for transaction of size " + common::to_string(tx_size)); Amount size_fee = fee_per_byte * tx_size; - if (tx_size > effective_median_size) { + if (tx_size > max_transaction_size) { fee = ((size_fee + dust_threshold - 1) / dust_threshold) * dust_threshold; - unoptimize_amounts(&have_coins, &dust_coins); + return_coins_to_index(&pretty_coins, &non_pretty_coins, &dust_coins); auto ets = get_maximum_tx_size(0, total_outputs + 2 * m_currency.get_max_amount_outputs(), anonymity); - auto max_inputs_count = (effective_median_size - ets) / get_maximum_tx_input_size(anonymity); - select_max_outputs( - &have_coins, &dust_coins, std::numeric_limits::max(), anonymity, max_inputs_count); + auto max_inputs_count = (max_transaction_size - ets) / get_maximum_tx_input_size(anonymity); + select_max_outputs(&pretty_coins, &non_pretty_coins, &dust_coins, std::numeric_limits::max(), + anonymity, max_inputs_count); auto total_anon = m_used_total - fee; - unoptimize_amounts(&have_coins, &dust_coins); - max_inputs_count = (effective_median_size - ets) / get_maximum_tx_input_size(0); - select_max_outputs(&have_coins, &dust_coins, std::numeric_limits::max(), 0, max_inputs_count); + return_coins_to_index(&pretty_coins, &non_pretty_coins, &dust_coins); + max_inputs_count = (max_transaction_size - ets) / get_maximum_tx_input_size(min_anonymity); + select_max_outputs( + &pretty_coins, &non_pretty_coins, &dust_coins, std::numeric_limits::max(), 0, max_inputs_count); auto total_zero_anon = m_used_total - fee; std::string msg = - "Transaction with desired amount is too big. Max amount you can send with requested anonymity is " + + "Transaction with desired amount is too big (cannot fit in block). Max amount you can send with requested anonymity is " + m_currency.format_amount(total_anon) + " (" + m_currency.format_amount(total_zero_anon) + - " with zero anonymity)"; + " with anonymity " + common::to_string(min_anonymity) + ")"; throw api::walletd::CreateTransaction::ErrorTransactionTooBig(msg, total_anon, total_zero_anon); } if (fee + change_dust_fee >= size_fee) { @@ -304,30 +309,31 @@ void UnspentSelector::select_optimal_outputs(Height block_height, Timestamp bloc return; } fee = ((size_fee - change_dust_fee + dust_threshold - 1) / dust_threshold) * dust_threshold; - unoptimize_amounts(&have_coins, &dust_coins); + return_coins_to_index(&pretty_coins, &non_pretty_coins, &dust_coins); } } -void UnspentSelector::create_have_coins(Height block_height, Timestamp block_time, Height confirmed_height, - HaveCoins *have_coins, DustCoins *dust_coins, size_t *max_digit) { +void UnspentSelector::create_coin_index( + PrettyCoins *pretty_coins, NonPrettyCoins *non_pretty_coins, NonPrettyCoins *dust_coins, size_t *max_digit) { *max_digit = 0; + // We wish our coin index reversed, because we use pop_back to get coins from there for (auto uit = m_unspents.rbegin(); uit != m_unspents.rend(); ++uit) { api::Output &un = *uit; - if (un.height >= confirmed_height) // unconfirmed - continue; - if (!m_currency.is_transaction_spend_time_unlocked(un.unlock_block_or_timestamp, block_height, block_time)) - continue; - if (!m_currency.is_dust(un.amount)) { - Amount am = un.amount; - size_t digit = 0; - while (am > 9) { - digit += 1; - am /= 10; - } + // if (m_currency.is_dust(un.amount)) { + // (*dust_coins)[un.amount].push_back(un); + // continue; + // } + Amount am = un.amount; + size_t digit = 0; + while (am % 10 == 0) { + digit += 1; + am /= 10; + } + if (am <= 9) { *max_digit = std::max(*max_digit, digit); - (*have_coins)[digit][static_cast(am)].push_back(un); + (*pretty_coins)[digit][static_cast(am)].push_back(un); } else - (*dust_coins)[un.amount].push_back(un); + (*non_pretty_coins)[un.amount].push_back(un); } } @@ -339,28 +345,34 @@ void UnspentSelector::combine_optimized_unspents() { m_optimization_unspents.clear(); } -void UnspentSelector::unoptimize_amounts(HaveCoins *have_coins, DustCoins *dust_coins) { +void UnspentSelector::return_coins_to_index( + PrettyCoins *pretty_coins, NonPrettyCoins *non_pretty_coins, NonPrettyCoins *dust_coins) { // First remove all optimized coins. for (auto &&un : m_optimization_unspents) { m_used_total -= un.amount; m_inputs_count -= 1; - if (!un.dust) { - Amount am = un.amount; - size_t digit = 0; - while (am > 9) { - digit += 1; - am /= 10; - } - (*have_coins)[digit][static_cast(am)].push_back(un); - } else - (*dust_coins)[un.amount].push_back(un); + // if (un.dust) { + // (*dust_coins)[un.amount].push_back(std::move(un)); + // continue; + // } + Amount am = un.amount; + size_t digit = 0; + while (am % 10 == 0) { + digit += 1; + am /= 10; + } + if (am <= 9) + (*pretty_coins)[digit][static_cast(am)].push_back(std::move(un)); + else + (*non_pretty_coins)[un.amount].push_back(std::move(un)); } m_optimization_unspents.clear(); } -void UnspentSelector::optimize_amounts(HaveCoins *have_coins, size_t max_digit, Amount total_amount) { - m_log(logging::INFO) << "Sub optimizing amount=" << fake_large + total_amount - m_used_total - << " total_amount=" << total_amount << " used_total=" << m_used_total << std::endl; +void UnspentSelector::optimize_amounts(PrettyCoins *pretty_coins, size_t max_digit, Amount total_amount) { + if (detailed_output) + m_log(logging::INFO) << "Sub optimizing amount=" << fake_large + total_amount - m_used_total + << " total_amount=" << total_amount << " used_total=" << m_used_total << std::endl; Amount digit_amount = 1; for (size_t digit = 0; digit != max_digit + 1; ++digit, digit_amount *= 10) { if (m_used_total >= total_amount && digit_amount > m_used_total) // No optimization far beyond requested sum @@ -368,8 +380,8 @@ void UnspentSelector::optimize_amounts(HaveCoins *have_coins, size_t max_digit, Amount am = 10 - ((fake_large + total_amount + digit_amount - 1 - m_used_total) / digit_amount) % 10; if (am == 10) continue; - auto dit = have_coins->find(digit); - if (dit == have_coins->end()) // No coins for digit + auto dit = pretty_coins->find(digit); + if (dit == pretty_coins->end()) // No coins for digit continue; size_t best_two_counts[2] = {}; size_t best_weight = 0; @@ -384,20 +396,21 @@ void UnspentSelector::optimize_amounts(HaveCoins *have_coins, size_t max_digit, } } if (best_weight != 0) { - m_log(logging::INFO) << "Found pair for digit=" << digit << " am=" << 10 - am << " coins=(" - << best_two_counts[0] << ", " << best_two_counts[1] << ") sum weight=" << best_weight - << std::endl; + if (detailed_output) + m_log(logging::INFO) << "Found pair for digit=" << digit << " am=" << 10 - am << " coins=(" + << best_two_counts[0] << ", " << best_two_counts[1] + << ") sum weight=" << best_weight << std::endl; for (size_t i = 0; i != 2; ++i) { auto &uns = dit->second[best_two_counts[i]]; auto &un = uns.back(); - m_optimization_unspents.push_back(un); m_used_total += un.amount; m_inputs_count += 1; + m_optimization_unspents.push_back(std::move(un)); uns.pop_back(); if (uns.empty()) dit->second.erase(best_two_counts[i]); if (dit->second.empty()) - have_coins->erase(dit); + pretty_coins->erase(dit); } continue; } @@ -412,31 +425,37 @@ void UnspentSelector::optimize_amounts(HaveCoins *have_coins, size_t max_digit, best_single = ait.first; } if (best_single != 0) { - m_log(logging::INFO) << "Found single for digit=" << digit << " am=" << 10 - am << " coin=" << best_single - << " weight=" << best_weight << std::endl; + if (detailed_output) + m_log(logging::INFO) << "Found single for digit=" << digit << " am=" << 10 - am + << " coin=" << best_single << " weight=" << best_weight << std::endl; auto &uns = dit->second[best_single]; auto &un = uns.back(); - m_optimization_unspents.push_back(un); m_used_total += un.amount; m_inputs_count += 1; + m_optimization_unspents.push_back(std::move(un)); uns.pop_back(); if (uns.empty()) dit->second.erase(best_single); if (dit->second.empty()) - have_coins->erase(dit); + pretty_coins->erase(dit); continue; } - m_log(logging::INFO) << "Found nothing for digit=" << digit << std::endl; + if (detailed_output) + m_log(logging::INFO) << "Found nothing for digit=" << digit << std::endl; } - m_log(logging::INFO) << "Sub optimized used_total=" << m_used_total << " for total=" << total_amount << std::endl; + if (detailed_output) + m_log(logging::INFO) << "Sub optimized used_total=" << m_used_total << " for total=" << total_amount + << std::endl; } -bool UnspentSelector::select_optimal_outputs(HaveCoins *have_coins, DustCoins *dust_coins, size_t max_digit, - Amount total_amount, size_t anonymity, size_t optimization_count, bool small_optimizations) { +bool UnspentSelector::select_optimal_outputs(PrettyCoins *pretty_coins, NonPrettyCoins *non_pretty_coins, + NonPrettyCoins *dust_coins, size_t max_digit, Amount total_amount, size_t anonymity, size_t optimization_count, + bool small_optimizations) { // Optimize for roundness of used_total - total_amount; // [digit:size:outputs] - m_log(logging::INFO) << "Optimizing amount=" << fake_large + total_amount - m_used_total - << " total_amount=" << total_amount << " used_total=" << m_used_total << std::endl; + if (detailed_output) + m_log(logging::INFO) << "Optimizing amount=" << fake_large + total_amount - m_used_total + << " total_amount=" << total_amount << " used_total=" << m_used_total << std::endl; if (anonymity == 0) { if (m_used_total < total_amount) { // Find smallest dust coin >= total_amount - used_total, it can be very @@ -444,10 +463,11 @@ bool UnspentSelector::select_optimal_outputs(HaveCoins *have_coins, DustCoins *d auto duit = dust_coins->lower_bound(total_amount - m_used_total); if (duit != dust_coins->end()) { auto &un = duit->second.back(); - m_log(logging::INFO) << "Found single large dust coin=" << un.amount << std::endl; - m_optimization_unspents.push_back(un); + if (detailed_output) + m_log(logging::INFO) << "Found single large dust coin=" << un.amount << std::endl; m_used_total += un.amount; m_inputs_count += 1; + m_optimization_unspents.push_back(std::move(un)); duit->second.pop_back(); if (duit->second.empty()) dust_coins->erase(duit); @@ -457,21 +477,36 @@ bool UnspentSelector::select_optimal_outputs(HaveCoins *have_coins, DustCoins *d while (m_used_total < total_amount && !dust_coins->empty() && optimization_count >= 1) { auto duit = --dust_coins->end(); auto &un = duit->second.back(); - m_log(logging::INFO) << "Found optimization dust coin=" << un.amount << std::endl; - m_optimization_unspents.push_back(un); + if (detailed_output) + m_log(logging::INFO) << "Found optimization dust coin=" << un.amount << std::endl; m_used_total += un.amount; m_inputs_count += 1; optimization_count -= 1; + m_optimization_unspents.push_back(std::move(un)); duit->second.pop_back(); if (duit->second.empty()) dust_coins->erase(duit); } } + // Fill with non-pretty coins, but no more than K coins. + while (m_used_total < total_amount && !non_pretty_coins->empty() && optimization_count >= 1) { + auto duit = --non_pretty_coins->end(); + auto &un = duit->second.back(); + if (detailed_output) + m_log(logging::INFO) << "Found optimization non-pretty coin=" << un.amount << std::endl; + m_used_total += un.amount; + m_inputs_count += 1; + optimization_count -= 1; + m_optimization_unspents.push_back(std::move(un)); + duit->second.pop_back(); + if (duit->second.empty()) + non_pretty_coins->erase(duit); + } // Add coins from large stacks, up to optimization_count while (optimization_count >= 10) { size_t best_weight = STACK_OPTIMIZATION_THRESHOLD; std::vector *best_stack = nullptr; - for (auto &hit : *have_coins) + for (auto &hit : *pretty_coins) for (auto &ait : hit.second) if (ait.second.size() > best_weight) { best_weight = ait.second.size(); @@ -481,38 +516,40 @@ bool UnspentSelector::select_optimal_outputs(HaveCoins *have_coins, DustCoins *d break; for (int i = 0; i != 10; ++i) { auto &un = best_stack->back(); - m_log(logging::INFO) << "Found optimization stack for coin=" << un.amount << std::endl; - m_optimization_unspents.push_back(un); + if (detailed_output) + m_log(logging::INFO) << "Found optimization stack for coin=" << un.amount << std::endl; m_used_total += un.amount; m_inputs_count += 1; optimization_count -= 1; + m_optimization_unspents.push_back(std::move(un)); best_stack->pop_back(); // Will never become empty because of threshold } } - optimize_amounts(have_coins, max_digit, total_amount); + optimize_amounts(pretty_coins, max_digit, total_amount); if (m_used_total >= total_amount) return true; // Find smallest coin >= total_amount - used_total bool found = false; Amount digit_amount = 1; for (size_t digit = 0; !found && digit != max_digit + 1; ++digit, digit_amount *= 10) { - auto dit = have_coins->find(digit); - if (dit == have_coins->end()) // No coins for digit + auto dit = pretty_coins->find(digit); + if (dit == pretty_coins->end()) // No coins for digit continue; for (auto ait = dit->second.begin(); ait != dit->second.end(); ++ait) if (!ait->second.empty() && ait->first * digit_amount >= total_amount - m_used_total) { - m_log(logging::INFO) << "Found single large coin for digit=" << digit << " coin=" << ait->first - << std::endl; + if (detailed_output) + m_log(logging::INFO) << "Found single large coin for digit=" << digit << " coin=" << ait->first + << std::endl; auto &uns = dit->second[ait->first]; auto &un = uns.back(); - m_optimization_unspents.push_back(un); m_used_total += un.amount; m_inputs_count += 1; + m_optimization_unspents.push_back(std::move(un)); uns.pop_back(); if (uns.empty()) dit->second.erase(ait); if (dit->second.empty()) - have_coins->erase(dit); + pretty_coins->erase(dit); found = true; break; } @@ -520,53 +557,74 @@ bool UnspentSelector::select_optimal_outputs(HaveCoins *have_coins, DustCoins *d if (m_used_total >= total_amount) return true; // Use largest coins (including dust if anonymity == 0) until amount satisfied - unoptimize_amounts(have_coins, dust_coins); - select_max_outputs(have_coins, dust_coins, total_amount, anonymity, std::numeric_limits::max()); + return_coins_to_index(pretty_coins, non_pretty_coins, dust_coins); + select_max_outputs( + pretty_coins, non_pretty_coins, dust_coins, total_amount, anonymity, std::numeric_limits::max()); if (small_optimizations) - optimize_amounts(have_coins, max_digit, total_amount); + optimize_amounts(pretty_coins, max_digit, total_amount); return m_used_total >= total_amount; } -void UnspentSelector::select_max_outputs( - HaveCoins *have_coins, DustCoins *dust_coins, Amount total_amount, size_t anonymity, size_t max_inputs_count) { +void UnspentSelector::select_max_outputs(PrettyCoins *pretty_coins, NonPrettyCoins *non_pretty_coins, + NonPrettyCoins *dust_coins, Amount total_amount, size_t anonymity, size_t max_inputs_count) { while (m_used_total < total_amount && m_inputs_count < max_inputs_count) { - if (have_coins->empty() && (anonymity != 0 || dust_coins->empty())) + if (pretty_coins->empty() && non_pretty_coins->empty() && (anonymity != 0 || dust_coins->empty())) return; Amount ha_amount = 0; + Amount np_amount = 0; Amount du_amount = 0; - if (!have_coins->empty()) { - auto dit = --have_coins->end(); + if (!pretty_coins->empty()) { + auto dit = --pretty_coins->end(); auto ait = --dit->second.end(); ha_amount = ait->second.back().amount; } + if (!non_pretty_coins->empty()) { + auto duit = --non_pretty_coins->end(); + np_amount = duit->second.back().amount; + } if (anonymity == 0 && !dust_coins->empty()) { auto duit = --dust_coins->end(); du_amount = duit->second.back().amount; } - if (ha_amount > du_amount) { - auto dit = --have_coins->end(); + if (ha_amount > np_amount && ha_amount > du_amount) { + auto dit = --pretty_coins->end(); auto ait = --dit->second.end(); auto &uns = ait->second; auto &un = uns.back(); - m_log(logging::INFO) << "Found filler coin=" << un.amount << std::endl; - m_optimization_unspents.push_back(un); + if (detailed_output) + m_log(logging::INFO) << "Found filler coin=" << un.amount << std::endl; m_used_total += un.amount; m_inputs_count += 1; + m_optimization_unspents.push_back(std::move(un)); uns.pop_back(); if (uns.empty()) dit->second.erase(ait); if (dit->second.empty()) - have_coins->erase(dit); - } else { - auto duit = --dust_coins->end(); + pretty_coins->erase(dit); + continue; + } + if (np_amount > ha_amount && np_amount > du_amount) { + auto duit = --non_pretty_coins->end(); auto &un = duit->second.back(); - m_log(logging::INFO) << "Found filler dust coin=" << un.amount << std::endl; - m_optimization_unspents.push_back(un); + if (detailed_output) + m_log(logging::INFO) << "Found filler non-pretty coin=" << un.amount << std::endl; m_used_total += un.amount; m_inputs_count += 1; + m_optimization_unspents.push_back(std::move(un)); duit->second.pop_back(); if (duit->second.empty()) - dust_coins->erase(duit); + non_pretty_coins->erase(duit); + continue; } + auto duit = --dust_coins->end(); + auto &un = duit->second.back(); + if (detailed_output) + m_log(logging::INFO) << "Found filler dust coin=" << un.amount << std::endl; + m_used_total += un.amount; + m_inputs_count += 1; + m_optimization_unspents.push_back(std::move(un)); + duit->second.pop_back(); + if (duit->second.empty()) + dust_coins->erase(duit); } } diff --git a/src/Core/TransactionBuilder.hpp b/src/Core/TransactionBuilder.hpp index cb7913a9..403659cb 100644 --- a/src/Core/TransactionBuilder.hpp +++ b/src/Core/TransactionBuilder.hpp @@ -3,101 +3,88 @@ #pragma once -#include #include "CryptoNote.hpp" -#include "TransactionExtra.hpp" -#include "Wallet.hpp" -#include "crypto/chacha8.hpp" +#include "logging/LoggerMessage.hpp" #include "rpc_api.hpp" -namespace bytecoin { +namespace cn { class Wallet; class Currency; +class WalletStateBasic; class TransactionBuilder { +public: Transaction m_transaction; struct InputDesc { std::vector outputs; size_t real_output_index = 0; - KeyPair eph_keys; - KeyInput input; - static bool less_amount(const InputDesc &a, const InputDesc &b) { return a.input.amount < b.input.amount; } + // KeyPair eph_keys; + // InputKey input; + static bool less_amount(const InputDesc &a, const InputDesc &b) { + return a.outputs.at(a.real_output_index).amount < b.outputs.at(b.real_output_index).amount; + } }; std::vector m_input_descs; struct OutputDesc { Amount amount; - AccountPublicAddress addr; + AccountAddress addr; static bool less_amount(const OutputDesc &a, const OutputDesc &b) { return a.amount < b.amount; } }; std::vector m_output_descs; - // TransactionExtra m_extra; - Amount m_outputs_amount = 0; - Amount m_inputs_amount = 0; - -public: - explicit TransactionBuilder(const Currency &, BlockOrTimestamp); - - void set_payment_id(const Hash &); - // void set_extra_nonce(const BinaryArray &); // before calling, make sure mix_outputs do not contain real_output... - size_t add_input( - const AccountKeys &sender_keys, api::Output real_output, const std::vector &mix_outputs); - size_t add_output(uint64_t amount, const AccountPublicAddress &to); - - Amount get_outputs_amount() const { return m_outputs_amount; } - Amount get_inputs_amount() const { return m_inputs_amount; } + void add_input(const std::vector &mix_outputs, size_t real_output_index); + void add_output(uint64_t amount, const AccountAddress &to); - Transaction sign(const Hash &tx_derivation_seed); + Transaction sign( + const WalletStateBasic &wallet_state, Wallet *wallet, const std::set *only_records); - BinaryArray generate_history(const crypto::chacha8_key &history_key) const; + static KeyPair transaction_keys_from_seed(const Hash &tx_inputs_hash, const Hash &tx_derivation_seed); + static KeyPair deterministic_keys_from_seed( + const Hash &tx_inputs_hash, const Hash &tx_derivation_seed, const BinaryArray &add); + static KeyPair deterministic_keys_from_seed( + const TransactionPrefix &tx, const Hash &tx_derivation_seed, const BinaryArray &add); - static KeyPair deterministic_keys_from_seed(const Hash &tx_inputs_hash, const Hash &tx_derivation_seed); - static KeyPair deterministic_keys_from_seed(const TransactionPrefix &tx, const Hash &tx_derivation_seed); - static bool generate_key_image_helper(const AccountKeys &ack, const PublicKey &tx_public_key, - size_t real_output_index, KeyPair &in_ephemeral, KeyImage &ki); - static bool derive_public_key( - const AccountPublicAddress &to, const SecretKey &tx_key, size_t output_index, PublicKey &ephemeral_key); - static std::vector absolute_output_offsets_to_relative(const std::vector &off); + static OutputKey create_output(const AccountAddress &to, const SecretKey &tx_secret_key, const Hash &tx_inputs_hash, + size_t output_index, const Hash &output_secret); }; class UnspentSelector { logging::LoggerRef m_log; const Currency &m_currency; typedef std::vector Unspents; - typedef std::map>> HaveCoins; - typedef std::map> DustCoins; + typedef std::map>> PrettyCoins; + typedef std::map> NonPrettyCoins; Unspents m_unspents; Unspents m_used_unspents; Unspents m_optimization_unspents; - void create_have_coins(Height block_height, Timestamp block_time, Height confirmed_height, HaveCoins *have_coins, - DustCoins *dust_coins, size_t *max_digit); - void unoptimize_amounts(HaveCoins *have_coins, DustCoins *dust_coins); - void optimize_amounts(HaveCoins *have_coins, size_t max_digit, Amount total_amount); + void create_coin_index( + PrettyCoins *pretty_coins, NonPrettyCoins *non_pretty_coins, NonPrettyCoins *dust_coins, size_t *max_digit); + void return_coins_to_index(PrettyCoins *pretty_coins, NonPrettyCoins *non_pretty_coins, NonPrettyCoins *dust_coins); + void optimize_amounts(PrettyCoins *pretty_coins, size_t max_digit, Amount total_amount); void combine_optimized_unspents(); Amount m_used_total = 0; size_t m_inputs_count = 0; std::vector m_ra_amounts; - bool select_optimal_outputs(HaveCoins *have_coins, DustCoins *dust_coins, size_t max_digit, Amount amount, - size_t anonymity, size_t optimization_count, bool small_optimizations); - void select_max_outputs( - HaveCoins *have_coins, DustCoins *dust_coins, Amount total_amount, size_t anonymity, size_t max_inputs_count); + bool select_optimal_outputs(PrettyCoins *pretty_coins, NonPrettyCoins *non_pretty_coins, NonPrettyCoins *dust_coins, + size_t max_digit, Amount amount, size_t anonymity, size_t optimization_count, bool small_optimizations); + void select_max_outputs(PrettyCoins *pretty_coins, NonPrettyCoins *non_pretty_coins, NonPrettyCoins *dust_coins, + Amount total_amount, size_t anonymity, size_t max_inputs_count); public: explicit UnspentSelector(logging::ILogger &logger, const Currency ¤cy, Unspents &&unspents); void reset(Unspents &&unspents); - void add_mixed_inputs(const SecretKey &view_secret_key, const Wallet *wallet, - const std::unordered_map &wallet_records, TransactionBuilder *builder, - uint32_t anonymity, api::bytecoind::GetRandomOutputs::Response &&ra_response); + size_t add_mixed_inputs( + TransactionBuilder *builder, size_t anonymity, api::cnd::GetRandomOutputs::Response &&ra_response); // if receiver_fee == nullptr, fee will be subtracted from change - void select_optimal_outputs(Height block_height, Timestamp block_time, Height confirmed_height, - size_t effective_median_size, size_t anonymity, Amount total_amount, size_t total_outputs, Amount fee_per_byte, - std::string optimization_level, Amount *change, Amount *receiver_fee); + void select_optimal_outputs(size_t max_transaction_size, size_t anonymity, size_t min_anonymity, + Amount total_amount, size_t total_outputs, Amount fee_per_byte, std::string optimization_level, Amount *change, + Amount *receiver_fee); Amount get_used_total() const { return m_used_total; } const std::vector &get_ra_amounts() const { return m_ra_amounts; } }; -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/TransactionExtra.cpp b/src/Core/TransactionExtra.cpp index a19d0e2c..2e9602c4 100644 --- a/src/Core/TransactionExtra.cpp +++ b/src/Core/TransactionExtra.cpp @@ -6,10 +6,11 @@ #include "CryptoNoteTools.hpp" #include "common/MemoryStreams.hpp" #include "common/StringTools.hpp" +#include "common/Varint.hpp" #include "seria/BinaryInputStream.hpp" #include "seria/BinaryOutputStream.hpp" -using namespace bytecoin; +using namespace cn; template bool set_field_good(const T &, U &) { @@ -28,12 +29,12 @@ bool find_field_in_extra(const BinaryArray &extra, T &field) { seria::BinaryInputStream ar(iss); while (!iss.empty()) { - int c = common::read(iss); + int c = iss.read_byte(); switch (c) { case TransactionExtraPadding::tag: { size_t size = 1; // tag is itself '0', counts towards max count for (; !iss.empty() && size <= TransactionExtraPadding::MAX_COUNT; ++size) { - if (common::read(iss) != 0) + if (iss.read_byte() != 0) return false; // all bytes should be zero } if (size > TransactionExtraPadding::MAX_COUNT) @@ -51,7 +52,7 @@ bool find_field_in_extra(const BinaryArray &extra, T &field) { } case TransactionExtraNonce::tag: { TransactionExtraNonce extra_nonce; - uint8_t size = common::read(iss); + uint8_t size = iss.read_byte(); // TODO - turn into varint <= 127? extra_nonce.nonce.resize(size); iss.read(extra_nonce.nonce.data(), extra_nonce.nonce.size()); // We have some base transactions (like in blocks 558479, 558984) @@ -63,11 +64,30 @@ bool find_field_in_extra(const BinaryArray &extra, T &field) { } case TransactionExtraMergeMiningTag::tag: { TransactionExtraMergeMiningTag mm_tag; - ser(mm_tag, ar); + std::string field_data; + ser(field_data, ar); + common::MemoryInputStream stream(field_data.data(), field_data.size()); + seria::BinaryInputStream input(stream); + ser(mm_tag, input); if (set_field_good(mm_tag, field)) return true; break; } + case TransactionExtraBlockCapacityVote::tag: { + TransactionExtraBlockCapacityVote bsv; + std::string field_data; + ser(field_data, ar); + common::MemoryInputStream stream(field_data.data(), field_data.size()); + seria::BinaryInputStream input(stream); + ser(bsv, input); + if (set_field_good(bsv, field)) + return true; + break; + } + default: { // We hope to skip unknown tags + std::string field_data; + ser(field_data, ar); + } } } } catch (std::exception &) { @@ -75,44 +95,60 @@ bool find_field_in_extra(const BinaryArray &extra, T &field) { return false; // Not found } -PublicKey bytecoin::extra_get_transaction_public_key(const BinaryArray &tx_extra) { +PublicKey cn::extra_get_transaction_public_key(const BinaryArray &tx_extra) { TransactionExtraPublicKey pub_key_field; if (!find_field_in_extra(tx_extra, pub_key_field)) return PublicKey{}; return pub_key_field.public_key; } -void bytecoin::extra_add_transaction_public_key(BinaryArray &tx_extra, const PublicKey &tx_pub_key) { +void cn::extra_add_transaction_public_key(BinaryArray &tx_extra, const PublicKey &tx_pub_key) { tx_extra.push_back(TransactionExtraPublicKey::tag); common::append(tx_extra, std::begin(tx_pub_key.data), std::end(tx_pub_key.data)); } -void bytecoin::extra_add_nonce(BinaryArray &tx_extra, const BinaryArray &extra_nonce) { +void cn::extra_add_nonce(BinaryArray &tx_extra, const BinaryArray &extra_nonce) { if (extra_nonce.size() > TransactionExtraNonce::MAX_COUNT) throw std::runtime_error("Extra nonce cannot be > " + common::to_string(TransactionExtraNonce::MAX_COUNT)); tx_extra.push_back(TransactionExtraNonce::tag); tx_extra.push_back(static_cast(extra_nonce.size())); - common::append(tx_extra, extra_nonce.begin(), extra_nonce.end()); + common::append(tx_extra, extra_nonce); } -void bytecoin::extra_add_merge_mining_tag(BinaryArray &tx_extra, const TransactionExtraMergeMiningTag &mm_tag) { - BinaryArray blob = seria::to_binary(mm_tag); +void cn::extra_add_merge_mining_tag(BinaryArray &tx_extra, const TransactionExtraMergeMiningTag &field) { + BinaryArray blob = seria::to_binary(field); tx_extra.push_back(TransactionExtraMergeMiningTag::tag); - common::append(tx_extra, blob.begin(), blob.end()); + common::append(tx_extra, common::get_varint_data(blob.size())); + common::append(tx_extra, blob); +} + +bool cn::extra_get_merge_mining_tag(const BinaryArray &tx_extra, TransactionExtraMergeMiningTag &field) { + return find_field_in_extra(tx_extra, field); } -bool bytecoin::extra_get_merge_mining_tag(const BinaryArray &tx_extra, TransactionExtraMergeMiningTag &mm_tag) { - return find_field_in_extra(tx_extra, mm_tag); +void cn::extra_add_block_capacity_vote(BinaryArray &tx_extra, size_t block_capacity) { + BinaryArray blob = seria::to_binary(block_capacity); + tx_extra.push_back(TransactionExtraBlockCapacityVote::tag); + common::append(tx_extra, common::get_varint_data(blob.size())); + common::append(tx_extra, blob); } -void bytecoin::extra_add_payment_id(BinaryArray &tx_extra, const Hash &payment_id) { +bool cn::extra_get_block_capacity_vote(const BinaryArray &tx_extra, size_t *block_capacity) { + TransactionExtraBlockCapacityVote field; + if (!find_field_in_extra(tx_extra, field)) + return false; + *block_capacity = field.block_capacity; + return true; +} + +void cn::extra_add_payment_id(BinaryArray &tx_extra, const Hash &payment_id) { BinaryArray extra_nonce; extra_nonce.push_back(TransactionExtraNonce::PAYMENT_ID); common::append(extra_nonce, std::begin(payment_id.data), std::end(payment_id.data)); extra_add_nonce(tx_extra, extra_nonce); } -bool bytecoin::extra_get_payment_id(const BinaryArray &tx_extra, Hash &payment_id) { +bool cn::extra_get_payment_id(const BinaryArray &tx_extra, Hash &payment_id) { TransactionExtraNonce extra_nonce; if (!find_field_in_extra(tx_extra, extra_nonce)) return false; @@ -124,27 +160,11 @@ bool bytecoin::extra_get_payment_id(const BinaryArray &tx_extra, Hash &payment_i return true; } -static void do_serialize(TransactionExtraMergeMiningTag &tag, seria::ISeria &s) { - s.begin_object(); - uint64_t depth = static_cast(tag.depth); - seria_kv("depth", depth, s); - tag.depth = static_cast(depth); - seria_kv("merkle_root", tag.merkle_root, s); - s.end_object(); +void seria::ser_members(TransactionExtraMergeMiningTag &v, ISeria &s) { + seria_kv("depth", v.depth, s); + seria_kv("merkle_root", v.merkle_root, s); } -void seria::ser(TransactionExtraMergeMiningTag &v, ISeria &s) { - if (s.is_input()) { - std::string field; - ser(field, s); - common::MemoryInputStream stream(field.data(), field.size()); - seria::BinaryInputStream input(stream); - do_serialize(v, input); - } else { - std::string field; - common::StringOutputStream os(field); - seria::BinaryOutputStream output(os); - do_serialize(v, output); - ser(field, s); - } +void seria::ser_members(TransactionExtraBlockCapacityVote &v, ISeria &s) { + seria_kv("block_capacity", v.block_capacity, s); } diff --git a/src/Core/TransactionExtra.hpp b/src/Core/TransactionExtra.hpp index 1bbf3d76..f287b2f3 100644 --- a/src/Core/TransactionExtra.hpp +++ b/src/Core/TransactionExtra.hpp @@ -9,7 +9,7 @@ #include "CryptoNote.hpp" #include "seria/ISeria.hpp" -namespace bytecoin { +namespace cn { struct TransactionExtraPadding { size_t size = 0; @@ -23,7 +23,8 @@ struct TransactionExtraPublicKey { struct TransactionExtraNonce { BinaryArray nonce; - enum { tag = 0x02, MAX_COUNT = 255, PAYMENT_ID = 0x00 }; + enum { tag = 0x02, MAX_COUNT = 127, PAYMENT_ID = 0x00 }; + // We limit MAX_COUNT so that single byte (former) is equal to varint encoding (now) }; struct TransactionExtraMergeMiningTag { @@ -32,30 +33,33 @@ struct TransactionExtraMergeMiningTag { enum { tag = 0x03 }; }; -// tx_extra_field format, except tx_extra_padding and tx_extra_pub_key: +struct TransactionExtraBlockCapacityVote { + size_t block_capacity = 0; + enum { tag = 0x04 }; +}; + +// tx_extra_field format, except TransactionExtraPadding, TransactionExtraPublicKey: // varint tag; -// varint size; +// varint size | byte size // varint data[]; -// typedef boost::variant -// TransactionExtraField; - -// bool parse_transaction_extra(const BinaryArray &tx_extra, std::vector &tx_extra_fields); -// bool write_transaction_extra(BinaryArray &tx_extra, const std::vector &tx_extra_fields); PublicKey extra_get_transaction_public_key(const BinaryArray &tx_extra); void extra_add_transaction_public_key(BinaryArray &tx_extra, const PublicKey &tx_pub_key); void extra_add_nonce(BinaryArray &tx_extra, const BinaryArray &extra_nonce); -void extra_add_merge_mining_tag(BinaryArray &tx_extra, const TransactionExtraMergeMiningTag &mm_tag); -bool extra_get_merge_mining_tag(const BinaryArray &tx_extra, TransactionExtraMergeMiningTag &mm_tag); +void extra_add_merge_mining_tag(BinaryArray &tx_extra, const TransactionExtraMergeMiningTag &field); +bool extra_get_merge_mining_tag(const BinaryArray &tx_extra, TransactionExtraMergeMiningTag &field); + +void extra_add_block_capacity_vote(BinaryArray &tx_extra, size_t block_capacity); +bool extra_get_block_capacity_vote(const BinaryArray &tx_extra, size_t *block_capacity); void extra_add_payment_id(BinaryArray &tx_extra, const Hash &payment_id); bool extra_get_payment_id(const BinaryArray &tx_extra, Hash &payment_id); -} +} // namespace cn namespace seria { class ISeria; -void ser(bytecoin::TransactionExtraMergeMiningTag &v, ISeria &s); -} +void ser_members(cn::TransactionExtraMergeMiningTag &v, ISeria &s); +void ser_members(cn::TransactionExtraBlockCapacityVote &v, ISeria &s); +} // namespace seria diff --git a/src/Core/Wallet.cpp b/src/Core/Wallet.cpp index 1c584c9d..147d8841 100644 --- a/src/Core/Wallet.cpp +++ b/src/Core/Wallet.cpp @@ -1,33 +1,134 @@ // Copyright (c) 2012-2018, The CryptoNote developers, The Bytecoin developers. // Licensed under the GNU Lesser General Public License. See LICENSE for details. +#include +#include +#include +#include +#include #include "CryptoNoteTools.hpp" +#include "TransactionBuilder.hpp" #include "WalletSerializationV1.hpp" #include "WalletState.hpp" +#include "common/BIPs.hpp" +#include "common/CRC32.hpp" #include "common/Math.hpp" #include "common/MemoryStreams.hpp" #include "common/StringTools.hpp" #include "common/Varint.hpp" +#include "common/Words.hpp" #include "crypto/crypto.hpp" #include "http/JsonRpc.hpp" #include "platform/Files.hpp" #include "platform/PathTools.hpp" #include "platform/Time.hpp" +#include "seria/BinaryInputStream.hpp" +#include "seria/BinaryOutputStream.hpp" -using namespace bytecoin; +using namespace cn; + +BinaryArray &operator|=(BinaryArray &a, const BinaryArray &b) { + common::append(a, b); + return a; +} + +BinaryArray operator|(const BinaryArray &a, const BinaryArray &b) { + BinaryArray tmp(a); + return tmp |= b; +} + +BinaryArray &operator|=(BinaryArray &a, const std::string &b) { return a |= common::as_binary_array(b); } + +BinaryArray operator|(const BinaryArray &a, const std::string &b) { + BinaryArray tmp(a); + return tmp |= b; +} + +BinaryArray &operator|=(BinaryArray &a, const Hash &b) { + common::append(a, std::begin(b.data), std::end(b.data)); + return a; +} + +BinaryArray operator|(const BinaryArray &a, const Hash &b) { + BinaryArray tmp(a); + return tmp |= b; +} + +BinaryArray &operator|=(const Hash &b, BinaryArray &a) { + common::append(a, std::begin(b.data), std::end(b.data)); + return a; +} + +BinaryArray operator|(const Hash &a, const BinaryArray &b) { + BinaryArray tmp(std::begin(a.data), std::end(a.data)); + return tmp |= b; +} +BinaryArray operator|(const Hash &a, const std::string &b) { + BinaryArray tmp(std::begin(a.data), std::end(a.data)); + return tmp |= b; +} + +static std::string net_append(const std::string &net) { return net == "main" ? std::string() : "_" + net + "net"; } + +Wallet::Wallet(const Currency ¤cy, logging::ILogger &log, const std::string &path) + : m_currency(currency), m_log(log, "Wallet"), m_path(path) {} + +static Hash derive_from_seed_legacy(const Hash &seed, const std::string &append) { + BinaryArray seed_data = common::as_binary_array(append); + common::append(seed_data, std::begin(seed.data), std::end(seed.data)); + return crypto::cn_fast_hash(seed_data.data(), seed_data.size()); +} + +static Hash derive_from_seed(const Hash &seed, const std::string &append) { + BinaryArray seed_data = seed | append; + return crypto::cn_fast_hash(seed_data.data(), seed_data.size()); +} + +static Hash derive_from_key(const crypto::chacha_key &key, const std::string &append) { + BinaryArray seed_data = BinaryArray(key.data, key.data + sizeof(key.data)) | append; + common::append(seed_data, std::begin(key.data), std::end(key.data)); + return crypto::cn_fast_hash(seed_data.data(), seed_data.size()); +} + +AccountAddress Wallet::get_first_address() const { return record_to_address(m_wallet_records.at(0)); } + +std::string Wallet::get_cache_name() const { + Hash h = crypto::cn_fast_hash(m_view_public_key.data, sizeof(m_view_public_key.data)); + return common::pod_to_hex(h) + (is_view_only() ? "-view-only" : std::string()); +} + +bool Wallet::is_our_address(const AccountAddress &v_addr) const { + if (v_addr.type() != typeid(AccountAddressSimple)) + return false; + auto &addr = boost::get(v_addr); + auto rit = m_records_map.find(addr.spend_public_key); + if (m_view_public_key != addr.view_public_key || rit == m_records_map.end()) + return false; + return m_wallet_records.at(rit->second).spend_public_key == addr.spend_public_key; +} + +bool Wallet::get_look_ahead_record(WalletRecord &record, const PublicKey &spend_public_key) { + auto rit = m_records_map.find(spend_public_key); + if (rit == m_records_map.end()) + return false; + invariant(m_wallet_records.at(rit->second).spend_public_key == spend_public_key, ""); + record = m_wallet_records.at(rit->second); + create_look_ahead_records(rit->second + 1); + return true; +} static const uint8_t SERIALIZATION_VERSION_V2 = 6; static const size_t CHECK_KEYS_COUNT = 128; // >8 KB checked at start and end of file #pragma pack(push, 1) struct EncryptedWalletRecord { - crypto::chacha8_iv iv; + crypto::chacha_iv iv; // Secret key, public key and creation timestamp uint8_t data[sizeof(PublicKey) + sizeof(SecretKey) + sizeof(uint64_t)]{}; }; struct ContainerStoragePrefix { // We moved uint8_t version out of this struct, because with it other fields become unaligned - crypto::chacha8_iv next_iv; + crypto::chacha_iv next_iv; EncryptedWalletRecord encrypted_view_keys; }; // struct ContainerStorageWalletRecord { @@ -38,7 +139,7 @@ struct ContainerStoragePrefix { #pragma pack(pop) static void decrypt_key_pair( - const EncryptedWalletRecord &r, PublicKey &pk, SecretKey &sk, Timestamp &ct, const WalletKey &key) { + const EncryptedWalletRecord &r, PublicKey &pk, SecretKey &sk, Timestamp &ct, const crypto::chacha_key &key) { // ContainerStorageWalletRecord rec; unsigned char rec_data[sizeof(r.data)]{}; chacha8(r.data, sizeof(r.data), key, r.iv, rec_data); @@ -48,26 +149,27 @@ static void decrypt_key_pair( common::uint_le_from_bytes(rec_data + sizeof(PublicKey) + sizeof(SecretKey), sizeof(uint64_t))); } -static void encrypt_key_pair(EncryptedWalletRecord &r, PublicKey pk, SecretKey sk, Timestamp ct, const WalletKey &key) { +static void encrypt_key_pair( + EncryptedWalletRecord &r, PublicKey pk, SecretKey sk, Timestamp ct, const crypto::chacha_key &key) { unsigned char rec_data[sizeof(r.data)]{}; memcpy(rec_data, pk.data, sizeof(PublicKey)); memcpy(rec_data + sizeof(PublicKey), sk.data, sizeof(SecretKey)); common::uint_le_to_bytes(rec_data + sizeof(PublicKey) + sizeof(SecretKey), sizeof(uint64_t), ct); - r.iv = crypto::rand(); + r.iv = crypto::rand(); chacha8(&rec_data, sizeof(r.data), key, r.iv, r.data); } -size_t Wallet::wallet_file_size(size_t records) { +size_t WalletContainerStorage::wallet_file_size(size_t records) { return 1 + sizeof(ContainerStoragePrefix) + sizeof(uint64_t) * 2 + sizeof(EncryptedWalletRecord) * records; } -void Wallet::load_container_storage() { +void WalletContainerStorage::load_container_storage() { uint8_t version = 0; ContainerStoragePrefix prefix{}; unsigned char count_capacity_data[2 * sizeof(uint64_t)]{}; - file->read(&version, 1); - file->read(&prefix, sizeof(prefix)); - file->read(count_capacity_data, 2 * sizeof(uint64_t)); + m_file->read(&version, 1); + m_file->read(&prefix, sizeof(prefix)); + m_file->read(count_capacity_data, 2 * sizeof(uint64_t)); uint64_t f_item_capacity = common::uint_le_from_bytes(count_capacity_data, sizeof(uint64_t)); uint64_t f_item_count = common::uint_le_from_bytes(count_capacity_data + sizeof(uint64_t), sizeof(uint64_t)); @@ -87,7 +189,7 @@ void Wallet::load_container_storage() { throw Exception( api::WALLET_FILE_DECRYPT_ERROR, "Restored item count is too big " + common::to_string(item_count)); std::vector all_encrypted(item_count); - file->read(reinterpret_cast(all_encrypted.data()), sizeof(EncryptedWalletRecord) * item_count); + m_file->read(reinterpret_cast(all_encrypted.data()), sizeof(EncryptedWalletRecord) * item_count); bool tracking_mode = false; m_wallet_records.reserve(item_count); for (size_t i = 0; i != item_count; ++i) { @@ -117,11 +219,11 @@ void Wallet::load_container_storage() { m_records_map.insert(std::make_pair(wallet_record.spend_public_key, m_wallet_records.size())); m_wallet_records.push_back(wallet_record); } - auto file_size = file->seek(0, SEEK_END); + auto file_size = m_file->seek(0, SEEK_END); auto should_be_file_size = wallet_file_size(item_count); if (file_size > should_be_file_size) { // We truncate legacy wallet cache try { - file->truncate(should_be_file_size); + m_file->truncate(should_be_file_size); m_log(logging::WARNING) << "Truncated wallet cache legacy wallet file to size=" << should_be_file_size << std::endl; } catch (const std::exception &) { // probably read only, ignore @@ -129,13 +231,13 @@ void Wallet::load_container_storage() { } } -void Wallet::load_legacy_wallet_file() { +void WalletContainerStorage::load_legacy_wallet_file() { // m_wallet_records.clear(); // std::vector wallets_container; WalletSerializerV1 s(m_view_public_key, m_view_secret_key, m_wallet_records); - s.load(m_wallet_key, *file.get()); + s.load(m_wallet_key, *m_file.get()); // m_wallet_records.reserve() // m_first_record = wallets_container.at(0); @@ -145,58 +247,76 @@ void Wallet::load_legacy_wallet_file() { } } -Wallet::Wallet(logging::ILogger &log, const std::string &path, const std::string &password, bool create, - const std::string &import_keys) - : m_log(log, "Wallet"), m_path(path), m_password(password) { +WalletContainerStorage::WalletContainerStorage( + const Currency ¤cy, logging::ILogger &log, const std::string &path, const crypto::chacha_key &wallet_key) + : Wallet(currency, log, path) { + m_wallet_key = wallet_key; + load(); +} + +WalletContainerStorage::WalletContainerStorage(const Currency ¤cy, logging::ILogger &log, const std::string &path, + const std::string &password, const std::string &import_keys, Timestamp creation_timestamp) + : Wallet(currency, log, path) { crypto::CryptoNightContext cn_ctx; - m_wallet_key = generate_chacha8_key(cn_ctx, password); - if (create) { - try { - file.reset(new platform::FileStream(path, platform::FileStream::READ_EXISTING)); - } catch (const common::StreamError &) { - // file does not exist - } - if (file.get()) // opened ok - throw Exception(api::WALLET_FILE_EXISTS, - "Will not overwrite existing wallet - delete it first or specify another file " + path); - - if (import_keys.empty()) { - m_oldest_timestamp = platform::now_unix_timestamp(); - crypto::random_keypair(m_view_public_key, m_view_secret_key); - m_wallet_records.push_back(WalletRecord{}); - m_wallet_records.at(0).creation_timestamp = m_oldest_timestamp; - crypto::random_keypair(m_wallet_records.at(0).spend_public_key, m_wallet_records.at(0).spend_secret_key); - } else { - if (import_keys.size() != 256) - throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Imported keys should be exactly 128 hex bytes"); - WalletRecord record{}; - if (!common::pod_from_hex(import_keys.substr(0, 64), record.spend_public_key) || - !common::pod_from_hex(import_keys.substr(64, 64), m_view_public_key) || - !common::pod_from_hex(import_keys.substr(128, 64), record.spend_secret_key) || - !common::pod_from_hex(import_keys.substr(192, 64), m_view_secret_key)) - throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Imported keys should contain only hex bytes"); - if (!keys_match(m_view_secret_key, m_view_public_key)) - throw Exception( - api::WALLET_FILE_DECRYPT_ERROR, "Imported secret view key does not match corresponding public key"); - if (record.spend_secret_key != SecretKey{} && !keys_match(record.spend_secret_key, record.spend_public_key)) - throw Exception(api::WALLET_FILE_DECRYPT_ERROR, - "Imported secret spend key does not match corresponding public key"); - m_wallet_records.push_back(record); - m_oldest_timestamp = 0; // Alas, will scan entire blockchain - } - m_records_map.insert(std::make_pair(m_wallet_records.at(0).spend_public_key, 0)); - save_and_check(); + m_wallet_key = generate_chacha8_key(cn_ctx, password.data(), password.size()); + // try { + m_file.reset(new platform::FileStream(path, platform::O_CREATE_NEW)); + // } catch (const common::StreamError &) { + // file does not exist + // } + // if (file.get()) // opened ok + // throw Exception(api::WALLET_FILE_EXISTS, + // "Will not overwrite existing wallet - delete it first or specify another file " + path); + + if (import_keys.empty()) { + m_oldest_timestamp = platform::now_unix_timestamp(); // ignore creation_timestamp + crypto::random_keypair(m_view_public_key, m_view_secret_key); + m_wallet_records.push_back(WalletRecord{}); + m_wallet_records.at(0).creation_timestamp = m_oldest_timestamp; + crypto::random_keypair(m_wallet_records.at(0).spend_public_key, m_wallet_records.at(0).spend_secret_key); + } else { + if (import_keys.size() != 256) + throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Imported keys should be exactly 128 hex bytes"); + WalletRecord record{}; + record.creation_timestamp = creation_timestamp; + if (!common::pod_from_hex(import_keys.substr(0, 64), &record.spend_public_key) || + !common::pod_from_hex(import_keys.substr(64, 64), &m_view_public_key) || + !common::pod_from_hex(import_keys.substr(128, 64), &record.spend_secret_key) || + !common::pod_from_hex(import_keys.substr(192, 64), &m_view_secret_key)) + throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Imported keys should contain only hex bytes"); + if (!keys_match(m_view_secret_key, m_view_public_key)) + throw Exception( + api::WALLET_FILE_DECRYPT_ERROR, "Imported secret view key does not match corresponding public key"); + if (record.spend_secret_key != SecretKey{} && !keys_match(record.spend_secret_key, record.spend_public_key)) + throw Exception( + api::WALLET_FILE_DECRYPT_ERROR, "Imported secret spend key does not match corresponding public key"); + m_wallet_records.push_back(record); + m_oldest_timestamp = 0; // Alas, will scan entire blockchain } + m_records_map.insert(std::make_pair(m_wallet_records.at(0).spend_public_key, 0)); + save_and_check(); + load(); +} + +WalletContainerStorage::WalletContainerStorage( + const Currency ¤cy, logging::ILogger &log, const std::string &path, const std::string &password) + : Wallet(currency, log, path) { + crypto::CryptoNightContext cn_ctx; + m_wallet_key = generate_chacha8_key(cn_ctx, password.data(), password.size()); + load(); +} + +void WalletContainerStorage::load() { try { - file.reset(new platform::FileStream(path, platform::FileStream::READ_WRITE_EXISTING)); + m_file.reset(new platform::FileStream(m_path, platform::O_OPEN_EXISTING)); } catch (const common::StreamError &) { // Read-only media? - file.reset(new platform::FileStream(path, platform::FileStream::READ_EXISTING)); + m_file.reset(new platform::FileStream(m_path, platform::O_READ_EXISTING)); } uint8_t version = 0; - file->read(&version, 1); + m_file->read(&version, 1); if (version > SERIALIZATION_VERSION_V2) throw Exception(api::WALLET_FILE_UNKNOWN_VERSION, "Unknown version"); - file->seek(0, SEEK_SET); + m_file->seek(0, SEEK_SET); if (version < SERIALIZATION_VERSION_V2) { try { load_legacy_wallet_file(); @@ -207,7 +327,7 @@ Wallet::Wallet(logging::ILogger &log, const std::string &path, const std::string std::throw_with_nested(Exception( api::WALLET_FILE_DECRYPT_ERROR, std::string("Error decrypting wallet file ") + common::what(ex))); } - file.reset(); // Indicates legacy format + m_file.reset(); // Indicates legacy format try { save_and_check(); // We try to overwrite legacy format with new format m_log(logging::WARNING) << "Overwritten legacy wallet file with new data format" << std::endl; @@ -224,39 +344,16 @@ Wallet::Wallet(logging::ILogger &log, const std::string &path, const std::string seed_data.assign(std::begin(m_view_secret_key.data), std::end(m_view_secret_key.data)); common::append(seed_data, std::begin(m_wallet_records.at(0).spend_secret_key.data), std::end(m_wallet_records.at(0).spend_secret_key.data)); - m_seed = crypto::cn_fast_hash(seed_data.data(), seed_data.size()); - - // const unsigned char derivation_prefix[] = "tx_derivation"; - // seed_data.assign(derivation_prefix, derivation_prefix + sizeof(derivation_prefix) - 1); - // common::append(seed_data, std::begin(m_seed.data), std::end(m_seed.data)); - // m_tx_derivation_seed = crypto::cn_fast_hash(seed_data.data(), seed_data.size()); - m_tx_derivation_seed = derive_from_seed("tx_derivation"); - - m_coinbase_tx_derivation_seed = derive_from_seed("coinbase_tx_derivation"); - - // const unsigned char history_filename_prefix[] = "history_filename"; - // seed_data.assign(history_filename_prefix, history_filename_prefix + sizeof(history_filename_prefix) - - // 1); - // common::append(seed_data, std::begin(m_seed.data), std::end(m_seed.data)); - // m_history_filename_seed = crypto::cn_fast_hash(seed_data.data(), seed_data.size()); - m_history_filename_seed = derive_from_seed("history_filename"); - - // const unsigned char history_prefix[] = "history"; - // seed_data.assign(history_prefix, history_prefix + sizeof(history_prefix) - 1); - // common::append(seed_data, std::begin(m_seed.data), std::end(m_seed.data)); - // m_history_key = crypto::chacha8_key{crypto::cn_fast_hash(seed_data.data(), seed_data.size())}; - m_history_key = crypto::chacha8_key{derive_from_seed("history")}; + m_seed = crypto::cn_fast_hash(seed_data.data(), seed_data.size()); + m_tx_derivation_seed = derive_from_seed_legacy(m_seed, "tx_derivation"); + m_history_filename_seed = derive_from_seed_legacy(m_seed, "history_filename"); + m_history_key = crypto::chacha_key{derive_from_seed_legacy(m_seed, "history")}; } } -Hash Wallet::derive_from_seed(const std::string &append) { - BinaryArray seed_data(append.data(), append.data() + append.size()); - common::append(seed_data, std::begin(m_seed.data), std::end(m_seed.data)); - return crypto::cn_fast_hash(seed_data.data(), seed_data.size()); -} - -void Wallet::save(const std::string &export_path, const WalletKey &wallet_key, bool view_only) { - platform::FileStream f(export_path, platform::FileStream::TRUNCATE_READ_WRITE); +void WalletContainerStorage::save(const std::string &export_path, const crypto::chacha_key &wallet_key, bool view_only, + platform::OpenMode open_mode) const { + platform::FileStream f(export_path, open_mode); uint8_t version = SERIALIZATION_VERSION_V2; ContainerStoragePrefix prefix{}; @@ -278,7 +375,7 @@ void Wallet::save(const std::string &export_path, const WalletKey &wallet_key, b f.fsync(); } -BinaryArray Wallet::export_keys() const { +std::string WalletContainerStorage::export_keys() const { BinaryArray result; common::append(result, std::begin(m_wallet_records.at(0).spend_public_key.data), std::end(m_wallet_records.at(0).spend_public_key.data)); @@ -286,40 +383,32 @@ BinaryArray Wallet::export_keys() const { common::append(result, std::begin(m_wallet_records.at(0).spend_secret_key.data), std::end(m_wallet_records.at(0).spend_secret_key.data)); common::append(result, std::begin(m_view_secret_key.data), std::end(m_view_secret_key.data)); - return result; + return common::to_hex(result); } -void Wallet::save_and_check() { +void WalletContainerStorage::save_and_check() { const std::string tmp_path = m_path + ".tmp"; - save(tmp_path, m_wallet_key, false); + save(tmp_path, m_wallet_key, false, platform::O_CREATE_ALWAYS); - Wallet other(m_log.get_logger(), tmp_path, m_password); + WalletContainerStorage other(m_currency, m_log.get_logger(), tmp_path, m_wallet_key); if (*this != other) throw Exception(api::WALLET_FILE_WRITE_ERROR, "Error writing wallet file - records do not match"); - file.reset(); + m_file.reset(); if (!platform::atomic_replace_file(tmp_path, m_path)) throw Exception(api::WALLET_FILE_WRITE_ERROR, "Error replacing wallet file"); - std::swap(file, other.file); + std::swap(m_file, other.m_file); } -void Wallet::set_password(const std::string &password) { - m_password = password; +void WalletContainerStorage::set_password(const std::string &password) { crypto::CryptoNightContext cn_ctx; - m_wallet_key = generate_chacha8_key(cn_ctx, m_password); + m_wallet_key = generate_chacha8_key(cn_ctx, password.data(), password.size()); save_and_check(); } -void Wallet::export_wallet(const std::string &export_path, const std::string &new_password, bool view_only) { +void WalletContainerStorage::export_wallet(const std::string &export_path, const std::string &new_password, + bool view_only, bool view_outgoing_addresses) const { std::unique_ptr export_file; - try { - export_file.reset(new platform::FileStream(export_path, platform::FileStream::READ_EXISTING)); - } catch (const common::StreamError &) { - // file does not exist - } - if (export_file.get()) // opened ok - throw Exception(api::WALLET_FILE_EXISTS, - "Will not overwrite existing wallet - delete it first or specify another file " + export_path); for (const auto &rec : m_wallet_records) { if (rec.spend_secret_key != SecretKey{}) { if (!keys_match(rec.spend_secret_key, rec.spend_public_key)) @@ -332,31 +421,27 @@ void Wallet::export_wallet(const std::string &export_path, const std::string &ne } } crypto::CryptoNightContext cn_ctx; - auto new_wallet_key = generate_chacha8_key(cn_ctx, new_password); - save(export_path, new_wallet_key, view_only); + auto new_wallet_key = generate_chacha8_key(cn_ctx, new_password.data(), new_password.size()); + save(export_path, new_wallet_key, view_only, platform::O_CREATE_NEW); } -bool Wallet::operator==(const Wallet &other) const { +bool WalletContainerStorage::operator==(const WalletContainerStorage &other) const { return m_view_public_key == other.m_view_public_key && m_view_secret_key == other.m_view_secret_key && m_oldest_timestamp == other.m_oldest_timestamp && m_wallet_records == other.m_wallet_records; } -AccountPublicAddress Wallet::get_first_address() const { - return AccountPublicAddress{m_wallet_records.at(0).spend_public_key, m_view_public_key}; -} - -std::vector Wallet::generate_new_addresses( +std::vector WalletContainerStorage::generate_new_addresses( const std::vector &sks, Timestamp ct, Timestamp now, bool *rescan_from_ct) { std::vector result; if (is_view_only()) throw Exception(101, "Generate new addresses impossible for view-only wallet"); - if (!file.get()) { // Legacy format, now overwrite + if (!m_file.get()) { // Legacy format, now overwrite m_log(logging::WARNING) << "Creation of new addresses forces overwrite of legacy format wallet" << std::endl; save_and_check(); } *rescan_from_ct = false; size_t append_pos = wallet_file_size(m_wallet_records.size()); - file->seek(append_pos, SEEK_SET); + m_file->seek(append_pos, SEEK_SET); for (auto &&sk : sks) { WalletRecord record{}; if (sk == SecretKey{}) { @@ -368,7 +453,7 @@ std::vector Wallet::generate_new_addresses( } else { record.creation_timestamp = ct; record.spend_secret_key = sk; - if (!secret_key_to_public_key(sk, record.spend_public_key)) + if (!secret_key_to_public_key(sk, &record.spend_public_key)) throw Exception(101, "Imported keypair is invalid - sk=" + common::pod_to_hex(sk)); } auto rit = m_records_map.find(record.spend_public_key); @@ -386,19 +471,19 @@ std::vector Wallet::generate_new_addresses( EncryptedWalletRecord enc_record; encrypt_key_pair( enc_record, record.spend_public_key, record.spend_secret_key, record.creation_timestamp, m_wallet_key); - file->write(&enc_record, sizeof(enc_record)); + m_file->write(&enc_record, sizeof(enc_record)); result.push_back(record); } - file->fsync(); - file->seek(1 + sizeof(ContainerStoragePrefix), SEEK_SET); + m_file->fsync(); + m_file->seek(1 + sizeof(ContainerStoragePrefix), SEEK_SET); unsigned char count_capacity_data[sizeof(uint64_t)]{}; common::uint_le_to_bytes(count_capacity_data, sizeof(uint64_t), m_wallet_records.size()); - file->write(count_capacity_data, sizeof(uint64_t)); - file->write(count_capacity_data, sizeof(uint64_t)); + m_file->write(count_capacity_data, sizeof(uint64_t)); + m_file->write(count_capacity_data, sizeof(uint64_t)); - file->fsync(); + m_file->fsync(); if (*rescan_from_ct) { // We never write to the middle of the file m_log(logging::WARNING) << "Updating creation timestamp of existing addresses to " << ct << " in a wallet file (might take minutes for large wallets)..." << std::endl; @@ -407,8 +492,28 @@ std::vector Wallet::generate_new_addresses( return result; } -void Wallet::on_first_output_found(Timestamp ts) { - if (ts == 0 || m_oldest_timestamp != 0) // TODO - investigate why ts == 0 is possible +AccountAddress WalletContainerStorage::record_to_address(const WalletRecord &record) const { + return AccountAddressSimple{record.spend_public_key, m_view_public_key}; +} + +bool WalletContainerStorage::get_record(WalletRecord &record, const AccountAddress &v_addr) const { + if (v_addr.type() != typeid(AccountAddressSimple)) + return false; + auto &addr = boost::get(v_addr); + auto rit = m_records_map.find(addr.spend_public_key); + if (m_view_public_key != addr.view_public_key || rit == m_records_map.end()) + return false; + if (rit->second >= get_actual_records_count()) + return false; + invariant(m_wallet_records.at(rit->second).spend_public_key == addr.spend_public_key, ""); + record = m_wallet_records.at(rit->second); + return true; +} + +void WalletContainerStorage::on_first_output_found(Timestamp ts) { + if (m_currency.net != "main") + return; // Legacy format has not place for other nets + if (ts == 0 || m_oldest_timestamp != 0) return; m_oldest_timestamp = ts; for (auto &&rec : m_wallet_records) @@ -419,45 +524,49 @@ void Wallet::on_first_output_found(Timestamp ts) { save_and_check(); } -std::string Wallet::get_cache_name() const { - Hash h = crypto::cn_fast_hash(m_view_public_key.data, sizeof(m_view_public_key.data)); - return common::pod_to_hex(h) + (is_view_only() ? "-view-only" : std::string()); +void WalletContainerStorage::backup(const std::string &dst_name, const std::string &pass) const { + const std::string dst_history_name = dst_name + ".history"; + const std::string dst_payments_name = dst_name + ".payments"; + if (!platform::create_folder_if_necessary(dst_payments_name)) + throw std::runtime_error("Could not create folder for backup " + dst_payments_name); + if (!platform::create_folder_if_necessary(dst_history_name)) + throw std::runtime_error("Could not create folder for backup " + dst_history_name); + export_wallet(dst_name, pass, false, false); + for (const auto &file : platform::get_filenames_in_folder(get_payment_queue_folder())) { + platform::copy_file(get_payment_queue_folder() + "/" + file, dst_payments_name + "/" + file); + } + for (const auto &file : platform::get_filenames_in_folder(get_history_folder())) { + platform::copy_file(get_history_folder() + "/" + file, dst_history_name + "/" + file); + } } -bool Wallet::is_our_address(const AccountPublicAddress &addr) const { - auto rit = m_records_map.find(addr.spend_public_key); - if (m_view_public_key != addr.view_public_key || rit == m_records_map.end()) - return false; - return m_wallet_records.at(rit->second).spend_public_key == addr.spend_public_key; +std::string WalletContainerStorage::get_history_folder() const { + return m_path + ".history" + net_append(m_currency.net); } -bool Wallet::get_record(WalletRecord &record, const AccountPublicAddress &addr) const { - auto rit = m_records_map.find(addr.spend_public_key); - if (m_view_public_key != addr.view_public_key || rit == m_records_map.end()) - return false; - if (m_wallet_records.at(rit->second).spend_public_key != addr.spend_public_key) - return false; // TODO - invariant - record = m_wallet_records.at(rit->second); - return true; +std::string WalletContainerStorage::get_payment_queue_folder() const { + return m_path + ".payments" + net_append(m_currency.net); } -bool Wallet::save_history(const Hash &bid, const History &used_addresses) const { +bool WalletContainerStorage::save_history(const Hash &tid, const History &used_addresses) { std::string history_folder = get_history_folder(); if (!platform::create_folders_if_necessary(history_folder)) return false; - crypto::chacha8_iv iv = crypto::rand(); + if (used_addresses.empty()) + return true; // saved empty history :) + crypto::chacha_iv iv = crypto::rand(); BinaryArray data; - for (auto &&to : used_addresses) { - common::append(data, std::begin(to.view_public_key.data), std::end(to.view_public_key.data)); - common::append(data, std::begin(to.spend_public_key.data), std::end(to.spend_public_key.data)); + for (auto &&addr : used_addresses) { + common::append(data, std::begin(addr.view_public_key.data), std::end(addr.view_public_key.data)); + common::append(data, std::begin(addr.spend_public_key.data), std::end(addr.spend_public_key.data)); } BinaryArray encrypted_data; encrypted_data.resize(data.size(), 0); crypto::chacha8(data.data(), data.size(), m_history_key, iv, encrypted_data.data()); encrypted_data.insert(encrypted_data.begin(), std::begin(iv.data), std::end(iv.data)); - BinaryArray filename_data(std::begin(bid.data), std::end(bid.data)); + BinaryArray filename_data(std::begin(tid.data), std::end(tid.data)); common::append(filename_data, std::begin(m_history_filename_seed.data), std::end(m_history_filename_seed.data)); Hash filename_hash = crypto::cn_fast_hash(filename_data.data(), filename_data.size()); @@ -466,27 +575,796 @@ bool Wallet::save_history(const Hash &bid, const History &used_addresses) const encrypted_data.data(), encrypted_data.size(), tmp_path); } -Wallet::History Wallet::load_history(const Hash &bid) const { +Wallet::History WalletContainerStorage::load_history(const Hash &tid) const { Wallet::History used_addresses; std::string history_folder = get_history_folder(); - BinaryArray filename_data(std::begin(bid.data), std::end(bid.data)); + BinaryArray filename_data(std::begin(tid.data), std::end(tid.data)); common::append(filename_data, std::begin(m_history_filename_seed.data), std::end(m_history_filename_seed.data)); Hash filename_hash = crypto::cn_fast_hash(filename_data.data(), filename_data.size()); BinaryArray hist; if (!platform::load_file(history_folder + "/" + common::pod_to_hex(filename_hash) + ".txh", hist) || - hist.size() < sizeof(crypto::chacha8_iv) || - (hist.size() - sizeof(crypto::chacha8_iv)) % (2 * sizeof(PublicKey)) != 0) + hist.size() < sizeof(crypto::chacha_iv) || + (hist.size() - sizeof(crypto::chacha_iv)) % (2 * sizeof(PublicKey)) != 0) return used_addresses; - const crypto::chacha8_iv *iv = (const crypto::chacha8_iv *)hist.data(); - BinaryArray dec(hist.size() - sizeof(crypto::chacha8_iv), 0); - crypto::chacha8(hist.data() + sizeof(crypto::chacha8_iv), hist.size() - sizeof(crypto::chacha8_iv), m_history_key, + const crypto::chacha_iv *iv = (const crypto::chacha_iv *)hist.data(); + BinaryArray dec(hist.size() - sizeof(crypto::chacha_iv), 0); + crypto::chacha8(hist.data() + sizeof(crypto::chacha_iv), hist.size() - sizeof(crypto::chacha_iv), m_history_key, *iv, dec.data()); for (size_t i = 0; i != dec.size() / (2 * sizeof(PublicKey)); ++i) { - AccountPublicAddress ad; + AccountAddressSimple ad; memcpy(ad.view_public_key.data, dec.data() + i * 2 * sizeof(PublicKey), sizeof(PublicKey)); memcpy(ad.spend_public_key.data, dec.data() + i * 2 * sizeof(PublicKey) + sizeof(PublicKey), sizeof(PublicKey)); used_addresses.insert(ad); } return used_addresses; } + +std::vector WalletContainerStorage::payment_queue_get() const { + std::vector result; + platform::remove_file(get_payment_queue_folder() + "/tmp.tx"); + for (const auto &file : platform::get_filenames_in_folder(get_payment_queue_folder())) { + common::BinaryArray body; + if (!platform::load_file(get_payment_queue_folder() + "/" + file, body)) + continue; + result.push_back(std::move(body)); + } + return result; +} + +void WalletContainerStorage::payment_queue_add(const Hash &tid, const BinaryArray &binary_transaction) { + const std::string file = get_payment_queue_folder() + "/" + common::pod_to_hex(tid) + ".tx"; + platform::create_folder_if_necessary(get_payment_queue_folder()); + if (!platform::atomic_save_file( + file, binary_transaction.data(), binary_transaction.size(), get_payment_queue_folder() + "/tmp.tx")) + m_log(logging::WARNING) << "Failed to save transaction " << tid << " to file " << file << std::endl; + else + m_log(logging::INFO) << "Saved transaction " << tid << " to file " << file << std::endl; +} + +void WalletContainerStorage::payment_queue_remove(const Hash &tid) { + const std::string file = get_payment_queue_folder() + "/" + common::pod_to_hex(tid) + ".tx"; + if (!platform::remove_file(file)) + m_log(logging::WARNING) << "Failed to remove PQ transaction " << tid << " from file " << file << std::endl; + else + m_log(logging::INFO) << "Removed PQ transaction " << tid << " from file " << file << std::endl; + platform::remove_file(get_payment_queue_folder()); // When it becomes empty +} + +void WalletContainerStorage::set_label(const std::string &address, const std::string &label) { + throw std::runtime_error("Linkable wallet file cannot store labels"); +} + +Wallet::OutputHandler WalletContainerStorage::get_output_handler() const { + SecretKey vsk_copy = m_view_secret_key; + return + [vsk_copy](const PublicKey &tx_public_key, boost::optional *kd, const Hash &tx_inputs_hash, + size_t output_index, const OutputKey &key_output, PublicKey *spend_public_key, SecretKey *secret_scalar) { + if (!*kd) { + try { + *kd = generate_key_derivation(tx_public_key, vsk_copy); + // tx_public_key is not checked by daemon, so can be invalid + } catch (const std::exception &) { + *kd = KeyDerivation{}; + } + } + *spend_public_key = underive_public_key(kd->get(), output_index, key_output.public_key); + }; +} + +bool WalletContainerStorage::detect_our_output(const Hash &tid, const Hash &tx_inputs_hash, + const boost::optional &kd, size_t out_index, const PublicKey &spend_public_key, + const SecretKey &secret_scalar, const OutputKey &key_output, Amount *amount, KeyPair *output_keypair, + AccountAddress *address) { + WalletRecord record; + if (!get_look_ahead_record(record, spend_public_key)) + return false; + AccountAddressSimple s_address{spend_public_key, get_view_public_key()}; + if (record.spend_secret_key != SecretKey{}) { + if (!kd) // tx_public_key was invalid + return false; + // We do some calcs twice here, but only for our outputs (which are usually very small %) + output_keypair->public_key = derive_public_key(kd.get(), out_index, spend_public_key); + output_keypair->secret_key = derive_secret_key(kd.get(), out_index, record.spend_secret_key); + if (output_keypair->public_key != key_output.public_key) + return false; + } + *address = s_address; + *amount = key_output.amount; + return true; +} + +bool WalletContainerStorage::detect_not_our_output(bool tx_amethyst, const Hash &tid, const Hash &tx_inputs_hash, + boost::optional *history, size_t out_index, const OutputKey &key_output, Amount *amount, + AccountAddress *address) { + KeyPair tx_keys; // We do some expensive calcs once and only if needed + if (tx_amethyst) { + KeyPair output_secret_keys = TransactionBuilder::deterministic_keys_from_seed( + tx_inputs_hash, m_tx_derivation_seed, common::get_varint_data(out_index)); + Hash output_secret = + crypto::cn_fast_hash(output_secret_keys.public_key.data, sizeof(output_secret_keys.public_key.data)); + AccountAddressSimple addr; + for (size_t i = 0; i != sizeof(output_secret); ++i) + addr.view_public_key.data[i] = output_secret.data[i] ^ key_output.encrypted_secret.data[i]; + if (crypto::key_isvalid(addr.view_public_key)) { + if (tx_keys.secret_key == SecretKey{}) + tx_keys = TransactionBuilder::transaction_keys_from_seed(tx_inputs_hash, m_tx_derivation_seed); + KeyDerivation to_derivation = generate_key_derivation(addr.view_public_key, tx_keys.secret_key); + addr.spend_public_key = underive_public_key(to_derivation, out_index, key_output.public_key); + *address = addr; + *amount = key_output.amount; + return true; + } + return false; + } + if (!*history) + *history = load_history(tid); + if (!history->get().empty()) { + if (tx_keys.secret_key == SecretKey{}) + tx_keys = TransactionBuilder::transaction_keys_from_seed(tx_inputs_hash, m_tx_derivation_seed); + for (const auto &addr : history->get()) { + const KeyDerivation derivation = generate_key_derivation(addr.view_public_key, tx_keys.secret_key); + const PublicKey guess_key = crypto::derive_public_key(derivation, out_index, addr.spend_public_key); + if (guess_key == key_output.public_key) { + *address = addr; + *amount = key_output.amount; + return true; + } + } + } + return false; +} + +static const std::string current_version = "CryptoNoteWallet1"; +static const size_t GENERATE_AHEAD = 20000; // TODO - move to better place + +std::string WalletHD::generate_mnemonic(size_t bits, uint32_t version) { + using common::BITS_PER_WORD; + using common::crc32_reverse_step_zero; + using common::crc32_step_zero; + using common::word_crc32_adj; + using common::word_ptrs; + using common::words_bylen; + using common::WORDS_COUNT; + using common::WORDS_MAX_LEN; + using common::WORDS_MIN_LEN; + std::unordered_map last_word(WORDS_COUNT); + for (size_t i = 0; i != WORDS_COUNT; i++) { + uint32_t crc32_suffix = version ^ word_crc32_adj[i]; + for (auto p = word_ptrs[i]; p != word_ptrs[i + 1]; p++) { + crc32_suffix = crc32_reverse_step_zero(crc32_suffix); + } + last_word[crc32_suffix] = i; + } + size_t words_in_prefix = (bits - 1) / BITS_PER_WORD + 1; + size_t words_total = words_in_prefix + 3; + std::unique_ptr word_ids(new size_t[words_total]); + while (true) { + uint32_t crc32_prefix = 0; + for (size_t i = 0; i != words_in_prefix; i++) { + size_t j = crypto::rand() % WORDS_COUNT; + word_ids[i] = j; + for (auto p = word_ptrs[j]; p != word_ptrs[j + 1]; p++) { + crc32_prefix = crc32_step_zero(crc32_prefix); + } + crc32_prefix ^= word_crc32_adj[j]; + } + for (size_t i = 0; i < WORDS_MIN_LEN; i++) { + crc32_prefix = crc32_step_zero(crc32_prefix); + } + const uint32_t *adj1 = word_crc32_adj; + for (size_t l1 = 0;; l1++) { + for (; adj1 != words_bylen[l1]; adj1++) { + uint32_t crc32_prefix2 = crc32_prefix ^ *adj1; + for (size_t i = 0; i < WORDS_MIN_LEN; i++) { + crc32_prefix2 = crc32_step_zero(crc32_prefix2); + } + const uint32_t *adj2 = word_crc32_adj; + for (size_t l2 = 0;; l2++) { + for (; adj2 != words_bylen[l2]; adj2++) { + auto it = last_word.find(crc32_prefix2 ^ *adj2); + if (it != last_word.end()) { + word_ids[words_in_prefix] = adj1 - word_crc32_adj; + word_ids[words_in_prefix + 1] = adj2 - word_crc32_adj; + word_ids[words_in_prefix + 2] = it->second; + size_t word0 = word_ids[0]; + std::string result(word_ptrs[word0], word_ptrs[word0 + 1]); + for (size_t i = 1; i != words_total; i++) { + result.push_back(' '); + size_t word = word_ids[i]; + result.append(word_ptrs[word], word_ptrs[word + 1]); + } + return result; + } + } + if (l2 == WORDS_MAX_LEN - WORDS_MIN_LEN) { + break; + } + crc32_prefix2 = crc32_step_zero(crc32_prefix2); + } + } + if (l1 == WORDS_MAX_LEN - WORDS_MIN_LEN) { + break; + } + crc32_prefix = crc32_step_zero(crc32_prefix); + } + } +} + +static const std::string ADDRESS_COUNT_PREFIX = "total_address_count"; +static const std::string CREATION_TIMESTAMP_PREFIX = "creation_timestamp"; + +using namespace platform; + +bool WalletHD::is_sqlite(const std::string &full_path) { + bool created = false; + sqlite::Dbi db_dbi; + try { + db_dbi.open_check_create(platform::O_READ_EXISTING, full_path, &created); + return true; + } catch (const std::exception &) { + } + return false; +} + +WalletHD::WalletHD(const Currency ¤cy, logging::ILogger &log, const std::string &path, + const std::string &password, bool readonly) + : Wallet(currency, log, path) { + bool created = false; + m_db_dbi.open_check_create(readonly ? platform::O_READ_EXISTING : platform::O_OPEN_EXISTING, path, &created); + BinaryArray salt = get_salt(); + common::append(salt, common::as_binary_array(password)); + crypto::CryptoNightContext cn_ctx; + m_wallet_key = generate_chacha8_key(cn_ctx, salt.data(), salt.size()); + try { + load(); + } catch (const Bip32Key::Exception &ex) { + std::throw_with_nested(Exception{api::WALLETD_MNEMONIC_CRC, "Wrong mnemonic"}); + } catch (const std::exception &ex) { + std::throw_with_nested(Exception{api::WALLET_FILE_DECRYPT_ERROR, "Wallet file invalid or wrong password"}); + } +} + +WalletHD::WalletHD(const Currency ¤cy, logging::ILogger &log, const std::string &path, + const std::string &password, const std::string &mnemonic, uint8_t address_type, Timestamp creation_timestamp, + const std::string &mnemonic_password) + : Wallet(currency, log, path) { + bool created = false; + m_db_dbi.open_check_create(platform::O_CREATE_NEW, path, &created); + m_db_dbi.exec( + "CREATE TABLE unencrypted(key BLOB PRIMARY KEY COLLATE BINARY NOT NULL, value BLOB NOT NULL) WITHOUT ROWID"); + m_db_dbi.exec( + "CREATE TABLE parameters(key_hash BLOB PRIMARY KEY COLLATE BINARY NOT NULL, key BLOB NOT NULL, value BLOB NOT NULL) WITHOUT ROWID"); + m_db_dbi.exec( + "CREATE TABLE labels(address_hash BLOB PRIMARY KEY NOT NULL, address BLOB NOT NULL, label BLOB NOT NULL) WITHOUT ROWID"); + m_db_dbi.exec( + "CREATE TABLE payment_queue(tid_hash BLOB COLLATE BINARY NOT NULL, net_hash BLOB COLLATE BINARY NOT NULL, tid BLOB NOT NULL, net BLOB NOT NULL, binary_transaction BLOB NOT NULL, PRIMARY KEY (tid_hash, net_hash)) WITHOUT ROWID"); + BinaryArray salt(sizeof(Hash)); + crypto::generate_random_bytes(salt.data(), salt.size()); + put_salt(salt); // The only unencrypted field + common::append(salt, common::as_binary_array(password)); + crypto::CryptoNightContext cn_ctx; + m_wallet_key = generate_chacha8_key(cn_ctx, salt.data(), salt.size()); + + if (mnemonic.empty()) + return; + put("version", current_version, true); + put("coinname", CRYPTONOTE_NAME, true); + put("address-type", BinaryArray(1, address_type), true); + put("mnemonic", cn::Bip32Key::check_bip39_mnemonic(mnemonic), true); + put("mnemonic-password", mnemonic_password, true); // write always to keep row count the same + put(ADDRESS_COUNT_PREFIX, seria::to_binary(m_used_address_count), true); + + on_first_output_found(creation_timestamp); + + try { + load(); + } catch (const Bip32Key::Exception &ex) { + std::throw_with_nested(Exception{api::WALLETD_MNEMONIC_CRC, "Wrong mnemonic"}); + } catch (const std::exception &ex) { + std::throw_with_nested(Exception{api::WALLET_FILE_DECRYPT_ERROR, "Wallet file invalid or wrong password"}); + } + commit(); +} + +void WalletHD::load() { + std::string version; + if (!get("version", version) || version != current_version) + throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Wallet version unknown - " + version); + std::string coinname; + if (!get("coinname", coinname) || coinname != CRYPTONOTE_NAME) + throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Wallet is for different coin - " + coinname); + std::string mnemonic; + BinaryArray address_type; + if (!get("address-type", address_type) || address_type.size() != 1) + throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Wallet corrupted, no address type"); + m_address_type = address_type.at(0); + if (m_address_type != AccountAddressUnlinkable::type_tag && + m_address_type != AccountAddressUnlinkable::type_tag_auditable) + throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Wallet address type unknown"); + if (get("mnemonic", mnemonic)) { + std::string mnemonic_password; + invariant(get("mnemonic-password", mnemonic_password), ""); + mnemonic = cn::Bip32Key::check_bip39_mnemonic(mnemonic); + cn::Bip32Key master_key = cn::Bip32Key::create_master_key(mnemonic, mnemonic_password); + cn::Bip32Key k0 = master_key.derive_key(0x8000002c); + cn::Bip32Key k1 = k0.derive_key(0x80000300); + cn::Bip32Key k2 = k1.derive_key(0x80000000 + m_address_type); + cn::Bip32Key k3 = k2.derive_key(0); + cn::Bip32Key k4 = k3.derive_key(0); + m_seed = crypto::cn_fast_hash(k4.get_priv_key().data(), k4.get_priv_key().size()); + m_tx_derivation_seed = derive_from_seed(m_seed, "tx_derivation"); + BinaryArray sk_data = m_seed | "spend_key_base"; + m_spend_key_base.secret_key = crypto::hash_to_scalar(sk_data.data(), sk_data.size()); + invariant(crypto::secret_key_to_public_key(m_spend_key_base.secret_key, &m_spend_key_base.public_key), ""); + } else { // View only + BinaryArray ba; + invariant(get("spend_key_base_public_key", ba) && ba.size() == sizeof(m_spend_key_base.public_key), ""); + memcpy(m_spend_key_base.public_key.data, ba.data(), ba.size()); + invariant(crypto::key_isvalid(m_spend_key_base.public_key), "Wallet Corrupted - spend key base is invalid"); + if (get("tx_derivation_seed", ba) && ba.size() == sizeof(m_tx_derivation_seed)) + memcpy(m_tx_derivation_seed.data, ba.data(), ba.size()); + // only if we have output_secret_derivation_seed, view-only wallet will be able to see outgoing addresses + } + BinaryArray vk_data = + BinaryArray{std::begin(m_spend_key_base.public_key.data), std::end(m_spend_key_base.public_key.data)} | + "view_key"; + m_view_secret_key = crypto::hash_to_scalar(vk_data.data(), vk_data.size()); + invariant(crypto::secret_key_to_public_key(m_view_secret_key, &m_view_public_key), ""); + { + BinaryArray ba; + if (get(ADDRESS_COUNT_PREFIX, ba)) + seria::from_binary(m_used_address_count, ba); + if (get(CREATION_TIMESTAMP_PREFIX + net_append(m_currency.net), ba)) + seria::from_binary(m_oldest_timestamp, ba); + else + m_oldest_timestamp = 0; + } + generate_ahead(); + + sqlite::Stmt stmt_get; + stmt_get.prepare(m_db_dbi, "SELECT address, label FROM labels"); + while (stmt_get.step()) { + auto address_size = stmt_get.column_bytes(0); + auto address_data = stmt_get.column_blob(0); + auto label_size = stmt_get.column_bytes(1); + auto label_data = stmt_get.column_blob(1); + BinaryArray ka = decrypt_data(m_wallet_key, address_data, address_size); + BinaryArray ba = decrypt_data(m_wallet_key, label_data, label_size); + + m_labels[std::string{ka.begin(), ka.end()}] = std::string{ba.begin(), ba.end()}; + } +} + +void WalletHD::generate_ahead1(size_t counter, std::vector &result) const { + std::vector key_result; + key_result.resize(result.size()); + Hash view_seed; + memcpy(view_seed.data, m_spend_key_base.public_key.data, sizeof(m_spend_key_base.public_key.data)); + crypto::generate_hd_spendkeys(m_spend_key_base, view_seed, counter, &key_result); + for (size_t i = 0; i != result.size(); ++i) { + WalletRecord &record = result[i]; + record.spend_secret_key = key_result.at(i).secret_key; + record.spend_public_key = key_result.at(i).public_key; + record.creation_timestamp = std::numeric_limits::max(); // So adding an address will never rescan + } +} + +void WalletHD::generate_ahead() { + if (m_wallet_records.size() >= m_used_address_count + GENERATE_AHEAD) + return; + size_t delta = m_used_address_count + GENERATE_AHEAD - m_wallet_records.size(); + std::vector> results; + if (delta < 1000) { // TODO - arbitrary constant when single-threaded generation is faster + results.resize(1); + results[0].resize(delta); + generate_ahead1(m_wallet_records.size(), results[0]); + } else { + const size_t thc = std::thread::hardware_concurrency(); + results.resize(thc); + std::vector workers; + for (size_t i = 0; i != thc; i++) { + size_t start = delta * i / thc; + results[i].resize(delta * (i + 1) / thc - start); + workers.push_back(std::thread( + std::bind(&WalletHD::generate_ahead1, this, m_wallet_records.size() + start, std::ref(results[i])))); + } + std::for_each(workers.begin(), workers.end(), [](std::thread &t) { t.join(); }); + } + m_wallet_records.reserve(m_used_address_count + GENERATE_AHEAD); + for (const auto &result : results) + for (const auto &record : result) { + m_records_map.insert(std::make_pair(record.spend_public_key, m_wallet_records.size())); + m_wallet_records.push_back(record); + } +} + +// std::string WalletHD::hash_for_key(const Hash & key)const{ +// BinaryArray seed_data(std::begin(key.data), std::end(key.data)); +// common::append(seed_data, std::begin(m_view_seed.data), std::end(m_view_seed.data)); +// Hash ha = crypto::cn_fast_hash(seed_data.data(), seed_data.size()); +// return std::string(ha.data, ha.data + sizeof(ha.data)); +//} + +BinaryArray WalletHD::encrypt_data(const crypto::chacha_key &wallet_key, const BinaryArray &data) { + const size_t MIN_SIZE = 256; + const size_t EXTRA_SIZE = sizeof(Hash) + 4; // iv, actual size in le + size_t actual_size = 1; + while (actual_size < data.size() + EXTRA_SIZE || actual_size < MIN_SIZE) + actual_size *= 2; + BinaryArray large_data(actual_size - sizeof(Hash)); + common::uint_le_to_bytes(large_data.data(), 4, data.size()); + memcpy(large_data.data() + 4, data.data(), data.size()); + BinaryArray enc_data(sizeof(Hash) + large_data.size()); + Hash iv = crypto::rand(); + memcpy(enc_data.data(), iv.data, sizeof(iv)); + BinaryArray key_data = BinaryArray{wallet_key.data, wallet_key.data + sizeof(wallet_key.data)} | iv; + crypto::chacha_key key{crypto::cn_fast_hash(key_data.data(), key_data.size())}; + chacha(20, large_data.data(), large_data.size(), key, crypto::chacha_iv{}, enc_data.data() + sizeof(Hash)); + return enc_data; +} + +BinaryArray WalletHD::decrypt_data(const crypto::chacha_key &wallet_key, const uint8_t *value_data, size_t value_size) { + Hash iv; + invariant(value_size >= 4 + sizeof(Hash), ""); + memcpy(iv.data, value_data, sizeof(Hash)); + BinaryArray result(value_size - sizeof(Hash)); + BinaryArray key_data = BinaryArray{wallet_key.data, wallet_key.data + sizeof(wallet_key.data)} | iv; + crypto::chacha_key key{crypto::cn_fast_hash(key_data.data(), key_data.size())}; + chacha(20, value_data + sizeof(Hash), result.size(), key, crypto::chacha_iv{}, result.data()); + auto real_size = common::uint_le_from_bytes(result.data(), 4); + invariant(real_size <= result.size() - 4, ""); + return BinaryArray{result.data() + 4, result.data() + 4 + real_size}; +} + +void WalletHD::put_salt(const BinaryArray &salt) { + sqlite::Stmt stmt_update; + stmt_update.prepare(m_db_dbi, "REPLACE INTO unencrypted (key, value) VALUES ('salt', ?)"); + stmt_update.bind_blob(1, salt.data(), salt.size()); + invariant(!stmt_update.step(), ""); +} +BinaryArray WalletHD::get_salt() const { + sqlite::Stmt stmt_get; + stmt_get.prepare(m_db_dbi, "SELECT value FROM unencrypted WHERE key = 'salt'"); + invariant(stmt_get.step(), ""); + auto salt_size = stmt_get.column_bytes(0); + auto salt_data = stmt_get.column_blob(0); + return BinaryArray{salt_data, salt_data + salt_size}; +} + +void WalletHD::put(const std::string &key, const BinaryArray &value, bool nooverwrite) { + Hash key_hash = derive_from_key(m_wallet_key, "db_parameters" + key); + BinaryArray enc_key = encrypt_data(m_wallet_key, BinaryArray{key.data(), key.data() + key.size()}); + BinaryArray enc_value = encrypt_data(m_wallet_key, value); + sqlite::Stmt stmt_update; + stmt_update.prepare(m_db_dbi, + nooverwrite ? "INSERT INTO parameters (key_hash, key, value) VALUES (?, ?, ?)" + : "REPLACE INTO parameters (key_hash, key, value) VALUES (?, ?, ?)"); + stmt_update.bind_blob(1, key_hash.data, sizeof(key_hash.data)); + stmt_update.bind_blob(2, enc_key.data(), enc_key.size()); + stmt_update.bind_blob(3, enc_value.data(), enc_value.size()); + invariant(!stmt_update.step(), ""); +} + +bool WalletHD::get(const std::string &key, common::BinaryArray &value) const { + Hash key_hash = derive_from_key(m_wallet_key, "db_parameters" + key); + + sqlite::Stmt stmt_get; + stmt_get.prepare(m_db_dbi, "SELECT value FROM parameters WHERE key_hash = ?"); + stmt_get.bind_blob(1, key_hash.data, sizeof(key_hash.data)); + if (!stmt_get.step()) + return false; + auto label_size = stmt_get.column_bytes(0); + auto label_data = stmt_get.column_blob(0); + value = decrypt_data(m_wallet_key, label_data, label_size); + return true; +} + +void WalletHD::put(const std::string &key, const std::string &value, bool nooverwrite) { + return put(key, common::as_binary_array(value), nooverwrite); +} +bool WalletHD::get(const std::string &key, std::string &value) const { + common::BinaryArray ba; + if (!get(key, ba)) + return false; + value = common::as_string(ba); + return true; +} + +std::vector WalletHD::generate_new_addresses( + const std::vector &sks, Timestamp ct, Timestamp now, bool *rescan_from_ct) { + for (const auto &sk : sks) + if (sk != SecretKey{}) + throw std::runtime_error("Generating non-deterministic addreses not supported by HD wallet"); + std::vector result; + if (sks.empty()) + return result; + auto was_used_address_count = m_used_address_count; + m_used_address_count += sks.size(); + generate_ahead(); + for (size_t i = 0; i != sks.size(); ++i) + result.push_back(m_wallet_records.at(was_used_address_count + i)); + put(ADDRESS_COUNT_PREFIX, seria::to_binary(m_used_address_count), false); + commit(); + return result; +} + +AccountAddress WalletHD::record_to_address(const WalletRecord &record) const { + PublicKey sv = generate_address_s_v(record.spend_public_key, m_view_secret_key); + // TODO - do multiplication only once + return AccountAddressUnlinkable{ + record.spend_public_key, sv, m_address_type == AccountAddressUnlinkable::type_tag_auditable}; +} + +bool WalletHD::get_record(WalletRecord &record, const AccountAddress &v_addr) const { + if (v_addr.type() != typeid(AccountAddressUnlinkable)) + return false; + auto &addr = boost::get(v_addr); + auto rit = m_records_map.find(addr.s); + if (rit == m_records_map.end() || rit->second >= get_actual_records_count()) + return false; + // TODO - do not call record_to_address + auto addr2 = record_to_address(m_wallet_records.at(rit->second)); + if (v_addr != addr2) + return false; + // invariant (m_wallet_records.at(rit->second).spend_public_key == addr.spend_public_key, ""); + record = m_wallet_records.at(rit->second); + return true; +} + +void WalletHD::set_password(const std::string &password) { + auto parameters = parameters_get(); + auto pq2 = payment_queue_get2(); + + m_db_dbi.exec("DELETE FROM payment_queue"); + m_db_dbi.exec("DELETE FROM parameters"); + m_db_dbi.exec("DELETE FROM labels"); + + BinaryArray salt(sizeof(Hash)); + crypto::generate_random_bytes(salt.data(), salt.size()); + put_salt(salt); + common::append(salt, BinaryArray{password.data(), password.data() + password.size()}); + crypto::CryptoNightContext cn_ctx; + m_wallet_key = generate_chacha8_key(cn_ctx, salt.data(), salt.size()); + + for (const auto &p : parameters) + put(p.first, p.second, true); + for (const auto &l : m_labels) + set_label(l.first, l.second); + for (const auto &el : pq2) + payment_queue_add(std::get<0>(el), std::get<1>(el), std::get<2>(el)); + commit(); +} + +void WalletHD::export_wallet(const std::string &export_path, const std::string &new_password, bool view_only, + bool view_outgoing_addresses) const { + WalletHD other(m_currency, m_log.get_logger(), export_path, new_password, std::string(), 0, 0, std::string()); + + if (!is_view_only() && view_only) { + other.put("spend_key_base_public_key", + BinaryArray{std::begin(m_spend_key_base.public_key.data), std::end(m_spend_key_base.public_key.data)}, + true); + if (view_outgoing_addresses) + other.put("tx_derivation_seed", + BinaryArray{std::begin(m_tx_derivation_seed.data), std::end(m_tx_derivation_seed.data)}, true); + for (const auto &p : parameters_get()) + if (p.first != "mnemonic" && p.first != "mnemonic-password") + other.put(p.first, p.second, true); + for (const auto &l : m_labels) + other.set_label(l.first, l.second); + } else { + for (const auto &p : parameters_get()) + other.put(p.first, p.second, true); + for (const auto &l : m_labels) + other.set_label(l.first, l.second); + for (const auto &el : payment_queue_get2()) + other.payment_queue_add(std::get<0>(el), std::get<1>(el), std::get<2>(el)); + } + other.commit(); +} + +std::string WalletHD::export_keys() const { + std::string mnemonic; + if (!get("mnemonic", mnemonic)) + throw std::runtime_error("Exporting keys (mnemonic) not supported by view-only HD wallet"); + return mnemonic; +} + +void WalletHD::on_first_output_found(Timestamp ts) { + BinaryArray ba; + if (m_oldest_timestamp != 0 || ts == 0) + return; + put(CREATION_TIMESTAMP_PREFIX + net_append(m_currency.net), seria::to_binary(ts), false); + commit(); +} + +void WalletHD::create_look_ahead_records(size_t count) { + if (count <= m_used_address_count) + return; + m_used_address_count = count; + generate_ahead(); + put(ADDRESS_COUNT_PREFIX, seria::to_binary(m_used_address_count), false); + commit(); +} + +void WalletHD::backup(const std::string &dst_name, const std::string &pass) const { + export_wallet(dst_name, pass, false, false); +} + +Wallet::History WalletHD::load_history(const Hash &tid) const { return Wallet::History{}; } + +std::vector WalletHD::payment_queue_get() const { + std::vector result; + auto pq2 = payment_queue_get2(); + for (const auto &el : pq2) + if (std::get<1>(el) == m_currency.net) + result.push_back(std::get<2>(el)); + return result; +} + +std::vector> WalletHD::payment_queue_get2() const { + std::vector> result; + sqlite::Stmt stmt_get; + stmt_get.prepare(m_db_dbi, "SELECT tid, net, binary_transaction FROM payment_queue"); + while (stmt_get.step()) { + auto tid_size = stmt_get.column_bytes(0); + auto tid_data = stmt_get.column_blob(0); + auto net_size = stmt_get.column_bytes(1); + auto net_data = stmt_get.column_blob(1); + auto btx_size = stmt_get.column_bytes(2); + auto btx_data = stmt_get.column_blob(2); + + BinaryArray key = decrypt_data(m_wallet_key, tid_data, tid_size); + invariant(key.size() == sizeof(Hash), ""); + Hash tid; + memcpy(tid.data, key.data(), sizeof(Hash)); + BinaryArray net = decrypt_data(m_wallet_key, net_data, net_size); + BinaryArray ba = decrypt_data(m_wallet_key, btx_data, btx_size); + result.push_back(std::make_tuple(tid, std::string(net.begin(), net.end()), ba)); + } + return result; +} + +void WalletHD::payment_queue_add(const Hash &tid, const std::string &net, const BinaryArray &binary_transaction) { + Hash tid_hash = + derive_from_key(m_wallet_key, "db_payment_queue_tid" + std::string(std::begin(tid.data), std::end(tid.data))); + Hash net_hash = derive_from_key(m_wallet_key, "db_payment_queue_net" + net); + BinaryArray enc_tid = encrypt_data(m_wallet_key, BinaryArray{tid.data, tid.data + sizeof(tid.data)}); + BinaryArray enc_net = encrypt_data(m_wallet_key, BinaryArray{net.data(), net.data() + net.size()}); + BinaryArray enc_value = encrypt_data(m_wallet_key, binary_transaction); + sqlite::Stmt stmt_update; + stmt_update.prepare(m_db_dbi, + "REPLACE INTO payment_queue (tid_hash, net_hash, tid, net, binary_transaction) VALUES (?, ?, ?, ?, ?)"); + stmt_update.bind_blob(1, tid_hash.data, sizeof(tid_hash.data)); + stmt_update.bind_blob(2, net_hash.data, sizeof(net_hash.data)); + stmt_update.bind_blob(3, enc_tid.data(), enc_tid.size()); + stmt_update.bind_blob(4, enc_net.data(), enc_net.size()); + stmt_update.bind_blob(5, enc_value.data(), enc_value.size()); + invariant(!stmt_update.step(), ""); +} + +std::vector> WalletHD::parameters_get() const { + std::vector> result; + sqlite::Stmt stmt_get; + stmt_get.prepare(m_db_dbi, "SELECT key, value FROM parameters"); + while (stmt_get.step()) { + auto key_size = stmt_get.column_bytes(0); + ; + auto key_data = stmt_get.column_blob(0); + auto value_size = stmt_get.column_bytes(1); + ; + auto value_data = stmt_get.column_blob(1); + + BinaryArray ka = decrypt_data(m_wallet_key, key_data, key_size); + BinaryArray ba = decrypt_data(m_wallet_key, value_data, value_size); + std::string key = std::string(ka.begin(), ka.end()); + result.push_back(std::make_pair(key, ba)); + } + return result; +} + +void WalletHD::payment_queue_add(const Hash &tid, const BinaryArray &binary_transaction) { + payment_queue_add(tid, m_currency.net, binary_transaction); +} + +void WalletHD::commit() { + m_db_dbi.commit_txn(); + m_db_dbi.begin_txn(); +} + +void WalletHD::payment_queue_remove(const Hash &tid) { + Hash tid_hash = + derive_from_key(m_wallet_key, "db_payment_queue_tid" + std::string(std::begin(tid.data), std::end(tid.data))); + Hash net_hash = derive_from_key(m_wallet_key, "db_payment_queue_net" + m_currency.net); + + sqlite::Stmt stmt_del; + stmt_del.prepare(m_db_dbi, "DELETE FROM payment_queue WHERE net_hash = ? AND tid_hash = ?"); + stmt_del.bind_blob(1, net_hash.data, sizeof(net_hash.data)); + stmt_del.bind_blob(2, tid_hash.data, sizeof(tid_hash.data)); + invariant(!stmt_del.step(), ""); + + if (tid.data[0] == 'x') // committing here is not so critical, improve speed here + commit(); +} + +void WalletHD::set_label(const std::string &address, const std::string &label) { + Hash address_hash = derive_from_key(m_wallet_key, "db_labels" + address); + BinaryArray enc_address = encrypt_data(m_wallet_key, BinaryArray{address.data(), address.data() + address.size()}); + BinaryArray enc_label = encrypt_data(m_wallet_key, BinaryArray{label.data(), label.data() + label.size()}); + + if (label.empty()) { + m_labels.erase(address); + sqlite::Stmt stmt_del; + stmt_del.prepare(m_db_dbi, "DELETE FROM labels WHERE address_hash = ?"); + stmt_del.bind_blob(1, address_hash.data, sizeof(address_hash.data)); + invariant(!stmt_del.step(), ""); + } else { + m_labels[address] = label; + + sqlite::Stmt stmt_update; + stmt_update.prepare(m_db_dbi, "REPLACE INTO labels (address_hash, address, label) VALUES (?, ?, ?)"); + stmt_update.bind_blob(1, address_hash.data, sizeof(address_hash.data)); + stmt_update.bind_blob(2, enc_address.data(), enc_address.size()); + stmt_update.bind_blob(3, enc_label.data(), enc_label.size()); + invariant(!stmt_update.step(), ""); + } + commit(); +} + +std::string WalletHD::get_label(const std::string &address) const { + auto lit = m_labels.find(address); + if (lit == m_labels.end()) + return std::string(); + return lit->second; +} + +Wallet::OutputHandler WalletHD::get_output_handler() const { + SecretKey vsk_copy = m_view_secret_key; + return + [vsk_copy](const PublicKey &tx_public_key, boost::optional *kd, const Hash &tx_inputs_hash, + size_t output_index, const OutputKey &key_output, PublicKey *spend_public_key, SecretKey *secret_scalar) { + *spend_public_key = crypto::unlinkable_underive_public_key(vsk_copy, tx_inputs_hash, output_index, + key_output.public_key, key_output.encrypted_secret, secret_scalar); + }; +} + +bool WalletHD::detect_our_output(const Hash &tid, const Hash &tx_inputs_hash, const boost::optional &kd, + size_t out_index, const PublicKey &spend_public_key, const SecretKey &secret_scalar, const OutputKey &key_output, + Amount *amount, KeyPair *output_keypair, AccountAddress *address) { + WalletRecord record; + if (!get_look_ahead_record(record, spend_public_key)) + return false; + AccountAddress addr = record_to_address(record); + const auto &u_address = boost::get(addr); // TODO - better logic + if (u_address.is_auditable != key_output.is_auditable) + return false; + if (record.spend_secret_key != SecretKey{}) { + output_keypair->secret_key = crypto::unlinkable_derive_secret_key(record.spend_secret_key, secret_scalar); + if (!crypto::secret_key_to_public_key(output_keypair->secret_key, &output_keypair->public_key) || + output_keypair->public_key != key_output.public_key) + return false; + } + *address = addr; + // std::cout << "My unlinkable output! out_index=" << out_index << + // "amount=" << key_output.amount << std::endl; + *amount = key_output.amount; + return true; +} + +bool WalletHD::detect_not_our_output(bool tx_amethyst, const Hash &tid, const Hash &tx_inputs_hash, + boost::optional *history, size_t out_index, const OutputKey &key_output, Amount *amount, + AccountAddress *address) { + KeyPair output_secret_keys = TransactionBuilder::deterministic_keys_from_seed( + tx_inputs_hash, m_tx_derivation_seed, common::get_varint_data(out_index)); + Hash output_secret = + crypto::cn_fast_hash(output_secret_keys.public_key.data, sizeof(output_secret_keys.public_key.data)); + AccountAddressUnlinkable u_address; + if (!crypto::unlinkable_underive_address(output_secret, tx_inputs_hash, out_index, key_output.public_key, + key_output.encrypted_secret, &u_address.s, &u_address.sv)) + return false; + u_address.is_auditable = key_output.is_auditable; + *amount = key_output.amount; + // We cannot generate key_image for others addresses + *address = u_address; + return true; +} diff --git a/src/Core/Wallet.hpp b/src/Core/Wallet.hpp index d44cde56..83c97325 100644 --- a/src/Core/Wallet.hpp +++ b/src/Core/Wallet.hpp @@ -7,14 +7,14 @@ #include #include "CryptoNote.hpp" #include "Currency.hpp" -#include "crypto/chacha8.hpp" +//#include "Multicore.hpp" +#include +#include "crypto/chacha.hpp" #include "logging/LoggerMessage.hpp" +#include "platform/DBsqlite3.hpp" #include "platform/Files.hpp" -namespace bytecoin { - -using WalletKey = crypto::chacha8_key; -using HistoryKey = crypto::chacha8_key; +namespace cn { struct WalletRecord { PublicKey spend_public_key{}; @@ -30,13 +30,13 @@ inline bool operator!=(const WalletRecord &lhs, const WalletRecord &rhs) { retur // stores at most 1 view secret key. 1 or more spend secret keys // We do not allow deleting first spend key. It is used in seed calculations -// All file formats are opened as is, and saved to V2 when changing something +// File formats are opened as is, and saved to V2 when changing something class Wallet { +protected: + const Currency &m_currency; logging::LoggerRef m_log; - std::unique_ptr file; std::string m_path; - std::string m_password; - WalletKey m_wallet_key; // very slow to generate, save + crypto::chacha_key m_wallet_key; // very slow to generate, save PublicKey m_view_public_key; SecretKey m_view_secret_key; @@ -45,61 +45,211 @@ class Wallet { // Timestamp m_creation_timestamp = 0; Timestamp m_oldest_timestamp = std::numeric_limits::max(); - Hash m_seed; // Main seed, never used directly - Hash m_tx_derivation_seed; // Hashed from seed - Hash m_coinbase_tx_derivation_seed; // Hashed from seed - HistoryKey m_history_key; // Hashed from seed - Hash m_history_filename_seed; // Hashed from seed - - void load_container_storage(); - void load_legacy_wallet_file(); - bool operator==(const Wallet &) const; - bool operator!=(const Wallet &other) const { return !(*this == other); } - - void save(const std::string &export_path, const WalletKey &wallet_key, bool view_only); - void save_and_check(); - - Hash derive_from_seed(const std::string &append); - + Hash m_seed; // Main seed, never used directly + Hash m_tx_derivation_seed; // Hashed from seed public: class Exception : public std::runtime_error { public: const int return_code; explicit Exception(int rc, const std::string &what) : std::runtime_error(what), return_code(rc) {} }; - Wallet(logging::ILogger &log, const std::string &path, const std::string &password, bool create = false, - const std::string &import_keys = std::string()); - std::vector generate_new_addresses(const std::vector &sks, Timestamp ct, Timestamp now, - bool *rescan_from_ct); // set secret_key to SecretKey{} to generate - void set_password(const std::string &password); - void export_wallet(const std::string &export_path, const std::string &new_password, bool view_only); - bool is_view_only() const { return m_wallet_records.at(0).spend_secret_key == SecretKey{}; } - BinaryArray export_keys() const; + Wallet(const Currency ¤cy, logging::ILogger &log, const std::string &path); + virtual ~Wallet() = default; + virtual void set_password(const std::string &password) = 0; + virtual void export_wallet(const std::string &export_path, const std::string &new_password, bool view_only, + bool view_outgoing_addresses) const = 0; + virtual bool is_view_only() const { return m_wallet_records.at(0).spend_secret_key == SecretKey{}; } + virtual bool is_deterministic() const { return false; } + virtual bool is_unlinkable() const { return false; } + virtual bool is_auditable() const { return false; } + bool is_det_viewonly() const { return is_view_only() && is_deterministic(); } + virtual std::string export_keys() const = 0; const PublicKey &get_view_public_key() const { return m_view_public_key; } const SecretKey &get_view_secret_key() const { return m_view_secret_key; } const std::vector &get_records() const { return m_wallet_records; } - bool get_record(WalletRecord &record, const AccountPublicAddress &) const; + virtual size_t get_actual_records_count() const { return m_wallet_records.size(); } + virtual bool get_record(WalletRecord &record, const AccountAddress &) const = 0; + virtual void create_look_ahead_records(size_t count) {} + bool get_look_ahead_record(WalletRecord &record, const PublicKey &); - bool is_our_address(const AccountPublicAddress &) const; - AccountPublicAddress get_first_address() const; + bool is_our_address(const AccountAddress &) const; + AccountAddress get_first_address() const; + virtual AccountAddress record_to_address(const WalletRecord &record) const = 0; - static size_t wallet_file_size(size_t records); + virtual std::vector generate_new_addresses(const std::vector &sks, Timestamp ct, + Timestamp now, + bool *rescan_from_ct) = 0; // set secret_key to SecretKey{} to generate std::string get_cache_name() const; + virtual Timestamp get_oldest_timestamp() const { return m_oldest_timestamp; } + virtual void on_first_output_found(Timestamp ts) = 0; + // called by WalletState, updates creation timestamp for imported wallet + const Hash &get_tx_derivation_seed() const { return m_tx_derivation_seed; } - const Hash &get_coinbase_tx_derivation_seed() const { return m_coinbase_tx_derivation_seed; } - const HistoryKey &get_history_key() const { return m_history_key; } - const Hash &get_history_filename_seed() const { return m_history_filename_seed; } - std::string get_history_folder() const { return m_path + ".history"; } - std::string get_payment_queue_folder() const { return m_path + ".payments"; } - - Timestamp get_oldest_timestamp() const { return m_oldest_timestamp; } - void on_first_output_found(Timestamp ts); // called by WalletState, updates creation timestamp for imported wallet - - typedef std::set History; - bool save_history(const Hash &bid, const History &used_addresses) const; - History load_history(const Hash &bid) const; + + virtual void backup(const std::string &dst_name, const std::string &pass) const = 0; + + typedef std::set History; + virtual bool save_history(const Hash &tid, const History &used_addresses) { return true; } + virtual History load_history(const Hash &tid) const = 0; + + virtual std::vector payment_queue_get() const = 0; + virtual void payment_queue_add(const Hash &tid, const BinaryArray &binary_transaction) = 0; + virtual void payment_queue_remove(const Hash &tid) = 0; + + virtual void set_label(const std::string &address, const std::string &label) = 0; + virtual std::string get_label(const std::string &address) const = 0; + + typedef std::function *, + const Hash &tx_inputs_hash, size_t out_index, const OutputKey &, PublicKey *, SecretKey *)> + OutputHandler; + // Self-contain functor with all info copied to be called from other threads + virtual OutputHandler get_output_handler() const = 0; + virtual bool detect_our_output(const Hash &tid, const Hash &tx_inputs_hash, + const boost::optional &kd, size_t out_index, const PublicKey &spend_public_key, + const SecretKey &secret_scalar, const OutputKey &, Amount *, KeyPair *output_keypair, AccountAddress *) = 0; + virtual bool detect_not_our_output(bool tx_amethyst, const Hash &tid, const Hash &tx_inputs_hash, + boost::optional *, size_t out_index, const OutputKey &, Amount *, AccountAddress *) = 0; +}; + +// stores at most 1 view secret key. 1 or more spend secret keys +// We do not allow deleting first spend key. It is used in seed calculations +// File formats are opened as is, and saved to V2 when changing something +class WalletContainerStorage : public Wallet { + std::unique_ptr m_file; + + crypto::chacha_key m_history_key; // Hashed from seed + Hash m_history_filename_seed; // Hashed from seed + + void load_container_storage(); + void load_legacy_wallet_file(); + bool operator==(const WalletContainerStorage &) const; + bool operator!=(const WalletContainerStorage &other) const { return !(*this == other); } + + void load(); + void save(const std::string &export_path, const crypto::chacha_key &wallet_key, bool view_only, + platform::OpenMode open_mode) const; + void save_and_check(); + + std::string get_history_folder() const; + std::string get_payment_queue_folder() const; + + WalletContainerStorage( + const Currency ¤cy, logging::ILogger &log, const std::string &path, const crypto::chacha_key &wallet_key); + +public: + WalletContainerStorage( + const Currency ¤cy, logging::ILogger &log, const std::string &path, const std::string &password); + WalletContainerStorage(const Currency ¤cy, logging::ILogger &log, const std::string &path, + const std::string &password, const std::string &import_keys, Timestamp creation_timestamp); + std::vector generate_new_addresses( + const std::vector &sks, Timestamp ct, Timestamp now, bool *rescan_from_ct) override; + AccountAddress record_to_address(const WalletRecord &record) const override; + bool get_record(WalletRecord &record, const AccountAddress &) const override; + void set_password(const std::string &password) override; + void export_wallet(const std::string &export_path, const std::string &new_password, bool view_only, + bool view_outgoing_addresses) const override; + std::string export_keys() const override; + + static size_t wallet_file_size(size_t records); + + void on_first_output_found(Timestamp ts) override; + + void backup(const std::string &dst_name, const std::string &pass) const override; + + bool save_history(const Hash &tid, const History &used_addresses) override; + History load_history(const Hash &tid) const override; + + std::vector payment_queue_get() const override; + void payment_queue_add(const Hash &tid, const BinaryArray &binary_transaction) override; + void payment_queue_remove(const Hash &tid) override; + + void set_label(const std::string &address, const std::string &label) override; + std::string get_label(const std::string &address) const override { return std::string(); } + + OutputHandler get_output_handler() const override; + bool detect_our_output(const Hash &tid, const Hash &tx_inputs_hash, const boost::optional &kd, + size_t out_index, const PublicKey &spend_public_key, const SecretKey &secret_scalar, const OutputKey &, + Amount *, KeyPair *output_keypair, AccountAddress *) override; + bool detect_not_our_output(bool tx_amethyst, const Hash &tid, const Hash &tx_inputs_hash, + boost::optional *, size_t out_index, const OutputKey &, Amount *, AccountAddress *) override; +}; + +// stores either mnemonic or some seeds if view-only +// stores number of used addresses, (per net) creation timestamp, (per net) payment queue +class WalletHD : public Wallet { + platform::sqlite::Dbi m_db_dbi; + uint8_t m_address_type = 0; + KeyPair m_spend_key_base; + size_t m_used_address_count = 1; + std::map m_labels; + + static BinaryArray encrypt_data(const crypto::chacha_key &wallet_key, const BinaryArray &data); + static BinaryArray decrypt_data(const crypto::chacha_key &wallet_key, const uint8_t *value_data, size_t value_size); + void put_salt(const BinaryArray &salt); + BinaryArray get_salt() const; + void commit(); + + void put(const std::string &key, const common::BinaryArray &value, bool nooverwrite); + bool get(const std::string &key, common::BinaryArray &value) const; + void put(const std::string &key, const std::string &value, bool nooverwrite); + bool get(const std::string &key, std::string &value) const; + + void generate_ahead(); + void generate_ahead1(size_t counter, std::vector &result) const; + void load(); + + std::vector> parameters_get() const; + std::vector> payment_queue_get2() const; + void payment_queue_add(const Hash &tid, const std::string &net, const BinaryArray &binary_transaction); + +public: + static std::string generate_mnemonic(size_t bits, uint32_t version); + static bool is_sqlite(const std::string &full_path); + // In contrast with WalletContainerStorage, we must know read_only flag, because otherwise + // inability to save address count will lead to wallet losing track of funds due to skipping outputs + WalletHD(const Currency ¤cy, logging::ILogger &log, const std::string &path, const std::string &password, + bool readonly); + WalletHD(const Currency ¤cy, logging::ILogger &log, const std::string &path, const std::string &password, + const std::string &mnemonic, uint8_t address_type, Timestamp creation_timestamp, + const std::string &mnemonic_password); + bool is_view_only() const override { return m_spend_key_base.secret_key == SecretKey{}; } + bool is_deterministic() const override { return true; } + bool is_unlinkable() const override { return true; } + bool is_auditable() const override { return m_address_type == AccountAddressUnlinkable::type_tag_auditable; } + size_t get_actual_records_count() const override { return m_used_address_count; } + std::vector generate_new_addresses( + const std::vector &sks, Timestamp ct, Timestamp now, bool *rescan_from_ct) override; + AccountAddress record_to_address(const WalletRecord &record) const override; + bool get_record(WalletRecord &record, const AccountAddress &) const override; + void set_password(const std::string &password) override; + void export_wallet(const std::string &export_path, const std::string &new_password, bool view_only, + bool view_outgoing_addresses) const override; + std::string export_keys() const override; + + // Date first unlinkable addresses appeared in stagenet blockchain + Timestamp get_oldest_timestamp() const override { return std::max(m_oldest_timestamp, 1540489975); } + void on_first_output_found(Timestamp ts) override; + void create_look_ahead_records(size_t count) override; + + void backup(const std::string &dst_name, const std::string &pass) const override; + + History load_history(const Hash &tid) const override; + + std::vector payment_queue_get() const override; + void payment_queue_add(const Hash &tid, const BinaryArray &binary_transaction) override; + void payment_queue_remove(const Hash &tid) override; + + void set_label(const std::string &address, const std::string &label) override; + std::string get_label(const std::string &address) const override; + + OutputHandler get_output_handler() const override; + bool detect_our_output(const Hash &tid, const Hash &tx_inputs_hash, const boost::optional &kd, + size_t out_index, const PublicKey &spend_public_key, const SecretKey &secret_scalar, const OutputKey &, + Amount *, KeyPair *output_keypair, AccountAddress *) override; + bool detect_not_our_output(bool tx_amethyst, const Hash &tid, const Hash &tx_inputs_hash, + boost::optional *, size_t out_index, const OutputKey &, Amount *, AccountAddress *) override; }; -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/WalletNode.cpp b/src/Core/WalletNode.cpp index f1d06c88..81b7fad3 100644 --- a/src/Core/WalletNode.cpp +++ b/src/Core/WalletNode.cpp @@ -5,28 +5,33 @@ #include "Config.hpp" #include "CryptoNoteTools.hpp" #include "TransactionBuilder.hpp" +#include "TransactionExtra.hpp" #include "common/Math.hpp" +#include "http/Client.hpp" +#include "http/Server.hpp" #include "platform/Time.hpp" #include "seria/BinaryInputStream.hpp" #include "seria/BinaryOutputStream.hpp" #include "seria/KVBinaryInputStream.hpp" #include "seria/KVBinaryOutputStream.hpp" -using namespace bytecoin; +using namespace cn; const WalletNode::HandlersMap WalletNode::m_jsonrpc_handlers = { - {api::walletd::GetStatus::method(), json_rpc::make_member_method(&WalletNode::handle_get_status)}, - {api::walletd::GetAddresses::method(), json_rpc::make_member_method(&WalletNode::handle_get_addresses)}, - {api::walletd::GetWalletInfo::method(), json_rpc::make_member_method(&WalletNode::handle_get_wallet_info)}, - {api::walletd::CreateAddresses::method(), json_rpc::make_member_method(&WalletNode::handle_create_address_list)}, - {api::walletd::GetViewKeyPair::method(), json_rpc::make_member_method(&WalletNode::handle_get_view_key)}, - {api::walletd::GetBalance::method(), json_rpc::make_member_method(&WalletNode::handle_get_balance)}, - {api::walletd::GetUnspents::method(), json_rpc::make_member_method(&WalletNode::handle_get_unspent)}, - {api::walletd::GetTransfers::method(), json_rpc::make_member_method(&WalletNode::handle_get_transfers)}, - {api::walletd::CreateTransaction::method(), json_rpc::make_member_method(&WalletNode::handle_create_transaction)}, - {api::walletd::SendTransaction::method(), json_rpc::make_member_method(&WalletNode::handle_send_transaction)}, - {api::walletd::CreateSendProof::method(), json_rpc::make_member_method(&WalletNode::handle_create_sendproof)}, - {api::walletd::GetTransaction::method(), json_rpc::make_member_method(&WalletNode::handle_get_transaction)}}; + {api::walletd::GetStatus::method(), json_rpc::make_member_method(&WalletNode::on_get_status)}, + {api::walletd::GetAddresses::method(), json_rpc::make_member_method(&WalletNode::on_get_addresses)}, + {api::walletd::GetWalletInfo::method(), json_rpc::make_member_method(&WalletNode::on_get_wallet_info)}, + {api::walletd::GetWalletRecords::method(), json_rpc::make_member_method(&WalletNode::on_get_wallet_records)}, + {api::walletd::SetAddressLabel::method(), json_rpc::make_member_method(&WalletNode::on_set_label)}, + {api::walletd::CreateAddresses::method(), json_rpc::make_member_method(&WalletNode::on_create_addresses)}, + {api::walletd::GetViewKeyPair::method(), json_rpc::make_member_method(&WalletNode::on_get_view_key)}, + {api::walletd::GetBalance::method(), json_rpc::make_member_method(&WalletNode::on_get_balance)}, + {api::walletd::GetUnspents::method(), json_rpc::make_member_method(&WalletNode::on_get_unspent)}, + {api::walletd::GetTransfers::method(), json_rpc::make_member_method(&WalletNode::on_get_transfers)}, + {api::walletd::CreateTransaction::method(), json_rpc::make_member_method(&WalletNode::on_create_transaction)}, + {api::walletd::SendTransaction::method(), json_rpc::make_member_method(&WalletNode::on_send_transaction)}, + {api::walletd::CreateSendproof::method(), json_rpc::make_member_method(&WalletNode::on_create_sendproof)}, + {api::walletd::GetTransaction::method(), json_rpc::make_member_method(&WalletNode::on_get_transaction)}}; WalletNode::WalletNode(Node *inproc_node, logging::ILogger &log, const Config &config, WalletState &wallet_state) : WalletSync(log, config, wallet_state, std::bind(&WalletNode::advance_long_poll, this)) @@ -37,7 +42,9 @@ WalletNode::WalletNode(Node *inproc_node, logging::ILogger &log, const Config &c std::bind(&WalletNode::on_api_http_disconnect, this, _1))); } -bool WalletNode::on_api_http_request(http::Client *who, http::RequestData &&request, http::ResponseData &response) { +WalletNode::~WalletNode() {} // we have unique_ptr to incomplete type + +bool WalletNode::on_api_http_request(http::Client *who, http::RequestBody &&request, http::ResponseBody &response) { response.r.add_headers_nocache(); bool method_found = false; if (request.r.uri == api::walletd::url()) { @@ -49,29 +56,28 @@ bool WalletNode::on_api_http_request(http::Client *who, http::RequestData &&requ return m_inproc_node->on_json_rpc(who, std::move(request), response); m_log(logging::INFO) << "http_request node tunneling url=" << request.r.uri << " start of body=" << request.body.substr(0, 200) << std::endl; - http::RequestData original_request; + http::RequestBody original_request; original_request.r = request.r; request.r.http_version_major = 1; request.r.http_version_minor = 1; request.r.keep_alive = true; request.r.basic_authorization = m_config.bytecoind_authorization; add_waiting_command(who, std::move(original_request), common::JsonValue(nullptr), std::move(request), - [=](const WaitingClient &wc, http::ResponseData &&send_response) mutable { + [](const WaitingClient &wc, http::ResponseBody &&send_response) mutable { send_response.r.http_version_major = wc.original_request.r.http_version_major; send_response.r.http_version_minor = wc.original_request.r.http_version_minor; send_response.r.keep_alive = wc.original_request.r.keep_alive; - // bytecoind never sends connection-close, so we are safe to retain all - // headers + // bytecoind never sends connection-close, so we are safe to retain all headers wc.original_who->write(std::move(send_response)); - }, - [=](const WaitingClient &wc, std::string err) { - http::ResponseData send_response; + }, + [](const WaitingClient &wc, std::string err) { + http::ResponseBody send_response; send_response.r.http_version_major = wc.original_request.r.http_version_major; send_response.r.http_version_minor = wc.original_request.r.http_version_minor; send_response.r.keep_alive = wc.original_request.r.keep_alive; send_response.r.status = 504; // TODO -test this code path wc.original_who->write(std::move(send_response)); - }); + }); return false; } @@ -88,7 +94,7 @@ void WalletNode::on_api_http_disconnect(http::Client *who) { } bool WalletNode::on_json_rpc( - http::Client *who, http::RequestData &&request, http::ResponseData &response, bool &method_found) { + http::Client *who, http::RequestBody &&request, http::ResponseBody &response, bool &method_found) { method_found = false; response.r.headers.push_back({"Content-Type", "application/json; charset=utf-8"}); @@ -101,10 +107,6 @@ bool WalletNode::on_json_rpc( auto it = m_jsonrpc_handlers.find(json_req.get_method()); if (it == m_jsonrpc_handlers.end()) { return false; - // m_log(logging::INFO) << "json request method not - // found - " << json_req.get_method() << std::endl; - // throw - // json_rpc::Error(json_rpc::METHOD_NOT_FOUND); } method_found = true; if (!m_config.walletd_authorization.empty() && @@ -113,9 +115,6 @@ bool WalletNode::on_json_rpc( response.r.status = 401; return true; } - // m_log(logging::INFO) << "json request method=" << - // json_req.get_method() - //<< std::endl; std::string response_body; if (!it->second(this, who, std::move(request), std::move(json_req), response_body)) return false; @@ -147,14 +146,12 @@ api::walletd::GetStatus::Response WalletNode::create_status_response() const { return response; } -bool WalletNode::handle_get_status(http::Client *who, http::RequestData &&raw_request, - json_rpc::Request &&raw_js_request, api::walletd::GetStatus::Request &&request, - api::walletd::GetStatus::Response &response) { +bool WalletNode::on_get_status(http::Client *who, http::RequestBody &&raw_request, json_rpc::Request &&raw_js_request, + api::walletd::GetStatus::Request &&request, api::walletd::GetStatus::Response &response) { response = create_status_response(); if (!response.ready_for_longpoll(request)) { - // m_log(logging::INFO) << "handle_get_status will long poll, json=" - //<< - // raw_request.body << std::endl; + // m_log(logging::INFO) << "on_get_status will long poll, json=" + // << raw_request.body << std::endl; LongPollClient lpc; lpc.original_who = who; lpc.original_request = raw_request; @@ -166,18 +163,21 @@ bool WalletNode::handle_get_status(http::Client *who, http::RequestData &&raw_re return true; } -bool WalletNode::handle_get_addresses(http::Client *, http::RequestData &&, json_rpc::Request &&, +bool WalletNode::on_get_addresses(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetAddresses::Request &&request, api::walletd::GetAddresses::Response &response) { - // TODO - from_address, max_count const Wallet &wa = m_wallet_state.get_wallet(); - response.total_address_count = common::integer_cast(wa.get_records().size()); - response.addresses.reserve(wa.get_records().size()); - if (request.need_secret_spend_keys) - response.secret_spend_keys.reserve(wa.get_records().size()); - for (size_t i = request.from_address; i < wa.get_records().size(); ++i) { + response.total_address_count = wa.get_actual_records_count(); + response.addresses.reserve(response.total_address_count); + if (request.need_secret_spend_keys) { + if (!m_config.secrets_via_api) + throw json_rpc::Error(json_rpc::INVALID_PARAMS, + "To allow getting secrets via API, walletd must be launched with '--secrets-via-api' argument."); + response.secret_spend_keys.reserve(response.total_address_count); + } + for (size_t i = request.from_address; i < response.total_address_count; ++i) { if (response.addresses.size() >= request.max_count) break; - AccountPublicAddress addr{wa.get_records().at(i).spend_public_key, wa.get_view_public_key()}; + AccountAddress addr = wa.record_to_address(wa.get_records().at(i)); response.addresses.push_back(m_wallet_state.get_currency().account_address_as_string(addr)); if (request.need_secret_spend_keys) response.secret_spend_keys.push_back(wa.get_records().at(i).spend_secret_key); @@ -185,38 +185,91 @@ bool WalletNode::handle_get_addresses(http::Client *, http::RequestData &&, json return true; } -bool WalletNode::handle_get_wallet_info(http::Client *, http::RequestData &&, json_rpc::Request &&, +bool WalletNode::on_get_wallet_info(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetWalletInfo::Request &&request, api::walletd::GetWalletInfo::Response &response) { const Wallet &wa = m_wallet_state.get_wallet(); response.view_only = wa.is_view_only(); - response.mineproof_secret = wa.get_coinbase_tx_derivation_seed(); - response.total_address_count = common::integer_cast(wa.get_records().size()); + response.deterministic = wa.is_deterministic(); + response.auditable = wa.is_auditable(); + response.total_address_count = wa.get_actual_records_count(); response.wallet_creation_timestamp = wa.get_oldest_timestamp(); response.first_address = m_wallet_state.get_currency().account_address_as_string(wa.get_first_address()); response.net = m_config.net; + if (request.need_secrets) { + if (!m_config.secrets_via_api) + throw json_rpc::Error(json_rpc::INVALID_PARAMS, + "To allow getting secrets via API, walletd must be launched with '--secrets-via-api' argument."); + if (wa.is_deterministic()) + response.mnemonic = m_wallet_state.get_wallet().export_keys(); + else + response.import_keys = m_wallet_state.get_wallet().export_keys(); + response.secret_view_key = wa.get_view_secret_key(); + response.public_view_key = wa.get_view_public_key(); + } + return true; +} +bool WalletNode::on_get_wallet_records(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::walletd::GetWalletRecords::Request &&request, api::walletd::GetWalletRecords::Response &response) { + const Wallet &wa = m_wallet_state.get_wallet(); + if (request.create) { + if (!wa.is_deterministic()) + throw json_rpc::Error( + json_rpc::INVALID_PARAMS, "wallet is not deterministic, impossible to create addresses by index"); + if (request.count == std::numeric_limits::max()) + throw json_rpc::Error(json_rpc::INVALID_PARAMS, "If 'create' set to true, you must also set 'max_count'"); + if (request.count > std::numeric_limits::max() - request.index) + throw json_rpc::Error(json_rpc::INVALID_PARAMS, "You asked to create too many addresses."); + m_wallet_state.create_addresses(request.index + request.count); + } + response.total_count = wa.get_actual_records_count(); + response.records.reserve(response.total_count); + for (size_t i = request.index; i < response.total_count; ++i) { + if (response.records.size() >= request.count) + break; + const WalletRecord &record = wa.get_records().at(i); + AccountAddress addr = wa.record_to_address(record); + api::walletd::GetWalletRecords::Record wr; + wr.index = i; + wr.address = m_wallet_state.get_currency().account_address_as_string(addr); + wr.label = wa.get_label(wr.address); + if (request.need_secrets) { + wr.public_spend_key = record.spend_public_key; + wr.secret_spend_key = record.spend_secret_key; + } + response.records.push_back(std::move(wr)); + } return true; } -bool WalletNode::handle_get_view_key(http::Client *, http::RequestData &&, json_rpc::Request &&, +bool WalletNode::on_set_label(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::walletd::SetAddressLabel::Request &&request, api::walletd::SetAddressLabel::Response &) { + m_wallet_state.get_wallet().set_label(request.address, request.label); + return true; +} + +bool WalletNode::on_get_view_key(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetViewKeyPair::Request &&, api::walletd::GetViewKeyPair::Response &response) { + if (!m_config.secrets_via_api) + throw json_rpc::Error(json_rpc::INVALID_PARAMS, + "To allow getting secrets via API, walletd must be launched with '--secrets-via-api' argument."); response.public_view_key = m_wallet_state.get_wallet().get_view_public_key(); response.secret_view_key = m_wallet_state.get_wallet().get_view_secret_key(); response.import_keys = m_wallet_state.get_wallet().export_keys(); return true; } -bool WalletNode::handle_create_address_list(http::Client *, http::RequestData &&, json_rpc::Request &&, +bool WalletNode::on_create_addresses(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::CreateAddresses::Request &&request, api::walletd::CreateAddresses::Response &response) { - if (request.secret_spend_keys.empty()) - return true; if (m_wallet_state.get_wallet().is_view_only()) throw json_rpc::Error(json_rpc::INVALID_PARAMS, "wallet is view-only, impossible to create addresses"); + if (request.secret_spend_keys.empty()) + return true; auto records = m_wallet_state.generate_new_addresses( request.secret_spend_keys, request.creation_timestamp, platform::now_unix_timestamp()); response.addresses.reserve(records.size()); response.secret_spend_keys.reserve(records.size()); for (auto &&rec : records) { - AccountPublicAddress addr{rec.spend_public_key, m_wallet_state.get_wallet().get_view_public_key()}; + AccountAddress addr = m_wallet_state.get_wallet().record_to_address(rec); response.addresses.push_back(m_wallet_state.get_currency().account_address_as_string(addr)); response.secret_spend_keys.push_back(rec.spend_secret_key); } @@ -226,14 +279,14 @@ bool WalletNode::handle_create_address_list(http::Client *, http::RequestData && void WalletNode::check_address_in_wallet_or_throw(const std::string &addr) const { if (addr.empty()) return; - AccountPublicAddress address; + AccountAddress address; if (!m_wallet_state.get_currency().parse_account_address_string(addr, &address)) throw api::ErrorAddress(api::ErrorAddress::ADDRESS_FAILED_TO_PARSE, "Failed to parse address", addr); if (!m_wallet_state.get_wallet().is_our_address(address)) throw api::ErrorAddress(api::ErrorAddress::ADDRESS_NOT_IN_WALLET, "Address not in wallet", addr); } -bool WalletNode::handle_get_balance(http::Client *, http::RequestData &&, json_rpc::Request &&, +bool WalletNode::on_get_balance(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetBalance::Request &&request, api::walletd::GetBalance::Response &response) { check_address_in_wallet_or_throw(request.address); Height height_or_depth = api::ErrorWrongHeight::fix_height_or_depth( @@ -242,7 +295,7 @@ bool WalletNode::handle_get_balance(http::Client *, http::RequestData &&, json_r return true; } -bool WalletNode::handle_get_unspent(http::Client *, http::RequestData &&, json_rpc::Request &&, +bool WalletNode::on_get_unspent(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetUnspents::Request &&request, api::walletd::GetUnspents::Response &response) { check_address_in_wallet_or_throw(request.address); Height height_or_depth = api::ErrorWrongHeight::fix_height_or_depth( @@ -254,12 +307,19 @@ bool WalletNode::handle_get_unspent(http::Client *, http::RequestData &&, json_r return true; } -bool WalletNode::handle_get_transfers(http::Client *, http::RequestData &&, json_rpc::Request &&, +bool WalletNode::on_get_transfers(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetTransfers::Request &&request, api::walletd::GetTransfers::Response &response) { check_address_in_wallet_or_throw(request.address); - response.next_to_height = request.to_height; response.next_from_height = request.from_height; - response.blocks = m_wallet_state.api_get_transfers( + response.next_to_height = request.to_height; + // GetTransfers API is documented to accept (from..to] range, + // but code works with traditional [from..to) range + // This is design error in API. We now have to do clumsy recalculations for iteration. + if (request.from_height != std::numeric_limits::max()) + request.from_height += 1; + if (request.to_height != std::numeric_limits::max()) + request.to_height += 1; + response.blocks = m_wallet_state.api_get_transfers( request.address, &request.from_height, &request.to_height, request.forward, request.desired_transaction_count); if (request.from_height < m_wallet_state.get_tip_height() && request.to_height >= m_wallet_state.get_tip_height()) { api::Block pool_block = m_wallet_state.api_get_pool_as_history(request.address); @@ -272,25 +332,17 @@ bool WalletNode::handle_get_transfers(http::Client *, http::RequestData &&, json } response.unlocked_transfers = m_wallet_state.api_get_unlocked_transfers(request.address, request.from_height, request.to_height); - // auto unlocked_outputs = - // m_wallet_state.api_get_unlocked_outputs(request.address, request.from_height, request.to_height); - // response.unlocked_transfers.reserve(unlocked_outputs.size()); - // for (auto &&lou : unlocked_outputs) { - // api::Transfer tr; - // tr.ours = true; - // tr.amount = lou.second.amount; - // tr.address = lou.second.address; - // tr.outputs.push_back(lou.second); - // response.unlocked_transfers.push_back(std::move(tr)); - // } - if (request.forward) - response.next_from_height = request.to_height; - else - response.next_to_height = request.from_height; + if (request.forward) { + if (request.to_height != std::numeric_limits::max()) + response.next_from_height = request.to_height - 1; + else + response.next_from_height = request.to_height; + } else + response.next_to_height = request.from_height - 1; return true; } -bool WalletNode::handle_create_transaction(http::Client *who, http::RequestData &&raw_request, +bool WalletNode::on_create_transaction(http::Client *who, http::RequestBody &&raw_request, json_rpc::Request &&raw_js_request, api::walletd::CreateTransaction::Request &&request, api::walletd::CreateTransaction::Response &response) { m_log(logging::TRACE) << "create_transaction request tip_height=" << m_wallet_state.get_tip_height() @@ -302,24 +354,38 @@ bool WalletNode::handle_create_transaction(http::Client *who, http::RequestData } if (!response.transactions_required.empty()) return true; - if (m_last_node_status.next_block_effective_median_size == 0) - throw json_rpc::Error(json_rpc::INVALID_PARAMS, "Next block median size unknown, need to sync to bytecoind"); if (request.transaction.anonymity > 100) // Arbitrary value - throw json_rpc::Error(api::walletd::CreateTransaction::NOT_ENOUGH_ANONYMITY, + throw json_rpc::Error(api::walletd::CreateTransaction::TOO_MUCH_ANONYMITY, "Wallet will not create transactions with anonymity > 100 because large anonymity values actually reduce anonymity due to tiny number of similar transactions"); - Height confirmed_height_or_depth = api::ErrorWrongHeight::fix_height_or_depth( - request.confirmed_height_or_depth, m_wallet_state.get_tip_height(), true, false); + if (request.transaction.anonymity != 0 && m_wallet_state.get_wallet().is_auditable()) + request.transaction.anonymity = 0; + // throw json_rpc::Error(api::walletd::CreateTransaction::TOO_MUCH_ANONYMITY, + // "Auditable wallet requires 0 anonymity when sending (checked and enforced by consensus)"); + const auto min_anonymity = + m_wallet_state.get_wallet().is_auditable() + ? 0 + : m_wallet_state.get_currency().minimum_anonymity(m_wallet_state.get_tip().major_version); + const auto good_anonymity = std::max(min_anonymity, request.transaction.anonymity); + Height confirmed_height = api::ErrorWrongHeight::fix_height_or_depth( + request.confirmed_height_or_depth, m_wallet_state.get_tip_height(), true, false); + bool is_amethyst = false; + { + api::BlockHeader confirmed_header; + if (m_wallet_state.read_chain(confirmed_height, confirmed_header) && + confirmed_header.major_version >= m_wallet_state.get_currency().amethyst_block_version) + is_amethyst = true; + } if (request.fee_per_byte == std::numeric_limits::max()) { if (m_last_node_status.recommended_fee_per_byte == 0) throw json_rpc::Error(json_rpc::INVALID_PARAMS, "'fee_per_byte' set to 0, and it is impossible to set it to 'status.recommended_fee_per_byte', " - "because walletd never connected to bytecoind after it was restarted"); + "because walletd never connected to " CRYPTONOTE_NAME "d after it was restarted"); request.fee_per_byte = m_last_node_status.recommended_fee_per_byte; } if (m_wallet_state.get_wallet().is_view_only()) throw json_rpc::Error(api::walletd::CreateTransaction::VIEW_ONLY_WALLET, "Unable to create transaction - view-only wallet contains no spend keys"); - AccountPublicAddress change_addr; + AccountAddress change_addr; if (request.any_spend_address && request.change_address.empty()) change_addr = m_wallet_state.get_wallet().get_first_address(); else { @@ -331,49 +397,54 @@ bool WalletNode::handle_create_transaction(http::Client *who, http::RequestData // We do not require that change_addr should be in our wallet if (request.spend_addresses.empty() && !request.any_spend_address) throw json_rpc::Error(json_rpc::INVALID_PARAMS, - "Empty spend addresses requires setting " - "'any_spend_address':true for additional protection"); + "Empty spend addresses requires setting 'any_spend_address':true for additional protection"); if (!request.spend_addresses.empty() && request.any_spend_address) throw json_rpc::Error(json_rpc::INVALID_PARAMS, - "Non-empty spend addresses requires setting " - "'any_spend_address':false for additional protection"); - std::unordered_map only_records; - // We protect against our programming errors by fetching spend keys in only_records - // and passing only_records to transaction builder, so that it cannot accidentally - // spend funds from unintended addresses + "Non-empty spend addresses requires setting 'any_spend_address':false for additional protection"); + std::set only_records; + // We protect against our programming errors by filling only_records and then checking against them during signing for (const auto &ad : request.spend_addresses) { - AccountPublicAddress addr; + AccountAddress addr; if (!m_wallet_state.get_currency().parse_account_address_string(ad, &addr)) throw api::ErrorAddress(api::ErrorAddress::ADDRESS_FAILED_TO_PARSE, "Failed to parse spend address", ad); WalletRecord record; if (!m_wallet_state.get_wallet().get_record(record, addr)) throw api::ErrorAddress(api::ErrorAddress::ADDRESS_NOT_IN_WALLET, "Address not in wallet", ad); - only_records[addr.spend_public_key] = record; + only_records.insert(addr); } - TransactionBuilder builder(m_wallet_state.get_currency(), request.transaction.unlock_block_or_timestamp); Wallet::History history; + TransactionBuilder builder; + builder.m_transaction.version = + is_amethyst ? m_wallet_state.get_currency().amethyst_transaction_version : uint8_t(1); + builder.m_transaction.unlock_block_or_timestamp = request.transaction.unlock_block_or_timestamp; if (request.transaction.payment_id != Hash{}) - builder.set_payment_id(request.transaction.payment_id); + extra_add_payment_id(builder.m_transaction.extra, request.transaction.payment_id); Amount sum_positive_transfers = 0; - std::map combined_outputs; + std::map combined_outputs; for (const auto &tr : request.transaction.transfers) { if (tr.amount <= 0) // Not an output throw json_rpc::Error(json_rpc::INVALID_PARAMS, - "Negative transfer amount " + common::to_string(tr.amount) + " for address " + tr.address); - AccountPublicAddress addr; + "Negative or zero transfer amount " + m_wallet_state.get_currency().format_amount(tr.amount) + + " for address " + tr.address); + AccountAddress addr; if (!m_wallet_state.get_currency().parse_account_address_string(tr.address, &addr)) throw api::ErrorAddress( api::ErrorAddress::ADDRESS_FAILED_TO_PARSE, "Failed to parse transfer address", tr.address); combined_outputs[addr] += tr.amount; - history.insert(addr); // TODO - check if unlinkable - sum_positive_transfers += tr.amount; + if (addr.type() == typeid(AccountAddressSimple)) + history.insert(boost::get(addr)); + if (!is_amethyst && addr.type() == typeid(AccountAddressUnlinkable)) + throw json_rpc::Error( + json_rpc::INVALID_PARAMS, "You cannot send to unlinkable address before amethyst upgrade"); + if (!add_amount(sum_positive_transfers, tr.amount)) + throw json_rpc::Error(json_rpc::INVALID_PARAMS, "Sum of transfers overflow max amount "); } size_t total_outputs = 0; for (const auto &aa : combined_outputs) { std::vector decomposed_amounts; - decompose_amount(aa.second, m_wallet_state.get_currency().default_dust_threshold, &decomposed_amounts); + decompose_amount(aa.second, m_wallet_state.get_currency().min_dust_threshold, &decomposed_amounts); total_outputs += decomposed_amounts.size(); } if (sum_positive_transfers == 0) @@ -384,37 +455,39 @@ bool WalletNode::handle_create_transaction(http::Client *who, http::RequestData Amount change = 0; Amount receiver_fee = 0; std::vector unspents; - Amount total_unspents = 0; - if (!request.spend_addresses.empty()) + Amount total_unspents = 0; + const Amount sum_positive_transfers_2x = sum_positive_transfers <= std::numeric_limits::max() / 2 + ? sum_positive_transfers * 2 + : std::numeric_limits::max(); + if (!request.spend_addresses.empty()) { for (auto &&ad : request.spend_addresses) { if (!m_wallet_state.api_add_unspent( - &unspents, &total_unspents, ad, confirmed_height_or_depth, sum_positive_transfers * 2)) + &unspents, &total_unspents, ad, confirmed_height, sum_positive_transfers_2x)) break; // found enough funds } - else + } else { m_wallet_state.api_add_unspent( - &unspents, &total_unspents, std::string(), confirmed_height_or_depth, sum_positive_transfers * 2); + &unspents, &total_unspents, std::string(), confirmed_height, sum_positive_transfers_2x); + } UnspentSelector selector(m_log.get_logger(), m_wallet_state.get_currency(), std::move(unspents)); // First we select just outputs with sum = 2x requires sum try { - selector.select_optimal_outputs(m_wallet_state.get_tip_height(), m_wallet_state.get_tip().timestamp, - confirmed_height_or_depth, m_last_node_status.next_block_effective_median_size, - request.transaction.anonymity, sum_positive_transfers, total_outputs, request.fee_per_byte, optimization, + selector.select_optimal_outputs(m_wallet_state.get_currency().get_recommended_max_transaction_size(), + good_anonymity, min_anonymity, sum_positive_transfers, total_outputs, request.fee_per_byte, optimization, &change, request.subtract_fee_from_amount ? &receiver_fee : nullptr); } catch (const std::exception &) { - // If selected outputs do not fit in next_block_effective_median_size, we try all outputs + // If selected outputs do not fit in recommended_max_transaction_size, we try all outputs unspents.clear(); total_unspents = 0; if (!request.spend_addresses.empty()) for (auto &&ad : request.spend_addresses) { - m_wallet_state.api_add_unspent(&unspents, &total_unspents, ad, confirmed_height_or_depth); + m_wallet_state.api_add_unspent(&unspents, &total_unspents, ad, confirmed_height); } else - m_wallet_state.api_add_unspent(&unspents, &total_unspents, std::string(), confirmed_height_or_depth); + m_wallet_state.api_add_unspent(&unspents, &total_unspents, std::string(), confirmed_height); selector.reset(std::move(unspents)); - selector.select_optimal_outputs(m_wallet_state.get_tip_height(), m_wallet_state.get_tip().timestamp, - confirmed_height_or_depth, m_last_node_status.next_block_effective_median_size, - request.transaction.anonymity, sum_positive_transfers, total_outputs, request.fee_per_byte, optimization, + selector.select_optimal_outputs(m_wallet_state.get_currency().get_recommended_max_transaction_size(), + good_anonymity, min_anonymity, sum_positive_transfers, total_outputs, request.fee_per_byte, optimization, &change, request.subtract_fee_from_amount ? &receiver_fee : nullptr); } if (receiver_fee != 0) { @@ -424,7 +497,8 @@ bool WalletNode::handle_create_transaction(http::Client *who, http::RequestData for (const auto &tr : request.transaction.transfers) { if (tr.amount <= 0) // Not an output throw json_rpc::Error(json_rpc::INVALID_PARAMS, - "Negative transfer amount " + common::to_string(tr.amount) + " for address " + tr.address); + "Negative transfer amount " + m_wallet_state.get_currency().format_amount(tr.amount) + + " for address " + tr.address); Amount am = static_cast(tr.amount); if (am <= receiver_fee) { receiver_fee -= am; @@ -432,64 +506,67 @@ bool WalletNode::handle_create_transaction(http::Client *who, http::RequestData } am -= receiver_fee; receiver_fee = 0; - AccountPublicAddress addr; + AccountAddress addr; if (!m_wallet_state.get_currency().parse_account_address_string(tr.address, &addr)) throw api::ErrorAddress( api::ErrorAddress::ADDRESS_FAILED_TO_PARSE, "Failed to parse transfer address", tr.address); combined_outputs[addr] += am; - history.insert(addr); // TODO - check if unlinkable + if (addr.type() == typeid(AccountAddressSimple)) + history.insert(boost::get(addr)); } if (combined_outputs.empty()) throw json_rpc::Error( json_rpc::INVALID_PARAMS, "Sum of all transfers was less than fee - no transfers would be made"); } // Selector ensures the change should be as "round" as possible - if (change > 0) { + if (change > 0) combined_outputs[change_addr] += change; - if (!m_wallet_state.get_wallet().is_our_address(change_addr)) - history.insert(change_addr); // TODO - check if unlinkable - } for (const auto &aa : combined_outputs) { std::vector decomposed_amounts; - decompose_amount(aa.second, m_wallet_state.get_currency().default_dust_threshold, &decomposed_amounts); + decompose_amount(aa.second, m_wallet_state.get_currency().min_dust_threshold, &decomposed_amounts); for (auto &&da : decomposed_amounts) builder.add_output(da, aa.first); } - api::bytecoind::GetRandomOutputs::Request ra_request; - ra_request.confirmed_height_or_depth = confirmed_height_or_depth; - ra_request.output_count = - request.transaction.anonymity + 1; // Ask excess output for the case of collision with our output + api::cnd::GetRandomOutputs::Request ra_request; + ra_request.confirmed_height_or_depth = confirmed_height; + ra_request.output_count = good_anonymity + 1; + // We ask excess output for the case of collision with our output + // We ask minimum anonymity, though less than requested might be returned ra_request.amounts = selector.get_ra_amounts(); - api::bytecoind::GetRandomOutputs::Response ra_response; - if (m_inproc_node || request.transaction.anonymity == 0) { - if(request.transaction.anonymity != 0) - m_inproc_node->on_get_random_outputs( - nullptr, http::RequestData(raw_request), json_rpc::Request(), std::move(ra_request), ra_response); - selector.add_mixed_inputs(m_wallet_state.get_wallet().get_view_secret_key(), - request.any_spend_address ? &m_wallet_state.get_wallet() : nullptr, only_records, &builder, - request.transaction.anonymity, std::move(ra_response)); - Transaction tx = builder.sign(m_wallet_state.get_wallet().get_tx_derivation_seed()); + if (m_inproc_node) { + api::cnd::GetRandomOutputs::Response ra_response; + m_inproc_node->on_get_random_outputs( + nullptr, http::RequestBody(raw_request), json_rpc::Request(), std::move(ra_request), ra_response); + const auto actual_anonymity = selector.add_mixed_inputs(&builder, good_anonymity, std::move(ra_response)); + if (actual_anonymity < request.transaction.anonymity) { + m_log(logging::TRACE) << "Transaction anonymity is " << actual_anonymity << "/" + << request.transaction.anonymity << std::endl; + // throw json_rpc::Error(api::walletd::CreateTransaction::NOT_ENOUGH_ANONYMITY, + // "Requested anonymity too high, please reduce anonymity for this transaction."); + } + Transaction tx = builder.sign( + m_wallet_state, &m_wallet_state.get_wallet(), request.any_spend_address ? nullptr : &only_records); response.binary_transaction = seria::to_binary(tx); - const Hash tx_hash = get_transaction_hash(tx); - if (request.save_history && !m_wallet_state.get_wallet().save_history(tx_hash, history)) { + const Hash tx_hash = get_transaction_hash(tx); + if (!is_amethyst && request.save_history && !m_wallet_state.get_wallet().save_history(tx_hash, history)) { m_log(logging::ERROR) << "Saving transaction history failed, you will need to pass list of destination addresses to generate sending proof for tx=" << tx_hash << std::endl; response.save_history_error = true; } api::Transaction ptx{}; - if (!m_wallet_state.parse_raw_transaction(ptx, tx, tx_hash)) - throw json_rpc::Error(json_rpc::INTERNAL_ERROR, "Created trsnsaction cannot be parsed"); + if (!m_wallet_state.parse_raw_transaction(false, ptx, std::move(tx), tx_hash)) + throw json_rpc::Error(json_rpc::INTERNAL_ERROR, "Created transaction cannot be parsed"); response.transaction = ptx; return true; } - api::walletd::CreateTransaction::Request request_copy = request; // TODO ??? - http::RequestData new_request = - json_rpc::create_request(api::bytecoind::url(), api::bytecoind::GetRandomOutputs::method(), ra_request); + + http::RequestBody new_request = + json_rpc::create_request(api::cnd::url(), api::cnd::GetRandomOutputs::method(), ra_request); new_request.r.basic_authorization = m_config.bytecoind_authorization; m_log(logging::TRACE) << "sending get_random_outputs, body=" << new_request.body << std::endl; add_waiting_command(who, std::move(raw_request), raw_js_request.get_id().get(), std::move(new_request), - [=](const WaitingClient &wc, http::ResponseData &&random_response) mutable { + [=](const WaitingClient &wc, http::ResponseBody &&random_response) mutable { m_log(logging::TRACE) << "got response to get_random_outputs, status=" << random_response.r.status << " body " << random_response.body << std::endl; if (random_response.r.status != 200) { @@ -498,120 +575,125 @@ bool WalletNode::handle_create_transaction(http::Client *who, http::RequestData Transaction tx{}; api::walletd::CreateTransaction::Response last_response; json_rpc::Response json_resp(random_response.body); - api::bytecoind::GetRandomOutputs::Response ra_response; + api::cnd::GetRandomOutputs::Response ra_response; json_resp.get_result(ra_response); - selector.add_mixed_inputs(m_wallet_state.get_wallet().get_view_secret_key(), - request.any_spend_address ? &m_wallet_state.get_wallet() : nullptr, only_records, &builder, - request.transaction.anonymity, std::move(ra_response)); - tx = builder.sign(m_wallet_state.get_wallet().get_tx_derivation_seed()); + const auto actual_anonymity = selector.add_mixed_inputs(&builder, good_anonymity, std::move(ra_response)); + if (actual_anonymity < request.transaction.anonymity) { + m_log(logging::TRACE) << "Transaction anonymity is " << actual_anonymity << "/" + << request.transaction.anonymity << std::endl; + // throw json_rpc::Error(api::walletd::CreateTransaction::NOT_ENOUGH_ANONYMITY, + // "Requested anonymity too high, please reduce anonymity for this + // transaction."); + } + tx = builder.sign( + m_wallet_state, &m_wallet_state.get_wallet(), request.any_spend_address ? nullptr : &only_records); last_response.binary_transaction = seria::to_binary(tx); - const Hash tx_hash = get_transaction_hash(tx); - if (request.save_history && !m_wallet_state.get_wallet().save_history(tx_hash, history)) { + const Hash tx_hash = get_transaction_hash(tx); + if (!is_amethyst && request.save_history && !m_wallet_state.get_wallet().save_history(tx_hash, history)) { m_log(logging::ERROR) << "Saving transaction history failed, you will need to pass list of destination addresses to generate sending proof for tx=" << tx_hash << std::endl; last_response.save_history_error = true; } - if (!m_wallet_state.parse_raw_transaction(last_response.transaction, tx, tx_hash)) - throw json_rpc::Error(json_rpc::INTERNAL_ERROR, "Created trsnsaction cannot be parsed"); - http::ResponseData last_http_response(wc.original_request.r); + http::ResponseBody last_http_response(wc.original_request.r); last_http_response.r.headers.push_back({"Content-Type", "application/json; charset=utf-8"}); last_http_response.r.status = 200; - last_http_response.set_body(json_rpc::create_response_body(last_response, wc.original_jsonrpc_id)); + if (!m_wallet_state.parse_raw_transaction(false, last_response.transaction, std::move(tx), tx_hash)) { + last_http_response.set_body(json_rpc::create_error_response_body( + json_rpc::Error(json_rpc::INTERNAL_ERROR, "Created transaction cannot be parsed"), + wc.original_jsonrpc_id)); + } else + last_http_response.set_body(json_rpc::create_response_body(last_response, wc.original_jsonrpc_id)); wc.original_who->write(std::move(last_http_response)); - }, + }, [=](const WaitingClient &wc, std::string err) mutable { - m_log(logging::INFO) << "got error to get_random_outputs from bytecoind, " << err << std::endl; - http::ResponseData last_http_response(wc.original_request.r); + m_log(logging::INFO) << "got error to get_random_outputs from " CRYPTONOTE_NAME "d, " << err << std::endl; + http::ResponseBody last_http_response(wc.original_request.r); last_http_response.r.headers.push_back({"Content-Type", "application/json; charset=utf-8"}); last_http_response.r.status = 200; last_http_response.set_body(json_rpc::create_error_response_body( json_rpc::Error(json_rpc::INTERNAL_ERROR, err), wc.original_jsonrpc_id)); wc.original_who->write(std::move(last_http_response)); - }); + }); return false; } -bool WalletNode::handle_create_sendproof(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::walletd::CreateSendProof::Request &&request, api::walletd::CreateSendProof::Response &response) { - std::set addresses; +bool WalletNode::on_create_sendproof(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::walletd::CreateSendproof::Request &&request, api::walletd::CreateSendproof::Response &response) { + std::set addresses; + TransactionPrefix tx; + api::Transaction ptx; + if (!m_wallet_state.api_get_transaction(request.transaction_hash, true, &tx, &ptx)) + throw json_rpc::Error(json_rpc::INTERNAL_ERROR, "Created trsnsaction cannot be parsed"); if (request.addresses.empty()) { - Wallet::History history = m_wallet_state.get_wallet().load_history(request.transaction_hash); - for (auto &&address : history) { - if (m_wallet_state.get_wallet().get_view_public_key() == address.view_public_key) - continue; // our address - addresses.insert(address); - } + for (const auto &tr : ptx.transfers) + if (tr.amount > 0 && !tr.address.empty()) { + AccountAddress address; + invariant(m_wallet_state.get_currency().parse_account_address_string(tr.address, &address), ""); + if (!m_wallet_state.get_wallet().is_our_address(address)) + addresses.insert(address); + } } for (auto &&addr : request.addresses) { - AccountPublicAddress address; + AccountAddress address; if (!m_wallet_state.get_currency().parse_account_address_string(addr, &address)) throw api::ErrorAddress(api::ErrorAddress::ADDRESS_FAILED_TO_PARSE, "Failed to parse address", addr); addresses.insert(address); } for (auto &&address : addresses) { - SendProof sp; + Sendproof sp; sp.transaction_hash = request.transaction_hash; sp.message = request.message; sp.address = address; - if (m_wallet_state.api_create_proof(sp)) { - // seria::JsonOutputStreamValue s; - // s.begin_object(); - // ser_members(sp, s, m_wallet_state.get_currency()); - // s.end_object(); - // response.sendproofs.push_back(s.move_value().to_string()); - // return s.move_value(); + if (m_wallet_state.api_create_proof(tx, sp)) { response.sendproofs.push_back(seria::to_json_value(sp, m_wallet_state.get_currency()).to_string()); - } + } else + response.sendproofs.push_back(std::string()); // No proof can be created for address } return true; } -bool WalletNode::handle_send_transaction(http::Client *who, http::RequestData &&raw_request, - json_rpc::Request &&raw_js_request, api::bytecoind::SendTransaction::Request &&request, - api::bytecoind::SendTransaction::Response &response) { +bool WalletNode::on_send_transaction(http::Client *who, http::RequestBody &&raw_request, + json_rpc::Request &&raw_js_request, api::cnd::SendTransaction::Request &&request, + api::cnd::SendTransaction::Response &response) { m_wallet_state.add_to_payment_queue(request.binary_transaction, true); advance_long_poll(); if (m_inproc_node) { - m_inproc_node->handle_send_transaction( + m_inproc_node->on_send_transaction( nullptr, std::move(raw_request), std::move(raw_js_request), std::move(request), response); return true; } - http::RequestData new_request; + http::RequestBody new_request; new_request.set_body(std::move(raw_request.body)); // We save on copying body here - new_request.r.set_firstline("POST", api::bytecoind::url(), 1, 1); + new_request.r.set_firstline("POST", api::cnd::url(), 1, 1); new_request.r.basic_authorization = m_config.bytecoind_authorization; add_waiting_command(who, std::move(raw_request), raw_js_request.get_id().get(), std::move(new_request), - [=](const WaitingClient &wc2, http::ResponseData &&send_response) mutable { - http::ResponseData resp(std::move(send_response)); + [](const WaitingClient &wc2, http::ResponseBody &&send_response) mutable { + http::ResponseBody resp(std::move(send_response)); resp.r.http_version_major = wc2.original_request.r.http_version_major; resp.r.http_version_minor = wc2.original_request.r.http_version_minor; resp.r.keep_alive = wc2.original_request.r.keep_alive; wc2.original_who->write(std::move(resp)); - }, - [=](const WaitingClient &wc2, std::string err) { - // http::ResponseData resp = json_rpc::create_error_response( - // wc2.original_request, json_rpc::Error(json_rpc::INTERNAL_ERROR, err), - // wc2.original_jsonrpc_id); - // wc2.original_who->write(std::move(resp)); - http::ResponseData resp(wc2.original_request.r); + }, + [](const WaitingClient &wc2, std::string err) { + http::ResponseBody resp(wc2.original_request.r); resp.r.headers.push_back({"Content-Type", "application/json; charset=utf-8"}); resp.r.status = 200; resp.set_body(json_rpc::create_error_response_body( json_rpc::Error(json_rpc::INTERNAL_ERROR, err), wc2.original_jsonrpc_id)); wc2.original_who->write(std::move(resp)); - }); + }); return false; } -bool WalletNode::handle_get_transaction(http::Client *, http::RequestData &&, json_rpc::Request &&, +bool WalletNode::on_get_transaction(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetTransaction::Request &&req, api::walletd::GetTransaction::Response &res) { TransactionPrefix tx; m_wallet_state.api_get_transaction(req.hash, true, &tx, &res.transaction); return true; } -void WalletNode::process_waiting_command_response(http::ResponseData &&resp) { +void WalletNode::process_waiting_command_response(http::ResponseBody &&resp) { WaitingClient cli = std::move(m_waiting_command_requests.front()); m_waiting_command_requests.pop_front(); m_command_request.reset(); @@ -660,16 +742,16 @@ void WalletNode::send_next_waiting_command() { m_commands_agent, std::move(m_waiting_command_requests.front().request), fun, e_fun); } -void WalletNode::add_waiting_command(http::Client *who, http::RequestData &&original_request, - const common::JsonValue &original_rpc_id, http::RequestData &&request, - std::function fun, - std::function err_fun) { +void WalletNode::add_waiting_command(http::Client *who, http::RequestBody &&original_request, + const common::JsonValue &original_rpc_id, http::RequestBody &&request, + std::function &&fun, + std::function &&err_fun) { WaitingClient wc2; wc2.original_who = who; wc2.original_request = std::move(original_request); wc2.original_jsonrpc_id = original_rpc_id; - wc2.fun = fun; - wc2.err_fun = err_fun; + wc2.fun = std::move(fun); + wc2.err_fun = std::move(err_fun); wc2.request = std::move(request); m_waiting_command_requests.push_back(wc2); send_next_waiting_command(); @@ -682,15 +764,13 @@ void WalletNode::advance_long_poll() { for (auto lit = m_long_poll_http_clients.begin(); lit != m_long_poll_http_clients.end();) if (resp.ready_for_longpoll(lit->original_get_status)) { - http::ResponseData last_http_response; + http::ResponseBody last_http_response; last_http_response.r.headers.push_back({"Content-Type", "application/json; charset=utf-8"}); last_http_response.r.status = 200; last_http_response.r.http_version_major = lit->original_request.r.http_version_major; last_http_response.r.http_version_minor = lit->original_request.r.http_version_minor; last_http_response.r.keep_alive = lit->original_request.r.keep_alive; last_http_response.set_body(json_rpc::create_response_body(resp, lit->original_jsonrpc_id)); - // m_log(logging::INFO) << "advance_long_poll will - // reply to long poll json=" << last_http_response.body << std::endl; lit->original_who->write(std::move(last_http_response)); lit = m_long_poll_http_clients.erase(lit); } else diff --git a/src/Core/WalletNode.hpp b/src/Core/WalletNode.hpp index 77a89017..85ea555b 100644 --- a/src/Core/WalletNode.hpp +++ b/src/Core/WalletNode.hpp @@ -5,41 +5,49 @@ #include "Node.hpp" #include "WalletSync.hpp" -#include "http/Server.hpp" -namespace bytecoin { +namespace http { +class Server; +class Client; +} // namespace http +namespace cn { class WalletNode : public WalletSync { public: explicit WalletNode(Node *inproc_node, logging::ILogger &, const Config &, WalletState &); + ~WalletNode(); - typedef std::function + typedef std::function JSONRPCHandlerFunction; - bool handle_get_status(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::walletd::GetStatus::Request &&, api::walletd::GetStatus::Response &); - bool handle_get_addresses(http::Client *, http::RequestData &&, json_rpc::Request &&, + bool on_get_status(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetStatus::Request &&, + api::walletd::GetStatus::Response &); + bool on_get_addresses(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetAddresses::Request &&, api::walletd::GetAddresses::Response &); - bool handle_get_wallet_info(http::Client *, http::RequestData &&, json_rpc::Request &&, + bool on_get_wallet_info(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetWalletInfo::Request &&, api::walletd::GetWalletInfo::Response &); - bool handle_create_address_list(http::Client *, http::RequestData &&, json_rpc::Request &&, + bool on_get_wallet_records(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::walletd::GetWalletRecords::Request &&, api::walletd::GetWalletRecords::Response &); + bool on_set_label(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::walletd::SetAddressLabel::Request &&, api::walletd::SetAddressLabel::Response &); + bool on_create_addresses(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::CreateAddresses::Request &&, api::walletd::CreateAddresses::Response &); - bool handle_get_view_key(http::Client *, http::RequestData &&, json_rpc::Request &&, + bool on_get_view_key(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetViewKeyPair::Request &&, api::walletd::GetViewKeyPair::Response &); - bool handle_get_unspent(http::Client *, http::RequestData &&, json_rpc::Request &&, + bool on_get_unspent(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetUnspents::Request &&, api::walletd::GetUnspents::Response &); - bool handle_get_balance(http::Client *, http::RequestData &&, json_rpc::Request &&, + bool on_get_balance(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetBalance::Request &&, api::walletd::GetBalance::Response &); - bool handle_get_transfers(http::Client *, http::RequestData &&, json_rpc::Request &&, + bool on_get_transfers(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetTransfers::Request &&, api::walletd::GetTransfers::Response &); - bool handle_create_transaction(http::Client *, http::RequestData &&, json_rpc::Request &&, + bool on_create_transaction(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::CreateTransaction::Request &&, api::walletd::CreateTransaction::Response &); - bool handle_create_sendproof(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::walletd::CreateSendProof::Request &&, api::walletd::CreateSendProof::Response &); - bool handle_send_transaction(http::Client *, http::RequestData &&, json_rpc::Request &&, - api::bytecoind::SendTransaction::Request &&, - api::bytecoind::SendTransaction::Response &); // We lock spent outputs until next pool sync - bool handle_get_transaction(http::Client *, http::RequestData &&, json_rpc::Request &&, + bool on_create_sendproof(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::walletd::CreateSendproof::Request &&, api::walletd::CreateSendproof::Response &); + bool on_send_transaction(http::Client *, http::RequestBody &&, json_rpc::Request &&, + api::cnd::SendTransaction::Request &&, + api::cnd::SendTransaction::Response &); // We lock spent outputs until next pool sync + bool on_get_transaction(http::Client *, http::RequestBody &&, json_rpc::Request &&, api::walletd::GetTransaction::Request &&, api::walletd::GetTransaction::Response &); typedef std::unordered_map HandlersMap; @@ -52,24 +60,24 @@ class WalletNode : public WalletSync { struct WaitingClient { http::Client *original_who = nullptr; - http::RequestData request; - http::RequestData original_request; + http::RequestBody request; + http::RequestBody original_request; common::JsonValue original_jsonrpc_id; - std::function fun; + std::function fun; std::function err_fun; }; std::deque m_waiting_command_requests; - void add_waiting_command(http::Client *who, http::RequestData &&original_request, - const common::JsonValue &original_rpc_id, http::RequestData &&request, - std::function fun, - std::function err_fun); + void add_waiting_command(http::Client *who, http::RequestBody &&original_request, + const common::JsonValue &original_rpc_id, http::RequestBody &&request, + std::function &&fun, + std::function &&err_fun); void send_next_waiting_command(); - void process_waiting_command_response(http::ResponseData &&resp); + void process_waiting_command_response(http::ResponseBody &&resp); void process_waiting_command_error(std::string err); struct LongPollClient { http::Client *original_who = nullptr; - http::RequestData original_request; + http::RequestBody original_request; common::JsonValue original_jsonrpc_id; api::walletd::GetStatus::Request original_get_status; }; @@ -78,11 +86,11 @@ class WalletNode : public WalletSync { api::walletd::GetStatus::Response create_status_response() const; - bool on_api_http_request(http::Client *, http::RequestData &&, http::ResponseData &); + bool on_api_http_request(http::Client *, http::RequestBody &&, http::ResponseBody &); void on_api_http_disconnect(http::Client *); - bool on_json_rpc(http::Client *, http::RequestData &&, http::ResponseData &, bool &method_found); + bool on_json_rpc(http::Client *, http::RequestBody &&, http::ResponseBody &, bool &method_found); void check_address_in_wallet_or_throw(const std::string &addr) const; }; -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/WalletSerializationV1.cpp b/src/Core/WalletSerializationV1.cpp index 92f644fd..000c46a4 100644 --- a/src/Core/WalletSerializationV1.cpp +++ b/src/Core/WalletSerializationV1.cpp @@ -49,7 +49,7 @@ struct UnlockTransactionJobDto { /*struct WalletTransactionDto { WalletTransactionDto() {} - WalletTransactionDto(const bytecoin::WalletTransaction &wallet) { + WalletTransactionDto(const cn::WalletTransaction &wallet) { state = wallet.state; timestamp = wallet.timestamp; block_height = wallet.block_height; @@ -61,7 +61,7 @@ struct UnlockTransactionJobDto { extra = wallet.extra; } - bytecoin::WalletTransactionState state; + cn::WalletTransactionState state; uint64_t timestamp; uint32_t block_height; Hash hash; @@ -76,7 +76,7 @@ struct UnlockTransactionJobDto { struct WalletTransferDto { explicit WalletTransferDto(uint32_t version) : amount(0), type(0), version(version) {} - WalletTransferDto(const bytecoin::WalletTransfer &tr, uint32_t version) + WalletTransferDto(const cn::WalletTransfer &tr, uint32_t version) : WalletTransferDto(version) { address = tr.address; amount = tr.amount; @@ -88,9 +88,9 @@ version(version) {} uint8_t type; uint32_t version; -};*/ +}; -/*void serialize(WalletRecordDto &v, bytecoin::ISerializer &s) { +void serialize(WalletRecordDto &v, cn::ISerializer &s) { s(v.spend_public_key, "spend_public_key"); s(v.spend_secret_key, "spend_secret_key"); s(v.pending_balance, "pending_balance"); @@ -110,14 +110,14 @@ struct KeysStorage { std::string read_cipher(common::IInputStream &source, const std::string &name) { std::string cipher; - // bytecoin::BinaryInputStreamSerializer s(source); + // cn::BinaryInputStreamSerializer s(source); seria::BinaryInputStream s(source); ser(cipher, s); // , name return cipher; } -std::string decrypt(const std::string &cipher, bytecoin::WalletSerializerV1::CryptoContext &crypto_ctx) { +std::string decrypt(const std::string &cipher, cn::WalletSerializerV1::CryptoContext &crypto_ctx) { std::string plain; plain.resize(cipher.size()); @@ -133,8 +133,8 @@ void deserialize(Object &obj, const std::string &name, const std::string &plain) } template -void deserialize_encrypted(Object &obj, const std::string &name, - bytecoin::WalletSerializerV1::CryptoContext &crypto_ctx, common::IInputStream &source) { +void deserialize_encrypted(Object &obj, const std::string &name, cn::WalletSerializerV1::CryptoContext &crypto_ctx, + common::IInputStream &source) { std::string cipher = read_cipher(source, name); std::string plain = decrypt(cipher, crypto_ctx); @@ -144,7 +144,7 @@ void deserialize_encrypted(Object &obj, const std::string &name, } // anonymous namespace namespace seria { -void ser(crypto::chacha8_iv &v, ISeria &s) { s.binary(v.data, sizeof(v.data)); } +void ser(crypto::chacha_iv &v, ISeria &s) { s.binary(v.data, sizeof(v.data)); } void ser_members(WalletRecordDto &v, ISeria &s) { seria_kv("spend_public_key", v.spend_public_key, s); seria_kv("spend_secret_key", v.spend_secret_key, s); @@ -159,9 +159,9 @@ void ser_members(KeysStorage &v, ISeria &s) { seria_kv("view_public_key", v.view_public_key, s); seria_kv("view_secret_key", v.view_secret_key, s); } -} +} // namespace seria -namespace bytecoin { +namespace cn { const uint32_t WalletSerializerV1::SERIALIZATION_VERSION = 5; @@ -174,7 +174,7 @@ WalletSerializerV1::WalletSerializerV1(crypto::PublicKey &view_public_key, crypt std::vector &wallets_container) : m_view_public_key(view_public_key), m_view_secret_key(view_secret_key), m_wallets_container(wallets_container) {} -void WalletSerializerV1::load(const crypto::chacha8_key &key, common::IInputStream &source) { +void WalletSerializerV1::load(const crypto::chacha_key &key, common::IInputStream &source) { seria::BinaryInputStream s(source); s.begin_object(); @@ -191,7 +191,7 @@ void WalletSerializerV1::load(const crypto::chacha8_key &key, common::IInputStre s.end_object(); } -void WalletSerializerV1::load_wallet(common::IInputStream &source, const crypto::chacha8_key &key, uint32_t version) { +void WalletSerializerV1::load_wallet(common::IInputStream &source, const crypto::chacha_key &key, uint32_t version) { CryptoContext crypto_ctx; load_iv(source, crypto_ctx.iv); @@ -203,7 +203,7 @@ void WalletSerializerV1::load_wallet(common::IInputStream &source, const crypto: load_wallets(source, crypto_ctx); } -void WalletSerializerV1::load_wallet_v1(common::IInputStream &source, const crypto::chacha8_key &key) { +void WalletSerializerV1::load_wallet_v1(common::IInputStream &source, const crypto::chacha_key &key) { CryptoContext crypto_ctx; seria::BinaryInputStream encrypted(source); @@ -255,7 +255,7 @@ uint32_t WalletSerializerV1::load_version(common::IInputStream &source) { return version; } -void WalletSerializerV1::load_iv(common::IInputStream &source, crypto::chacha8_iv &iv) { +void WalletSerializerV1::load_iv(common::IInputStream &source, crypto::chacha_iv &iv) { seria::BinaryInputStream s(source); s.binary(static_cast(&iv.data), sizeof(iv.data)); // , "chacha_iv" @@ -333,4 +333,4 @@ void WalletSerializerV1::load_wallets(common::IInputStream &source, CryptoContex } } -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/WalletSerializationV1.hpp b/src/Core/WalletSerializationV1.hpp index 169b59b0..3ecdc7b6 100644 --- a/src/Core/WalletSerializationV1.hpp +++ b/src/Core/WalletSerializationV1.hpp @@ -5,21 +5,21 @@ #include "Core/Wallet.hpp" // for WalletRecord #include "common/Streams.hpp" -#include "crypto/chacha8.hpp" +#include "crypto/chacha.hpp" #include "seria/ISeria.hpp" -namespace bytecoin { +namespace cn { class WalletSerializerV1 { public: WalletSerializerV1( PublicKey &view_public_key, SecretKey &view_secret_key, std::vector &wallets_container); - void load(const crypto::chacha8_key &key, common::IInputStream &source); + void load(const crypto::chacha_key &key, common::IInputStream &source); struct CryptoContext { - crypto::chacha8_key key; - crypto::chacha8_iv iv; + crypto::chacha_key key; + crypto::chacha_iv iv; void inc_iv(); }; @@ -27,11 +27,11 @@ class WalletSerializerV1 { private: static const uint32_t SERIALIZATION_VERSION; - void load_wallet(common::IInputStream &source, const crypto::chacha8_key &key, uint32_t version); - void load_wallet_v1(common::IInputStream &source, const crypto::chacha8_key &key); + void load_wallet(common::IInputStream &source, const crypto::chacha_key &key, uint32_t version); + void load_wallet_v1(common::IInputStream &source, const crypto::chacha_key &key); uint32_t load_version(common::IInputStream &source); - void load_iv(common::IInputStream &source, crypto::chacha8_iv &iv); + void load_iv(common::IInputStream &source, crypto::chacha_iv &iv); void load_keys(common::IInputStream &source, CryptoContext &); void load_public_key(common::IInputStream &source, CryptoContext &); void load_secret_key(common::IInputStream &source, CryptoContext &); @@ -47,4 +47,4 @@ class WalletSerializerV1 { std::vector &m_wallets_container; }; -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/WalletState.cpp b/src/Core/WalletState.cpp index da9e58bb..b9387fcc 100644 --- a/src/Core/WalletState.cpp +++ b/src/Core/WalletState.cpp @@ -5,16 +5,19 @@ #include "Config.hpp" #include "CryptoNoteTools.hpp" #include "TransactionBuilder.hpp" +#include "TransactionExtra.hpp" +#include "common/Varint.hpp" #include "common/string.hpp" #include "crypto/crypto.hpp" #include "platform/PathTools.hpp" +#include "platform/Time.hpp" #include "seria/BinaryInputStream.hpp" #include "seria/KVBinaryInputStream.hpp" #include "seria/KVBinaryOutputStream.hpp" static const std::string ADDRESSES_PREFIX = "a"; // this is not undone -using namespace bytecoin; +using namespace cn; using namespace platform; Amount WalletState::DeltaState::add_incoming_output(const api::Output &output, const Hash &tid) { @@ -23,35 +26,44 @@ Amount WalletState::DeltaState::add_incoming_output(const api::Output &output, c } Amount WalletState::DeltaState::add_incoming_keyimage(Height height, const KeyImage &key_image) { - // m_used_keyimages[key_image] += 1; // We add all keyimages, not only recognized keyimages when using mem pool + auto tit = m_transactions.find(m_last_added_transaction); + if (tit == m_transactions.end()) + return 0; + if (tit->second.used_ki_or_pk.insert(key_image).second) + m_used_kis_or_pks[key_image] += 1; + return 0; // It does not know +} + +Amount WalletState::DeltaState::add_incoming_deterministic_input( + Height block_height, Amount am, size_t gi, const PublicKey &pk) { + auto tit = m_transactions.find(m_last_added_transaction); + if (tit == m_transactions.end()) + return 0; + if (tit->second.used_ki_or_pk.insert(pk).second) + m_used_kis_or_pks[pk] += 1; return 0; // It does not know } void WalletState::DeltaState::add_transaction( Height, const Hash &tid, const TransactionPrefix &tx, const api::Transaction &ptx) { - invariant(m_transactions.insert(std::make_pair(tid, std::make_pair(tx, ptx))).second, + invariant(m_transactions.insert(std::make_pair(tid, DeltaStateTransaction{tx, ptx, {}})).second, "transaction already exists. Invariant dead"); - for (const auto &input : tx.inputs) { - if (input.type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(input); - m_used_keyimages[in.key_image] += 1; - } - } + m_last_added_transaction = tid; } void WalletState::DeltaState::undo_transaction(const Hash &tid) { auto tit = m_transactions.find(tid); if (tit == m_transactions.end()) return; - const TransactionPrefix &tx = tit->second.first; + const TransactionPrefix &tx = tit->second.tx; for (const auto &output : tx.outputs) { - if (output.target.type() == typeid(KeyOutput)) { - const KeyOutput &key_output = boost::get(output.target); - auto uit = m_unspents.find(key_output.public_key); + if (output.type() == typeid(OutputKey)) { + const auto &key_output = boost::get(output); + auto uit = m_unspents.find(key_output.public_key); if (uit == m_unspents.end()) // Actually should never be empty continue; // Not our output for (auto oit = uit->second.begin(); oit != uit->second.end(); ++oit) - if (oit->amount == output.amount) { // We need to pop right output, or balance will be trashed + if (oit->amount == key_output.amount) { // We need to pop right output, or balance will be trashed oit = uit->second.erase(oit); break; } @@ -59,56 +71,41 @@ void WalletState::DeltaState::undo_transaction(const Hash &tid) { uit = m_unspents.erase(uit); } } - for (const auto &input : tx.inputs) { - if (input.type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(input); - auto kit = m_used_keyimages.find(in.key_image); - invariant(kit != m_used_keyimages.end(), ""); - kit->second -= 1; - invariant(kit->second >= 0, ""); - if (kit->second == 0) - kit = m_used_keyimages.erase(kit); - } + for (const auto &ki_or_pk : tit->second.used_ki_or_pk) { + auto kit = m_used_kis_or_pks.find(ki_or_pk); + invariant(kit != m_used_kis_or_pks.end(), ""); + kit->second -= 1; + invariant(kit->second >= 0, ""); + if (kit->second == 0) + kit = m_used_kis_or_pks.erase(kit); } tit = m_transactions.erase(tit); } void WalletState::DeltaState::clear() { - m_used_keyimages.clear(); + m_used_kis_or_pks.clear(); m_unspents.clear(); m_transactions.clear(); } -// bool WalletState::DeltaState::is_spent(const api::Output &output) const { -// return m_used_keyimages.count(output.key_image) != 0; -//} - WalletState::WalletState(Wallet &wallet, logging::ILogger &log, const Config &config, const Currency ¤cy) - : WalletStateBasic(log, config, currency, wallet.get_cache_name()) - , log_redo_block(std::chrono::steady_clock::now()) + : WalletStateBasic( + log, config, currency, wallet.get_cache_name(), wallet.is_view_only() && wallet.is_deterministic()) + , m_log_redo_block(std::chrono::steady_clock::now()) , m_wallet(wallet) { wallet_addresses_updated(); - platform::remove_file(m_wallet.get_payment_queue_folder() + "/tmp.tx"); - for (const auto &file : platform::get_filenames_in_folder(wallet.get_payment_queue_folder())) { - common::BinaryArray body; - if (!platform::load_file(wallet.get_payment_queue_folder() + "/" + file, body)) - continue; + auto pq = m_wallet.payment_queue_get(); + for (const auto &body : pq) { Transaction tx; try { add_to_payment_queue(body, false); } catch (const std::exception &) { - m_log(logging::WARNING) << "Error adding transaction to payment queue from file " << file << std::endl; + m_log(logging::WARNING) << "Error adding transaction to payment queue " << std::endl; continue; } } } -Height WalletState::get_pq_confirmations() const { - if (m_config.net == "test") - return 20; // std::max(20, 720 / platform::get_time_multiplier_for_tests()); - return m_currency.expected_blocks_per_day(); -} - void WalletState::wallet_addresses_updated() { Timestamp undo_timestamp = std::numeric_limits::max(); try { @@ -126,9 +123,8 @@ void WalletState::wallet_addresses_updated() { if (undo_timestamp == std::numeric_limits::max()) { return; // db.commit() not worth here, will just update addresses again in case of ctrl-c } - while (!empty_chain() && - get_tip().timestamp + m_currency.block_future_time_limit >= - undo_timestamp) { // Undo excess blocks in case timestamps are out of order + while (!empty_chain() && get_tip().timestamp + m_currency.block_future_time_limit >= + undo_timestamp) { // Undo excess blocks in case timestamps are out of order pop_chain(); } fix_empty_chain(); @@ -151,6 +147,13 @@ std::vector WalletState::generate_new_addresses( return result; } +void WalletState::create_addresses(size_t count) { + if (count <= m_wallet.get_actual_records_count()) + return; + m_wallet.create_look_ahead_records(count); + wallet_addresses_updated(); +} + bool WalletState::add_to_payment_queue(const BinaryArray &binary_transaction, bool save_file) { Transaction tx; seria::from_binary(tx, binary_transaction); @@ -159,26 +162,19 @@ bool WalletState::add_to_payment_queue(const BinaryArray &binary_transaction, bo auto git = by_hash_index.find(tid); if (git != by_hash_index.end()) return true; // alredy here, nop - const std::string file = m_wallet.get_payment_queue_folder() + "/" + common::pod_to_hex(tid) + ".tx"; - if (save_file) { - platform::create_folder_if_necessary(m_wallet.get_payment_queue_folder()); - if (!platform::atomic_save_file(file, binary_transaction.data(), binary_transaction.size(), - m_wallet.get_payment_queue_folder() + "/tmp.tx")) - m_log(logging::WARNING) << "Failed to save transaction " << tid << " to file " << file << std::endl; - else - m_log(logging::INFO) << "Saved transaction " << tid << " to file " << file << std::endl; - } + if (save_file) + m_wallet.payment_queue_add(tid, binary_transaction); TransactionPrefix tx_prefix; api::Transaction ptx; QueueEntry entry{tid, binary_transaction, 0, 0}; // std::cout << "by_hash_index.size=" << by_hash_index.size() << std::endl; m_pq_version += 1; if (api_get_transaction(tid, false, &tx_prefix, &ptx)) { - entry.remove_height = ptx.block_height + get_pq_confirmations(); - m_log(logging::INFO) << "Now PQ transaction " << tid << " is in BC, remove_height=" << entry.remove_height - << std::endl; - entry.fee_per_kb = ptx.fee / binary_transaction.size(); + entry.remove_height = ptx.block_height + m_config.payment_queue_confirmations; + entry.fee_per_kb = ptx.fee / binary_transaction.size(); payment_queue.insert(entry); + m_log(logging::INFO) << "Now PQ transaction " << tid << " is in BC, remove_height=" << entry.remove_height + << " payment_queue.size=" << payment_queue.size() << std::endl; return true; } entry.fee_per_kb = get_tx_fee(tx) / binary_transaction.size(); @@ -205,24 +201,19 @@ BinaryArray WalletState::get_next_from_sending_queue(Hash *previous_hash) { return git->binary_transaction; } -void WalletState::process_payment_queue_send_error(Hash hash, const api::bytecoind::SendTransaction::Error &error) { - if (error.code == api::bytecoind::SendTransaction::OUTPUT_ALREADY_SPENT || - error.code == api::bytecoind::SendTransaction::WRONG_OUTPUT_REFERENCE) { - if (get_tip_height() > error.conflict_height + get_pq_confirmations()) { - auto &by_hash_index = payment_queue.get(); - auto git = by_hash_index.find(hash); - if (git != by_hash_index.end()) - by_hash_index.erase(git); - remove_transaction_from_mempool(hash, true); - const std::string file = m_wallet.get_payment_queue_folder() + "/" + common::pod_to_hex(hash) + ".tx"; - if (!platform::remove_file(file)) - m_log(logging::WARNING) << "Failed to remove PQ transaction " << hash << " from file " << file - << std::endl; - else - m_log(logging::INFO) << "Removed PQ transaction " << hash << " from file " << file << std::endl; - platform::remove_file(m_wallet.get_payment_queue_folder()); // When it becomes empty - m_pq_version += 1; - } +void WalletState::process_payment_queue_send_error(Hash hash, const api::cnd::SendTransaction::Error &error) { + if (error.code == api::cnd::SendTransaction::INVALID_TRANSACTION_BINARY_FORMAT || + ((error.code == api::cnd::SendTransaction::OUTPUT_ALREADY_SPENT || + error.code == api::cnd::SendTransaction::WRONG_OUTPUT_REFERENCE) && + get_tip_height() > error.conflict_height + m_config.payment_queue_confirmations)) { + m_log(logging::INFO) << "Removing transaction from PQ because send error " << error.what() << std::endl; + auto &by_hash_index = payment_queue.get(); + auto git = by_hash_index.find(hash); + if (git != by_hash_index.end()) + by_hash_index.erase(git); + remove_transaction_from_mempool(hash, true); + m_wallet.payment_queue_remove(hash); + m_pq_version += 1; } } @@ -269,7 +260,7 @@ void WalletState::fix_payment_queue_after_undo_redo() { continue; QueueEntry entry = *git; by_hash_index.erase(git); - entry.remove_height = ptx.block_height + get_pq_confirmations(); + entry.remove_height = ptx.block_height + m_config.payment_queue_confirmations; payment_queue.insert(entry); pq_modified = true; remove_transaction_from_mempool(tid, true); @@ -278,13 +269,7 @@ void WalletState::fix_payment_queue_after_undo_redo() { // std::cout << "by_remove_height_index.size=" << by_remove_height_index.size() << std::endl; auto git = by_remove_height_index.lower_bound(1); while (git != by_remove_height_index.end() && get_tip_height() >= git->remove_height) { - const std::string file = m_wallet.get_payment_queue_folder() + "/" + common::pod_to_hex(git->hash) + ".tx"; - if (!platform::remove_file(file)) - m_log(logging::WARNING) << "Failed to remove PQ transaction " << git->hash << " from file " << file - << std::endl; - else - m_log(logging::INFO) << "Removed PQ transaction " << git->hash << " from file " << file << std::endl; - platform::remove_file(m_wallet.get_payment_queue_folder()); // When it becomes empty + m_wallet.payment_queue_remove(git->hash); git = by_remove_height_index.erase(git); pq_modified = true; } @@ -292,14 +277,15 @@ void WalletState::fix_payment_queue_after_undo_redo() { m_pq_version += 1; } -void WalletState::add_transaction_to_mempool(Hash tid, TransactionPrefix &&tx, bool from_pq) { +void WalletState::add_transaction_to_mempool(Hash tid, Transaction &&tx, bool from_pq) { if (m_memory_state.get_transactions().count(tid) != 0) return; - m_log(logging::INFO) << "Now " << (from_pq ? "PQ" : "node") << " transaction " << tid << " is in MS " << std::endl; - std::vector global_indices(tx.outputs.size(), 0); - PreparedWalletTransaction pwtx(TransactionPrefix(tx), m_wallet.get_view_secret_key()); - if (!redo_transaction( - pwtx, global_indices, &m_memory_state, false, tid, get_tip_height() + 1, Hash{}, get_tip().timestamp)) { + const auto now = platform::now_unix_timestamp(); + m_log(logging::INFO) << "Now " << (from_pq ? "PQ" : "node") << " transaction " << tid + << " is in MS size before adding=" << m_memory_state.get_transactions().size() << std::endl; + std::vector global_indices(tx.outputs.size(), 0); + PreparedWalletTransaction pwtx(std::move(tx), m_wallet.get_output_handler()); + if (!redo_transaction(pwtx, global_indices, &m_memory_state, false, tid, get_tip_height() + 1, Hash{}, now)) { } // just ignore result } @@ -314,19 +300,19 @@ void WalletState::remove_transaction_from_mempool(Hash tid, bool from_pq) { m_memory_state.undo_transaction(tid); } -bool WalletState::sync_with_blockchain(api::bytecoind::SyncBlocks::Response &resp) { +bool WalletState::sync_with_blockchain(api::cnd::SyncBlocks::Response &resp) { if (resp.blocks.empty()) // Our creation timestamp > last block timestamp, so - // no blocks + // no blocks return true; try { - while (get_tip_height() > resp.start_height + resp.blocks.size() - 1 && - !empty_chain()) { // first undo excess blocks at head + while (get_tip_height() > resp.start_height + resp.blocks.size() - 1 && !empty_chain()) { + // first undo excess blocks at head pop_chain(); m_tx_pool_version = 1; } - while (get_tip_height() >= resp.start_height && - !empty_chain()) { // then undo all blocks at head with different bids - const api::BlockHeader &other_header = resp.blocks[get_tip_height() - resp.start_height].header; + while (get_tip_height() >= resp.start_height && !empty_chain()) { + // then undo all blocks at head with different bids + const auto &other_header = resp.blocks[get_tip_height() - resp.start_height].header; if (get_tip_bid() == other_header.hash) break; if (get_tip_height() == 0) @@ -342,11 +328,11 @@ bool WalletState::sync_with_blockchain(api::bytecoind::SyncBlocks::Response &res if (empty_chain()) reset_chain(resp.start_height); preparator.cancel_work(); - preparator.start_work(resp, m_wallet.get_view_secret_key()); + preparator.start_work(resp, m_wallet.get_output_handler()); while (get_tip_height() + 1 < resp.start_height + resp.blocks.size()) { - size_t bin = get_tip_height() + 1 - resp.start_height; - const api::BlockHeader &header = resp.blocks.at(bin).header; - if (!empty_chain() && header.previous_block_hash != get_tip_bid()) // TODO - investigate first condition + size_t bin = get_tip_height() + 1 - resp.start_height; + const auto &header = resp.blocks.at(bin).header; + if (!empty_chain() && header.previous_block_hash != get_tip_bid()) return false; if (header.timestamp + m_currency.block_future_time_limit >= m_wallet.get_oldest_timestamp()) { const auto &block_gi = resp.blocks.at(bin).output_indexes; @@ -357,14 +343,14 @@ bool WalletState::sync_with_blockchain(api::bytecoind::SyncBlocks::Response &res // pop_chain(); // redo_block(header, pb, block_gi, m_tip_height + 1); auto now = std::chrono::steady_clock::now(); - if (std::chrono::duration_cast(now - log_redo_block).count() > 1000 || + if (std::chrono::duration_cast(now - m_log_redo_block).count() > 1000 || get_tip_height() + 1 == resp.status.top_known_block_height) { - log_redo_block = now; + m_log_redo_block = now; m_log(logging::INFO) << "WalletState redo block, height=" << get_tip_height() + 1 << "/" << resp.status.top_known_block_height << std::endl; - } else - m_log(logging::TRACE) << "WalletState redo block, height=" << get_tip_height() + 1 << "/" - << resp.status.top_known_block_height << std::endl; + } // else + // m_log(logging::TRACE) << "WalletState redo block, height=" << get_tip_height() + 1 << "/" + // << resp.status.top_known_block_height << std::endl; } push_chain(header); m_tx_pool_version = 1; @@ -385,13 +371,16 @@ std::vector WalletState::get_tx_pool_hashes() const { return std::vector(m_pool_hashes.begin(), m_pool_hashes.end()); } -bool WalletState::sync_with_blockchain(api::bytecoind::SyncMemPool::Response &resp) { +bool WalletState::sync_with_blockchain(api::cnd::SyncMemPool::Response &resp) { for (const auto &tid : resp.removed_hashes) { if (m_pool_hashes.erase(tid) != 0) remove_transaction_from_mempool(tid, false); } for (size_t i = 0; i != resp.added_raw_transactions.size(); ++i) { - TransactionPrefix &tx = resp.added_raw_transactions[i]; + Transaction tx; + static_cast(tx) = std::move(resp.added_raw_transactions[i]); + if (i < resp.added_signatures.size()) + tx.signatures = std::move(resp.added_signatures[i]); // seria::from_binary(tx, resp.added_binary_transactions[i]); Hash tid = resp.added_transactions.at(i).hash; // get_transaction_hash(tx); m_pool_hashes.insert(tid); @@ -425,126 +414,71 @@ bool WalletState::redo_block(const api::BlockHeader &header, const PreparedWalle } // We return output transfers in ptx, input transfers in input_transfers -bool WalletState::parse_raw_transaction(api::Transaction *ptx, std::vector *input_transfers, +bool WalletState::parse_raw_transaction(bool is_base, api::Transaction *ptx, + std::vector *input_transfers, std::vector *output_transfers, Amount *unrecognized_inputs_amount, const PreparedWalletTransaction &pwtx, Hash tid, - const std::vector &global_indices, Height block_height) const { + const std::vector &global_indices, Height block_height) const { if (global_indices.size() != pwtx.tx.outputs.size()) // Bad node return false; // Without global indices we cannot do anything with transaction const TransactionPrefix &tx = pwtx.tx; - PublicKey tx_public_key = extra_get_transaction_public_key(tx.extra); - if (pwtx.derivation == KeyDerivation{}) - return false; - Wallet::History history = m_wallet.load_history(tid); + const bool is_tx_amethyst = tx.version >= m_currency.amethyst_transaction_version; + boost::optional history; KeyPair tx_keys; + ptx->coinbase = is_base; ptx->hash = tid; + ptx->inputs_hash = pwtx.inputs_hash; + ptx->prefix_hash = pwtx.prefix_hash; ptx->block_height = block_height; - ptx->anonymity = std::numeric_limits::max(); + ptx->anonymity = std::numeric_limits::max(); ptx->unlock_block_or_timestamp = tx.unlock_block_or_timestamp; - ptx->public_key = tx_public_key; + ptx->public_key = extra_get_transaction_public_key(tx.extra); ptx->extra = tx.extra; extra_get_payment_id(tx.extra, ptx->payment_id); - uint32_t out_index = 0; - Amount output_amount = 0; - size_t key_index = 0; - bool our_transaction = false; - // We combine outputs into transfers by address - std::map transfer_map_outputs[2]; // We index by ours - for (const auto &output : tx.outputs) { - output_amount += output.amount; - if (output.target.type() == typeid(KeyOutput)) { - const KeyOutput &key_output = boost::get(output.target); - PublicKey spend_key = pwtx.spend_keys.at(key_index); - bool our_key = false; - if (spend_key != PublicKey{}) { - AccountPublicAddress address{spend_key, m_wallet.get_view_public_key()}; - WalletRecord record; - if (m_wallet.get_record(record, address)) { - KeyPair in_ephemeral; - if (derive_public_key(pwtx.derivation, out_index, spend_key, in_ephemeral.public_key)) { - derive_secret_key(pwtx.derivation, out_index, record.spend_secret_key, in_ephemeral.secret_key); - // std::cout << "My output! - // out_index=" << out_index << "amount=" << output.amount << std::endl; - api::Output out; - out.amount = output.amount; - out.index = global_indices.at(out_index); - out.dust = m_currency.is_dust(output.amount); - out.height = block_height; - out.index_in_transaction = out_index; - if (record.spend_secret_key != SecretKey{}) - generate_key_image(in_ephemeral.public_key, in_ephemeral.secret_key, out.key_image); - out.public_key = key_output.public_key; - out.transaction_public_key = tx_public_key; - out.unlock_block_or_timestamp = tx.unlock_block_or_timestamp; - api::Transfer &transfer = transfer_map_outputs[true][address]; - if (transfer.address.empty()) - transfer.address = m_currency.account_address_as_string(address); - out.address = transfer.address; - Amount confirmed_balance_delta = 0; - if (try_add_incoming_output(out, &confirmed_balance_delta)) { - transfer.amount += confirmed_balance_delta; - transfer.outputs.push_back(out); - } - our_transaction = true; - our_key = true; - } - } - } - if (!our_key && !history.empty()) { - if (tx_keys.secret_key == SecretKey{}) // do expensive calcs once and only if needed - tx_keys = TransactionBuilder::deterministic_keys_from_seed(tx, m_wallet.get_tx_derivation_seed()); - for (auto &&addr : history) { - PublicKey guess_key{}; - TransactionBuilder::derive_public_key(addr, tx_keys.secret_key, out_index, guess_key); - if (guess_key == key_output.public_key) { - api::Output out; - out.amount = output.amount; - out.index = global_indices.at(out_index); - out.dust = m_currency.is_dust(output.amount); - out.height = block_height; - // We cannot generate key_image for others addresses - out.index_in_transaction = out_index; - out.public_key = key_output.public_key; - out.transaction_public_key = tx_public_key; - out.unlock_block_or_timestamp = tx.unlock_block_or_timestamp; - api::Transfer &transfer = transfer_map_outputs[false][addr]; - if (transfer.address.empty()) - transfer.address = m_currency.account_address_as_string(addr); - out.address = transfer.address; - transfer.amount += output.amount; - transfer.outputs.push_back(out); - } - } - } - ++key_index; - } - ++out_index; - } - for (bool ours : {false, true}) - for (auto &&tm : transfer_map_outputs[ours]) { - tm.second.locked = ptx->unlock_block_or_timestamp != 0; - tm.second.ours = ours; - tm.second.transaction_hash = tid; - if (tm.second.amount != 0) // We use map as a map of addresses - ptx->transfers.push_back(std::move(tm.second)); - } + bool our_inputs = false; + bool our_outputs = false; + std::map transfer_map_inputs; *unrecognized_inputs_amount = 0; Amount input_amount = 0; - for (const auto &input : tx.inputs) { - if (input.type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(input); - input_amount += in.amount; - ptx->anonymity = std::min(ptx->anonymity, static_cast(in.output_indexes.size() - 1)); + for (size_t in_index = 0; in_index != tx.inputs.size(); ++in_index) { + const auto &input = tx.inputs.at(in_index); + if (input.type() == typeid(InputKey)) { + const InputKey &in = boost::get(input); + if (!add_amount(input_amount, in.amount)) + return false; + ptx->anonymity = std::min(ptx->anonymity, in.output_indexes.size() - 1); api::Output existing_output; + if (m_wallet.is_det_viewonly() && pwtx.sigs.type() == typeid(RingSignature3)) { + auto &signatures = boost::get(pwtx.sigs); + auto my_mixin_index = in.output_indexes.size(); + if (in_index < signatures.r.size() && in.output_indexes.size() == signatures.r[in_index].size()) + my_mixin_index = crypto::find_deterministic_input3( + pwtx.prefix_hash, in_index, signatures.r[in_index], m_wallet.get_view_secret_key()); + if (my_mixin_index < in.output_indexes.size()) { + size_t my_index = 0; + for (size_t i = 0; i < my_mixin_index + 1; ++i) { + my_index += in.output_indexes.at(i); + } + if (try_adding_deterministic_input(in.amount, my_index, &existing_output)) { + api::Transfer &transfer = transfer_map_inputs[existing_output.address]; + transfer.amount -= static_cast(existing_output.amount); + transfer.ours = true; + transfer.outputs.push_back(existing_output); + our_inputs = true; + continue; + } + } + } if (try_adding_incoming_keyimage(in.key_image, &existing_output)) { api::Transfer &transfer = transfer_map_inputs[existing_output.address]; transfer.amount -= static_cast(existing_output.amount); transfer.ours = true; transfer.outputs.push_back(existing_output); - our_transaction = true; - } else - *unrecognized_inputs_amount += in.amount; + our_inputs = true; + continue; + } + *unrecognized_inputs_amount += in.amount; } } for (auto &&tm : transfer_map_inputs) { @@ -552,23 +486,87 @@ bool WalletState::parse_raw_transaction(api::Transaction *ptx, std::vectorpush_back(tm.second); } + // We combine outputs into transfers by address + Amount output_amount = 0; + std::map transfer_map_outputs[2]; // We index by ours + for (size_t out_index = 0; out_index != tx.outputs.size(); ++out_index) { + const auto &output = tx.outputs.at(out_index); + if (output.type() != typeid(OutputKey)) + continue; + const auto &key_output = boost::get(output); + const auto &spend_public_key = pwtx.spend_keys.at(out_index); + const auto &spend_secret = pwtx.output_secret_scalars.at(out_index); + if (!add_amount(output_amount, key_output.amount)) + return false; + bool our_key = false; + api::Output out; + out.index = global_indices.at(out_index); + out.height = block_height; + out.index_in_transaction = out_index; + out.public_key = key_output.public_key; + out.transaction_hash = tid; + out.unlock_block_or_timestamp = tx.unlock_block_or_timestamp; + + AccountAddress address; + KeyPair output_keypair; + if (m_wallet.detect_our_output(tid, pwtx.inputs_hash, pwtx.derivation, out_index, spend_public_key, + spend_secret, key_output, &out.amount, &output_keypair, &address)) { + // out.dust = m_currency.is_dust(key_output.amount); + if (output_keypair.secret_key != SecretKey{}) + out.key_image = generate_key_image(output_keypair.public_key, output_keypair.secret_key); + api::Transfer &transfer = transfer_map_outputs[true][address]; + if (transfer.address.empty()) + transfer.address = m_currency.account_address_as_string(address); + out.address = transfer.address; + Amount confirmed_balance_delta = 0; + if (try_add_incoming_output(out, &confirmed_balance_delta)) { + transfer.amount += confirmed_balance_delta; + transfer.outputs.push_back(out); + } + our_outputs = true; + our_key = true; + } + if (!our_key && our_inputs && + m_wallet.detect_not_our_output( + is_tx_amethyst, tid, pwtx.inputs_hash, &history, out_index, key_output, &out.amount, &address)) { + // out.dust = m_currency.is_dust(key_output.amount); + api::Transfer &transfer = transfer_map_outputs[false][address]; + if (transfer.address.empty()) + transfer.address = m_currency.account_address_as_string(address); + out.address = transfer.address; + transfer.amount += key_output.amount; + transfer.outputs.push_back(out); + } + } + for (bool ours : {false, true}) + for (auto &&tm : transfer_map_outputs[ours]) { + tm.second.locked = ptx->unlock_block_or_timestamp != 0; + tm.second.ours = ours; + tm.second.transaction_hash = tid; + if (tm.second.amount != 0) // We use map as a map of addresses + output_transfers->push_back(std::move(tm.second)); + } ptx->amount = output_amount; + if (output_amount > input_amount && !is_base) + return false; if (input_amount >= output_amount) ptx->fee = input_amount - output_amount; - if (ptx->anonymity == std::numeric_limits::max()) + if (ptx->anonymity == std::numeric_limits::max()) ptx->anonymity = 0; // No key inputs - return our_transaction; + return our_outputs || our_inputs; } -bool WalletState::parse_raw_transaction(api::Transaction &ptx, const TransactionPrefix &tx, Hash tid) const { - std::vector global_indices(tx.outputs.size(), 0); +bool WalletState::parse_raw_transaction(bool is_base, api::Transaction &ptx, Transaction &&tx, Hash tid) const { + std::vector global_indices(tx.outputs.size(), 0); Amount unrecognized_inputs_amount = 0; - PreparedWalletTransaction pwtx(TransactionPrefix(tx), m_wallet.get_view_secret_key()); + PreparedWalletTransaction pwtx(std::move(tx), m_wallet.get_output_handler()); std::vector input_transfers; - parse_raw_transaction( - &ptx, &input_transfers, &unrecognized_inputs_amount, pwtx, tid, global_indices, get_tip_height()); + std::vector output_transfers; + parse_raw_transaction(is_base, &ptx, &input_transfers, &output_transfers, &unrecognized_inputs_amount, pwtx, tid, + global_indices, get_tip_height()); // We do not know "from" addresses, so leave address empty ptx.transfers.insert(ptx.transfers.end(), input_transfers.begin(), input_transfers.end()); + ptx.transfers.insert(ptx.transfers.end(), output_transfers.begin(), output_transfers.end()); if (unrecognized_inputs_amount != 0) { api::Transfer input_transfer; input_transfer.amount = -static_cast(unrecognized_inputs_amount); @@ -578,27 +576,27 @@ bool WalletState::parse_raw_transaction(api::Transaction &ptx, const Transaction return true; } -const std::map &WalletState::get_used_key_images() const { return m_memory_state.get_used_key_images(); } - -void WalletState::on_first_transaction_found(Timestamp ts) { - if (m_currency.net == "main") // TODO - per net first timestamp in future wallet format - m_wallet.on_first_output_found(ts); +const std::map &WalletState::get_mempool_kis_or_pks() const { + return m_memory_state.get_used_kis_or_pks(); } -bool WalletState::redo_transaction(const PreparedWalletTransaction &pwtx, const std::vector &global_indices, +void WalletState::on_first_transaction_found(Timestamp ts) { m_wallet.on_first_output_found(ts); } + +bool WalletState::redo_transaction(const PreparedWalletTransaction &pwtx, const std::vector &global_indices, IWalletState *delta_state, bool is_base, Hash tid, Height block_height, Hash bid, Timestamp tx_timestamp) { api::Transaction ptx; Amount unrecognized_inputs_amount = 0; std::vector input_transfers; - if (!parse_raw_transaction( - &ptx, &input_transfers, &unrecognized_inputs_amount, pwtx, tid, global_indices, block_height)) + std::vector output_transfers; + if (!parse_raw_transaction(is_base, &ptx, &input_transfers, &output_transfers, &unrecognized_inputs_amount, pwtx, + tid, global_indices, block_height)) return false; // not ours - if (is_base) - ptx.fee = 0; ptx.block_hash = bid; - ptx.coinbase = is_base; ptx.timestamp = tx_timestamp; - for (auto &&tr : ptx.transfers) { // add and fix outputs + ptx.transfers.insert(ptx.transfers.end(), input_transfers.begin(), input_transfers.end()); + ptx.transfers.insert(ptx.transfers.end(), output_transfers.begin(), output_transfers.end()); + delta_state->add_transaction(block_height, tid, pwtx.tx, ptx); + for (auto &&tr : output_transfers) { // add and fix outputs if (!tr.ours) continue; for (auto &&out : tr.outputs) { @@ -608,11 +606,13 @@ bool WalletState::redo_transaction(const PreparedWalletTransaction &pwtx, const } for (auto &&tr : input_transfers) { for (auto &&out : tr.outputs) { - delta_state->add_incoming_keyimage(block_height, out.key_image); + if (m_wallet.is_det_viewonly()) + delta_state->add_incoming_deterministic_input(block_height, out.amount, out.index, out.public_key); + else + delta_state->add_incoming_keyimage(block_height, out.key_image); } } - ptx.transfers.insert(ptx.transfers.end(), input_transfers.begin(), input_transfers.end()); - delta_state->add_transaction(block_height, tid, pwtx.tx, ptx); + // order of add_transaction is important - DeltaState associates subsequent add_ with last added transaction return true; } @@ -643,48 +643,73 @@ bool WalletState::api_has_transaction(Hash tid, bool check_pool) const { return has_transaction(tid); } -bool WalletState::api_get_transaction(Hash tid, bool check_pool, TransactionPrefix *tx, api::Transaction *ptx) const { +bool WalletState::api_get_transaction(Hash tid, bool check_pool, TransactionPrefix *tx, api::Transaction *atx) const { if (check_pool) { auto mit = m_memory_state.get_transactions().find(tid); if (mit != m_memory_state.get_transactions().end()) { - *tx = mit->second.first; - *ptx = mit->second.second; + *tx = mit->second.tx; + *atx = mit->second.atx; return true; } } - return get_transaction(tid, tx, ptx); + return get_transaction(tid, tx, atx); } -bool WalletState::api_create_proof(SendProof &sp) const { - TransactionPrefix tx; - api::Transaction ptx; - if (!api_get_transaction(sp.transaction_hash, true, &tx, &ptx)) - return false; - KeyPair tx_keys = TransactionBuilder::deterministic_keys_from_seed( - tx, ptx.coinbase ? m_wallet.get_coinbase_tx_derivation_seed() : m_wallet.get_tx_derivation_seed()); - if (!crypto::generate_key_derivation(sp.address.view_public_key, tx_keys.secret_key, sp.derivation)) - return false; - Hash message_hash = crypto::cn_fast_hash(sp.message.data(), sp.message.size()); - if (!crypto::generate_sendproof(tx_keys.public_key, tx_keys.secret_key, sp.address.view_public_key, sp.derivation, - message_hash, sp.signature)) - return false; - Amount total_amount = 0; - size_t key_index = 0; - uint32_t out_index = 0; - for (const auto &output : tx.outputs) { - if (output.target.type() == typeid(KeyOutput)) { - const KeyOutput &key_output = boost::get(output.target); - PublicKey spend_key; - if (underive_public_key(sp.derivation, key_index, key_output.public_key, spend_key) && - spend_key == sp.address.spend_public_key) { - total_amount += output.amount; - } - ++key_index; +bool WalletState::api_create_proof(const TransactionPrefix &tx, Sendproof &sp) const { + const Hash tx_inputs_hash = get_transaction_inputs_hash(tx); + KeyPair tx_keys = TransactionBuilder::transaction_keys_from_seed(tx_inputs_hash, m_wallet.get_tx_derivation_seed()); + if (sp.address.type() == typeid(AccountAddressSimple)) { + auto &addr = boost::get(sp.address); + SendproofKey var; + var.derivation = crypto::generate_key_derivation(addr.view_public_key, tx_keys.secret_key); + Hash message_hash = crypto::cn_fast_hash(sp.message.data(), sp.message.size()); + var.signature = crypto::generate_sendproof( + tx_keys.public_key, tx_keys.secret_key, addr.view_public_key, var.derivation, message_hash); + Amount total_amount = 0; + for (size_t out_index = 0; out_index != tx.outputs.size(); ++out_index) { + const auto &output = tx.outputs.at(out_index); + if (output.type() != typeid(OutputKey)) + continue; + const auto &key_output = boost::get(output); + const PublicKey spend_key = underive_public_key(var.derivation, out_index, key_output.public_key); + if (spend_key != addr.spend_public_key) + continue; + total_amount += key_output.amount; + } + sp.amount = total_amount; + sp.proof = var; + return total_amount != 0; + } + if (sp.address.type() == typeid(AccountAddressUnlinkable)) { + auto &addr = boost::get(sp.address); + Hash tx_inputs_hash = get_transaction_inputs_hash(tx); + SendproofUnlinkable var; + Amount total_amount = 0; + for (size_t out_index = 0; out_index != tx.outputs.size(); ++out_index) { + const auto &output = tx.outputs.at(out_index); + if (output.type() != typeid(OutputKey)) + continue; + const auto &key_output = boost::get(output); + KeyPair output_secret_keys = TransactionBuilder::deterministic_keys_from_seed( + tx_inputs_hash, m_wallet.get_tx_derivation_seed(), common::get_varint_data(out_index)); + Hash output_secret = + crypto::cn_fast_hash(output_secret_keys.public_key.data, sizeof(output_secret_keys.public_key.data)); + AccountAddressUnlinkable address; + if (!crypto::unlinkable_underive_address(output_secret, tx_inputs_hash, out_index, key_output.public_key, + key_output.encrypted_secret, &address.s, &address.sv) || + address != addr) + continue; + SendproofUnlinkable::Element el{out_index, output_secret_keys.public_key, Signature{}}; + el.signature = + crypto::generate_signature(output_secret, output_secret_keys.public_key, output_secret_keys.secret_key); + var.elements.push_back(el); + total_amount += key_output.amount; } - ++out_index; + sp.amount = total_amount; + sp.proof = var; + return total_amount != 0; } - sp.amount = total_amount; - return total_amount != 0; + return false; } api::Block WalletState::api_get_pool_as_history(const std::string &address) const { @@ -692,10 +717,8 @@ api::Block WalletState::api_get_pool_as_history(const std::string &address) cons api::Block current_block; current_block.header.height = get_tip_height() + 1; for (const auto &hit : m_memory_state.get_transactions()) { - auto tx = hit.second.second; + auto tx = hit.second.atx; tx.block_height = get_tip_height() + 1; - // for(auto & tr : tx.transfers) // TODO - remove after DB version switch - // tr.transaction_hash = hit.second.second.hash; if (!address.empty()) { for (auto tit = tx.transfers.begin(); tit != tx.transfers.end();) if (tit->address == address) diff --git a/src/Core/WalletState.hpp b/src/Core/WalletState.hpp index 6407f3e3..2d1df332 100644 --- a/src/Core/WalletState.hpp +++ b/src/Core/WalletState.hpp @@ -13,38 +13,45 @@ #include "Multicore.hpp" #include "Wallet.hpp" #include "WalletStateBasic.hpp" -#include "crypto/chacha8.hpp" #include "platform/DB.hpp" -namespace bytecoin { +namespace cn { class Config; class WalletState : public WalletStateBasic { class DeltaState : public IWalletState { typedef std::map> Unspents; - Unspents m_unspents; - std::map m_used_keyimages; // counter, because double spends are allowed in pool - std::map> m_transactions; public: + struct DeltaStateTransaction { + TransactionPrefix tx; + api::Transaction atx; + std::set used_ki_or_pk; + }; explicit DeltaState() {} void clear(); const Unspents &get_unspents() const { return m_unspents; } - const std::map &get_used_key_images() const { return m_used_keyimages; } + const std::map &get_used_kis_or_pks() const { return m_used_kis_or_pks; } - const std::map> &get_transactions() const { - return m_transactions; - } - // bool is_spent(const api::Output &) const; + const std::map &get_transactions() const { return m_transactions; } void undo_transaction(const Hash &tid); // For mem pool - virtual Amount add_incoming_output(const api::Output &, const Hash &tid) override; // added amount may be lower - virtual Amount add_incoming_keyimage(Height, const KeyImage &) override; - virtual void add_transaction( + Amount add_incoming_output(const api::Output &, const Hash &tid) override; // added amount may be lower + Amount add_incoming_keyimage(Height, const KeyImage &) override; + Amount add_incoming_deterministic_input( + Height block_height, Amount am, size_t gi, const PublicKey &pk) override; + void add_transaction( Height, const Hash &tid, const TransactionPrefix &tx, const api::Transaction &ptx) override; + + private: + Unspents m_unspents; + std::map + m_used_kis_or_pks; // counter, because double spends are allowed in pool + std::map m_transactions; + Hash m_last_added_transaction; }; public: @@ -53,44 +60,45 @@ class WalletState : public WalletStateBasic { const Wallet &get_wallet() const { return m_wallet; } Wallet &get_wallet() { return m_wallet; } - bool sync_with_blockchain(api::bytecoind::SyncBlocks::Response &); // We move from it - bool sync_with_blockchain(api::bytecoind::SyncMemPool::Response &); // We move from it + bool sync_with_blockchain(api::cnd::SyncBlocks::Response &); // We move from it + bool sync_with_blockchain(api::cnd::SyncMemPool::Response &); // We move from it virtual std::vector api_get_locked_or_unconfirmed_unspent( const std::string &address, Height confirmed_height) const override; virtual api::Balance get_balance(const std::string &address, Height confirmed_height) const override; bool add_to_payment_queue(const BinaryArray &binary_transaction, bool save_file); - void process_payment_queue_send_error(Hash hash, const api::bytecoind::SendTransaction::Error &error); + void process_payment_queue_send_error(Hash hash, const api::cnd::SendTransaction::Error &error); BinaryArray get_next_from_sending_queue(Hash *previous_hash); bool api_has_transaction(Hash tid, bool check_pool) const; - bool api_get_transaction(Hash tid, bool check_pool, TransactionPrefix *tx, api::Transaction *ptx) const; + bool api_get_transaction(Hash tid, bool check_pool, TransactionPrefix *tx, api::Transaction *atx) const; - bool parse_raw_transaction(api::Transaction &ptx, const TransactionPrefix &tx, Hash tid) const; + bool parse_raw_transaction(bool is_base, api::Transaction &ptx, Transaction &&tx, Hash tid) const; // Read state - bool api_create_proof(SendProof &sp) const; + bool api_create_proof(const TransactionPrefix &tx, Sendproof &sp) const; api::Block api_get_pool_as_history(const std::string &address) const; - uint32_t get_tx_pool_version() const { return m_tx_pool_version; } - uint32_t get_pq_version() const { return m_pq_version; } + size_t get_tx_pool_version() const { return m_tx_pool_version; } + size_t get_pq_version() const { return m_pq_version; } std::vector get_tx_pool_hashes() const; void wallet_addresses_updated(); // generating through state prevents undo of blocks within 2*block_future_time_limit from now std::vector generate_new_addresses(const std::vector &sks, Timestamp ct, Timestamp now); + void create_addresses(size_t count); protected: bool redo_block(const api::BlockHeader &header, const PreparedWalletBlock &block, const BlockChainState::BlockGlobalIndices &global_indices, Height height); - bool parse_raw_transaction(api::Transaction *ptx, std::vector *input_transfers, - Amount *output_amount, const PreparedWalletTransaction &pwtx, Hash tid, - const std::vector &global_indices, Height block_heights) const; - bool redo_transaction(const PreparedWalletTransaction &pwtx, const std::vector &global_indices, + bool parse_raw_transaction(bool is_base, api::Transaction *ptx, std::vector *input_transfers, + std::vector *output_transfers, Amount *output_amount, const PreparedWalletTransaction &pwtx, + Hash tid, const std::vector &global_indices, Height block_heights) const; + bool redo_transaction(const PreparedWalletTransaction &pwtx, const std::vector &global_indices, IWalletState *delta_state, bool is_base, Hash tid, Height block_height, Hash bid, Timestamp tx_timestamp); - const std::map &get_used_key_images() const override; + const std::map &get_mempool_kis_or_pks() const override; void on_first_transaction_found(Timestamp ts) override; struct QueueEntry { @@ -100,18 +108,17 @@ class WalletState : public WalletStateBasic { Height remove_height = 0; bool in_blockchain() const { return remove_height != 0; } }; - Height get_pq_confirmations() const; private: - uint32_t m_tx_pool_version = 1; - uint32_t m_pq_version = 1; - std::chrono::steady_clock::time_point log_redo_block; + size_t m_tx_pool_version = 1; + size_t m_pq_version = 1; + std::chrono::steady_clock::time_point m_log_redo_block; Wallet &m_wallet; DeltaState m_memory_state; std::set m_pool_hashes; - void add_transaction_to_mempool(Hash tid, TransactionPrefix &&tx, bool from_pq); + void add_transaction_to_mempool(Hash tid, Transaction &&tx, bool from_pq); void remove_transaction_from_mempool(Hash tid, bool from_pq); struct by_hash {}; @@ -133,4 +140,4 @@ class WalletState : public WalletStateBasic { WalletPreparatorMulticore preparator; }; -} // namespace bytecoin +} // namespace cn diff --git a/src/Core/WalletStateBasic.cpp b/src/Core/WalletStateBasic.cpp index f1ff27ec..fba69c0e 100644 --- a/src/Core/WalletStateBasic.cpp +++ b/src/Core/WalletStateBasic.cpp @@ -17,7 +17,7 @@ static const auto LEVEL = logging::TRACE; -static const std::string version_current = "4"; +static const std::string version_current = "9"; static const std::string INDEX_UID_to_STATE = "X"; // We do not store it for empty blocks @@ -37,6 +37,9 @@ static const std::string INDEX_ADDRESS_to_BALANCE = "ba"; // for get_balance // (ki) -> (he, am, gi) <- find largest coin from same ki group (can be spent or not) static const std::string INDEX_KEYIMAGE_to_HE_AM_GI = "ki"; // ki->output_key, if !view_only +// (am, gi) -> (he, pk) <- find coin by am, gi (can be spent or not) +static const std::string INDEX_AM_GI_to_HE_PK = "g"; + // (he, am, gi) -> output <- find available unspents (never locked or already unlocked) // (addr, he, am, gi) -> () <- find available unspents by addr (never locked or already unlocked) static const std::string INDEX_HE_AM_GI_to_OUTPUT = "un"; @@ -52,14 +55,10 @@ static const std::string LOCKED_INDEX_KI_AM_GI = "li"; // (unl_he, am, gi) -> (output) <- find yet locked by height // (unl_ti, am, gi) -> (output) <- find yet locked by timestamp -static const std::string LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT = "lh"; // key contain clamped unlock_block_or_timestamp -static const std::string LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT = - "lt"; // key contain clamped unlock_block_or_timestamp - -static const std::string LOCKED_INDEX_AM_GI_to_TID = "Y"; // Remove on next DB upgrade -// Index is simply overwritten and not undone. +static const std::string LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT = "lh"; // key contain unlock_block_or_timestamp +static const std::string LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT = "lt"; // key contain unlock_block_or_timestamp -using namespace bytecoin; +using namespace cn; using namespace platform; void seria::ser_members(WalletStateBasic::HeightAmounGi &v, ISeria &s) { @@ -73,36 +72,20 @@ void seria::ser_members(WalletStateBasic::UndoValue &v, seria::ISeria &s) { seria_kv("value", v.value, s); } -/*template -std::string to_binary_key(const T &s) { - static_assert(std::is_standard_layout::value, "T must be Standard Layout"); - return common::to_hex(&s, sizeof(s)); // WalletState::DB::to_binary_key((const unsigned char *)&s, sizeof(s)); -} - -template -void from_binary_key(const std::string &key, T &s) { - static_assert(std::is_standard_layout::value, "T must be Standard Layout"); - invariant(common::pod_from_hex(key, s), "from_binary_key failed for key " + key); -}*/ - -WalletStateBasic::WalletStateBasic( - logging::ILogger &log, const Config &config, const Currency ¤cy, const std::string &cache_name) +WalletStateBasic::WalletStateBasic(logging::ILogger &log, const Config &config, const Currency ¤cy, + const std::string &cache_name, bool is_det_viewonly) : m_genesis_bid(currency.genesis_block_hash) , m_config(config) , m_currency(currency) , m_log(log, "WalletState") - , m_db(false, config.get_data_folder("wallet_cache") + "/" + cache_name, 0x2000000000) // 128 gb -{ + , m_db(platform::O_OPEN_ALWAYS, config.get_data_folder("wallet_cache") + "/" + cache_name, 0x2000000000) // 128 gb + , is_det_viewonly(is_det_viewonly) { std::string version; std::string other_genesis_bid; std::string other_cache_name; m_db.get("$version", version); m_db.get("$genesis_bid", other_genesis_bid); m_db.get("$cache_name", other_cache_name); - if (version == "3") { - version_3_to_4(); - m_db.get("$version", version); - } if (version != version_current || other_genesis_bid != common::pod_to_hex(m_genesis_bid) || other_cache_name != cache_name) { if (!version.empty()) @@ -134,60 +117,27 @@ WalletStateBasic::WalletStateBasic( } } -void WalletStateBasic::version_3_to_4() { - // size_t total_items = m_db.get_approximate_items_count(); - m_log(logging::INFO) - << "Updating locked transactions index. This will take a minute or two depending on your wallet size and computer speed..." - << std::endl; - int transaction_counter = 0; - int upgrade_counter = 0; - int last_hash_byte = -1; - for (DB::Cursor cur = m_db.begin(INDEX_TID_to_TRANSACTIONS); !cur.end(); cur.next()) { - const std::string &suf = cur.get_suffix(); - const char *be = suf.data(); - const char *en = be + suf.size(); - Hash tid; - invariant(en - be == sizeof(tid.data), "INDEX_TID_to_TRANSACTIONS corrupted"); - DB::from_binary_key(cur.get_suffix(), 0, tid.data, sizeof(tid.data)); - if (tid.data[0] != last_hash_byte) { - last_hash_byte = tid.data[0]; - m_log(logging::INFO) << "Transactions processed " << transaction_counter << " (" - << (last_hash_byte * 100 / 256) << "%)" << std::endl; - } - std::pair pa; - seria::from_binary(pa, cur.get_value_array()); - transaction_counter += 1; - if (pa.second.unlock_block_or_timestamp == 0) - continue; - for (const auto &tr : pa.second.transfers) - for (const auto &output : tr.outputs) { - put_am_gi_tid(output.amount, output.index, tid); - upgrade_counter += 1; - } - } - m_db.put("$version", "4", false); - m_log(logging::INFO) << "Updating locked transactions index finished. " << transaction_counter - << " transactions processed, " << upgrade_counter << " DB records modified." << std::endl; -} -void WalletStateBasic::put_am_gi_tid(Amount am, uint32_t gi, Hash tid) { - std::string unkey = LOCKED_INDEX_AM_GI_to_TID + common::write_varint_sqlite4(am) + common::write_varint_sqlite4(gi); - BinaryArray ba = seria::to_binary(tid); - m_db.put(unkey, ba, false); +void WalletStateBasic::put_am_gi_he(Amount am, size_t gi, Height he, const PublicKey &pk) { + std::string unkey = INDEX_AM_GI_to_HE_PK + common::write_varint_sqlite4(am) + common::write_varint_sqlite4(gi); + BinaryArray ba = seria::to_binary(std::make_pair(he, pk)); + put_with_undo(unkey, ba, true); } -Hash WalletStateBasic::get_am_gi_tid(Amount am, uint32_t gi) const { - std::string unkey = LOCKED_INDEX_AM_GI_to_TID + common::write_varint_sqlite4(am) + common::write_varint_sqlite4(gi); +bool WalletStateBasic::get_am_gi_he(Amount am, size_t gi, Height *he, PublicKey *pk) const { + std::string unkey = INDEX_AM_GI_to_HE_PK + common::write_varint_sqlite4(am) + common::write_varint_sqlite4(gi); BinaryArray ba; if (!m_db.get(unkey, ba)) - return Hash{}; - Hash tid; - seria::from_binary(tid, ba); - return tid; + return false; + std::pair pa; + seria::from_binary(pa, ba); + *he = pa.first; + *pk = pa.second; + return true; } void WalletStateBasic::combine_balance( api::Balance &balance, const api::Output &output, int locked_op, int spendable_op) { - common::Uint128 &mod = output.dust ? balance.spendable_dust : balance.spendable; - uint64_t &mod_coins = output.dust ? balance.spendable_dust_outputs : balance.spendable_outputs; + auto &mod = output.dust ? balance.spendable_dust : balance.spendable; + auto &mod_coins = output.dust ? balance.spendable_dust_outputs : balance.spendable_outputs; if (locked_op > 0) balance.locked_or_unconfirmed += output.amount; if (locked_op < 0) @@ -206,10 +156,17 @@ void WalletStateBasic::db_commit() { m_log(logging::TRACE) << "WalletState::db_commit finished..." << std::endl; } +static crypto::EllipticCurvePoint ki_or_pk(const api::Output &v, bool is_det_viewonly) { + crypto::EllipticCurvePoint p = v.key_image; + if (is_det_viewonly) + p = v.public_key; + return p; +} + std::string WalletStateBasic::format_output(const api::Output &v) { std::stringstream str; str << " he=" << v.height << " am=" << m_currency.format_amount(v.amount) << " gi=" << v.index - << " ki=" << v.key_image << " addr=" << v.address + << " ki=" << ki_or_pk(v, is_det_viewonly) << " addr=" << v.address << (v.unlock_block_or_timestamp == 0 ? "" : " unl=" + common::to_string(v.unlock_block_or_timestamp)); return str.str(); } @@ -219,7 +176,6 @@ void WalletStateBasic::push_chain(const api::BlockHeader &header) { BinaryArray ba = seria::to_binary(header); m_db.put(INDEX_HEIGHT_to_HEADER + common::write_varint_sqlite4(m_tip_height), ba, true); m_tip = header; - // m_memory_state.set_height(m_tip_height + 1); - TODO do not forget save_db_state(m_tip_height, current_undo_map); current_undo_map.clear(); } @@ -245,7 +201,7 @@ void WalletStateBasic::reset_chain(Height new_tail_height) { m_tip_height = m_tail_height - 1; } -bool WalletStateBasic::read_chain(uint32_t height, api::BlockHeader &header) const { +bool WalletStateBasic::read_chain(Height height, api::BlockHeader &header) const { BinaryArray rb; if (!m_db.get(INDEX_HEIGHT_to_HEADER + common::write_varint_sqlite4(height), rb)) return false; @@ -253,7 +209,7 @@ bool WalletStateBasic::read_chain(uint32_t height, api::BlockHeader &header) con return true; } -api::BlockHeader WalletStateBasic::read_chain(uint32_t height) const { +api::BlockHeader WalletStateBasic::read_chain(Height height) const { api::BlockHeader ha; invariant(read_chain(height, ha), "read_header_chain failed"); return ha; @@ -262,7 +218,7 @@ api::BlockHeader WalletStateBasic::read_chain(uint32_t height) const { std::vector WalletStateBasic::get_sparse_chain() const { std::vector tip_path; - uint32_t jump = 0; + Height jump = 0; if (m_tip_height + 1 > m_tail_height) while (m_tip_height >= jump + m_tail_height) { tip_path.push_back(read_chain(m_tip_height - jump).hash); @@ -305,7 +261,7 @@ void WalletStateBasic::del_with_undo(const std::string &key, bool mustexist) { // current_undo_map.erase(kit); } -void WalletStateBasic::save_db_state(uint32_t state, const UndoMap &undo_map) { +void WalletStateBasic::save_db_state(Height state, const UndoMap &undo_map) { if (undo_map.empty()) return; const auto key = INDEX_UID_to_STATE + common::write_varint_sqlite4(state); @@ -313,7 +269,7 @@ void WalletStateBasic::save_db_state(uint32_t state, const UndoMap &undo_map) { m_db.put(key, value, true); } -void WalletStateBasic::undo_db_state(uint32_t state) { +void WalletStateBasic::undo_db_state(Height state) { const auto key = INDEX_UID_to_STATE + common::write_varint_sqlite4(state); common::BinaryArray value; if (!m_db.get(key, value)) @@ -331,7 +287,7 @@ void WalletStateBasic::undo_db_state(uint32_t state) { bool WalletStateBasic::try_add_incoming_output(const api::Output &output, Amount *confirmed_balance_delta) const { HeightAmounGi heamgi; - bool ki_exists = read_by_keyimage(output.key_image, &heamgi); + bool ki_exists = read_by_keyimage(ki_or_pk(output, is_det_viewonly), &heamgi); api::Output existing_output; bool is_existing_unspent = ki_exists && read_from_unspent_index(heamgi, &existing_output); if (ki_exists && !is_existing_unspent) @@ -354,7 +310,7 @@ bool WalletStateBasic::try_add_incoming_output(const api::Output &output, Amount Amount WalletStateBasic::add_incoming_output(const api::Output &output, const Hash &tid, bool just_unlocked) { HeightAmounGi heamgi; - bool ki_exists = read_by_keyimage(output.key_image, &heamgi); + bool ki_exists = read_by_keyimage(ki_or_pk(output, is_det_viewonly), &heamgi); api::Output existing_output; bool is_existing_unspent = ki_exists && read_from_unspent_index(heamgi, &existing_output); if (ki_exists && !is_existing_unspent) { @@ -362,15 +318,18 @@ Amount WalletStateBasic::add_incoming_output(const api::Output &output, const Ha return 0; } if (output.unlock_block_or_timestamp != 0 && !just_unlocked) { // incoming + if (is_det_viewonly) + put_am_gi_he(output.amount, output.index, output.height, output.public_key); add_to_lock_index(output, tid); return output.amount; } Amount added_amount = output.amount; - if (ki_exists) { // implies is_existing_unspent - // Replacement of coins in the same KI group cannot be done safely - // Because replacement can occur while transaction to spend first coin is in mempool - // This will lead to crediting attacker's account first, then substracting from exchange account. - // if (output.amount <= heamgi.amount || output.address != existing_output.address) { + if (ki_exists) { + // implies is_existing_unspent + // Replacement of coins in the same KI group cannot be done safely + // Because replacement can occur while transaction to spend first coin is in mempool + // This will lead to crediting attacker's account first, then substracting from exchange account. + // if (output.amount <= heamgi.amount || output.address != existing_output.address) { m_log(logging::WARNING) << " Duplicate key_output attack, ignoring output because have another one unspent with same or larger amount or different address, " << format_output(existing_output) << std::endl; @@ -387,7 +346,9 @@ Amount WalletStateBasic::add_incoming_output(const api::Output &output, const Ha heamgi.height = output.height; heamgi.amount = output.amount; heamgi.global_index = output.index; - update_keyimage(output.key_image, heamgi, !ki_exists); + update_keyimage(ki_or_pk(output, is_det_viewonly), heamgi, !ki_exists); + if (is_det_viewonly) + put_am_gi_he(output.amount, output.index, output.height, output.public_key); return added_amount; } @@ -397,8 +358,14 @@ Amount WalletStateBasic::add_incoming_output(const api::Output &output, const Ha } Amount WalletStateBasic::add_incoming_keyimage(Height block_height, const KeyImage &key_image) { - m_log(LEVEL) << "Incoming keyimage " << key_image << std::endl; - std::string prefix = LOCKED_INDEX_KI_AM_GI + DB::to_binary_key(key_image.data, sizeof(key_image.data)); + if (is_det_viewonly) + return 0; + return add_incoming_ki_or_pk(block_height, key_image); +} + +Amount WalletStateBasic::add_incoming_ki_or_pk(Height block_height, const crypto::EllipticCurvePoint &ki_or_pk) { + m_log(LEVEL) << "Incoming ki_or_pk " << ki_or_pk << std::endl; + std::string prefix = LOCKED_INDEX_KI_AM_GI + DB::to_binary_key(ki_or_pk.data, sizeof(ki_or_pk.data)); // find and remove in locked std::vector found_in_locked; for (DB::Cursor cur = m_db.begin(prefix); !cur.end(); cur.next()) { @@ -406,15 +373,14 @@ Amount WalletStateBasic::add_incoming_keyimage(Height block_height, const KeyIma const char *be = suf.data(); const char *en = be + suf.size(); Amount am = common::read_varint_sqlite4(be, en); - uint32_t gi = common::integer_cast(common::read_varint_sqlite4(be, en)); + size_t gi = common::integer_cast(common::read_varint_sqlite4(be, en)); invariant(en - be == 0, ""); BlockOrTimestamp unl = 0; seria::from_binary(unl, cur.get_value_array()); - uint32_t clamped_unlock_time = static_cast(std::min(unl, 0xFFFFFFFF)); - std::string unkey = m_currency.is_transaction_spend_time_block(unl) ? LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT - : LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT; - unkey += common::write_varint_sqlite4(clamped_unlock_time) + common::write_varint_sqlite4(am) + - common::write_varint_sqlite4(gi); + std::string unkey = m_currency.is_block_or_timestamp_block(unl) ? LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT + : LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT; + unkey += + common::write_varint_sqlite4(unl) + common::write_varint_sqlite4(am) + common::write_varint_sqlite4(gi); BinaryArray output_ba; invariant(m_db.get(unkey, output_ba), ""); api::Output output; @@ -426,7 +392,7 @@ Amount WalletStateBasic::add_incoming_keyimage(Height block_height, const KeyIma } Amount removed_amount = 0; HeightAmounGi heamgi; - bool ki_exists = read_by_keyimage(key_image, &heamgi); + bool ki_exists = read_by_keyimage(ki_or_pk, &heamgi); api::Output existing_output; if (ki_exists && read_from_unspent_index(heamgi, &existing_output)) { removed_amount = existing_output.amount; @@ -436,29 +402,35 @@ Amount WalletStateBasic::add_incoming_keyimage(Height block_height, const KeyIma } bool WalletStateBasic::try_adding_incoming_keyimage(const KeyImage &key_image, api::Output *spending_output) const { + if (is_det_viewonly) + return false; + return try_adding_incoming_ki_or_pk(key_image, spending_output); +} + +bool WalletStateBasic::try_adding_incoming_ki_or_pk( + const crypto::EllipticCurvePoint &ki_or_pk, api::Output *spending_output) const { bool candidate_found = false; HeightAmounGi heamgi; - bool ki_exists = read_by_keyimage(key_image, &heamgi); + bool ki_exists = read_by_keyimage(ki_or_pk, &heamgi); if (ki_exists && read_from_unspent_index(heamgi, spending_output)) { candidate_found = true; } - std::string prefix = LOCKED_INDEX_KI_AM_GI + DB::to_binary_key(key_image.data, sizeof(key_image.data)); + std::string prefix = LOCKED_INDEX_KI_AM_GI + DB::to_binary_key(ki_or_pk.data, sizeof(ki_or_pk.data)); for (DB::Cursor cur = m_db.begin(prefix); !cur.end(); cur.next()) { const std::string &suf = cur.get_suffix(); const char *be = suf.data(); const char *en = be + suf.size(); Amount am = common::read_varint_sqlite4(be, en); - uint32_t gi = common::integer_cast(common::read_varint_sqlite4(be, en)); + size_t gi = common::integer_cast(common::read_varint_sqlite4(be, en)); invariant(en - be == 0, ""); if (candidate_found && am <= spending_output->amount) continue; BlockOrTimestamp unl = 0; seria::from_binary(unl, cur.get_value_array()); - uint32_t clamped_unlock_time = static_cast(std::min(unl, 0xFFFFFFFF)); - std::string unkey = m_currency.is_transaction_spend_time_block(unl) ? LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT - : LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT; - unkey += common::write_varint_sqlite4(clamped_unlock_time) + common::write_varint_sqlite4(am) + - common::write_varint_sqlite4(gi); + std::string unkey = m_currency.is_block_or_timestamp_block(unl) ? LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT + : LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT; + unkey += + common::write_varint_sqlite4(unl) + common::write_varint_sqlite4(am) + common::write_varint_sqlite4(gi); BinaryArray output_ba; invariant(m_db.get(unkey, output_ba), ""); api::Output output; @@ -472,6 +444,28 @@ bool WalletStateBasic::try_adding_incoming_keyimage(const KeyImage &key_image, a return candidate_found; } +bool WalletStateBasic::try_adding_deterministic_input(Amount am, size_t gi, api::Output *spending_output) const { + if (!is_det_viewonly) + return false; + HeightAmounGi heamgi{0, am, gi}; + PublicKey pk; + if (!get_am_gi_he(am, gi, &heamgi.height, &pk)) + return false; + return try_adding_incoming_ki_or_pk(pk, spending_output); +} + +Amount WalletStateBasic::add_incoming_deterministic_input( + Height block_height, Amount am, size_t gi, const PublicKey &pk) { + if (!is_det_viewonly) + return 0; + HeightAmounGi heamgi{0, am, gi}; + PublicKey pk2; + if (!get_am_gi_he(am, gi, &heamgi.height, &pk2)) + return 0; + invariant(pk == pk2, ""); + return add_incoming_ki_or_pk(block_height, pk); +} + void WalletStateBasic::add_transaction( Height height, const Hash &tid, const TransactionPrefix &tx, const api::Transaction &ptx) { auto cur = m_db.begin(INDEX_TID_to_TRANSACTIONS); @@ -494,13 +488,13 @@ void WalletStateBasic::add_transaction( bool WalletStateBasic::api_add_unspent(std::vector *result, Amount *total_amount, const std::string &address, Height confirmed_height, Amount max_amount) const { - auto recently_unlocked = get_unlocked_outputs(address, confirmed_height, std::numeric_limits::max()); + auto recently_unlocked = get_unlocked_outputs(address, confirmed_height + 1, std::numeric_limits::max()); const size_t min_count = 10000; // We return up to 10k outputs after we find requested sum - return for_each_in_unspent_index(address, Height(-1), confirmed_height, [&](const api::Output &output) -> bool { + return for_each_in_unspent_index(address, 0, confirmed_height + 1, [&](api::Output &&output) -> bool { if (!is_memory_spent(output) && recently_unlocked.count(std::make_pair(output.amount, output.index)) == 0) { - result->push_back(output); - if (!output.dust) // We ensure total can be spent with non-zero anonymity - *total_amount += output.amount; + // if (!output.dust) // We ensure total can be spent with non-zero anonymity + // *total_amount += output.amount; + result->push_back(std::move(output)); if (*total_amount >= max_amount && result->size() >= min_count) return false; // Stop looking for } @@ -509,12 +503,13 @@ bool WalletStateBasic::api_add_unspent(std::vector *result, Amount } std::vector WalletStateBasic::api_get_transfers( - const std::string &address, Height *from_height, Height *to_height, bool forward, uint32_t desired_tx_count) const { + const std::string &address, Height *from_height, Height *to_height, bool forward, size_t desired_tx_count) const { std::vector result; if (*from_height >= *to_height) return result; - auto prefix = INDEX_ADDRESS_HEIGHT_TID + address + "/"; - std::string middle = common::write_varint_sqlite4(forward ? *from_height + 1 : *to_height); + auto prefix = INDEX_ADDRESS_HEIGHT_TID + address + "/"; + std::string middle = + common::write_varint_sqlite4(forward ? *from_height : *to_height - 1); // to_height != 0 checked in if above api::Block current_block; size_t total_transactions_found = 0; for (DB::Cursor cur = forward ? m_db.begin(prefix, middle) : m_db.rbegin(prefix, middle); !cur.end(); cur.next()) { @@ -525,9 +520,9 @@ std::vector WalletStateBasic::api_get_transfers( Hash tid; invariant(en - be == sizeof(tid.data), "CD_TIPS_PREFIX corrupted"); DB::from_binary_key(cur.get_suffix(), cur.get_suffix().size() - sizeof(tid.data), tid.data, sizeof(tid.data)); - if (forward && height > *to_height) + if (forward && height >= *to_height) break; - if (!forward && height <= *from_height) + if (!forward && height < *from_height) break; TransactionPrefix ptx; api::Transaction tx; @@ -546,9 +541,9 @@ std::vector WalletStateBasic::api_get_transfers( current_block = api::Block(); if (total_transactions_found >= desired_tx_count) { if (forward) - *to_height = height - 1; + *to_height = height; else - *from_height = height; + *from_height = height + 1; break; } } @@ -568,12 +563,12 @@ std::vector WalletStateBasic::api_get_locked_or_unconfirmed_unspent Height confirmed_height) const { std::vector result; for_each_in_unspent_index( - address, confirmed_height, std::numeric_limits::max(), [&](const api::Output &output) -> bool { + address, confirmed_height + 1, std::numeric_limits::max(), [&](api::Output &&output) -> bool { if (!is_memory_spent(output)) - result.push_back(output); + result.push_back(std::move(output)); return true; - }); - auto recently_unlocked = get_unlocked_outputs(address, confirmed_height, std::numeric_limits::max()); + }); + auto recently_unlocked = get_unlocked_outputs(address, confirmed_height + 1, std::numeric_limits::max()); for (auto &&lou : recently_unlocked) { HeightAmounGi heamgi; heamgi.height = lou.second.height; @@ -586,9 +581,11 @@ std::vector WalletStateBasic::api_get_locked_or_unconfirmed_unspent if (lou.second.height <= confirmed_height) result.push_back(lou.second); } - std::map, api::Output> still_locked; - read_unlock_index(&still_locked, LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT, address, uint32_t(-1), 0xFFFFFFFF); - read_unlock_index(&still_locked, LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT, address, uint32_t(-1), 0xFFFFFFFF); + std::map, api::Output> still_locked; + read_unlock_index(&still_locked, LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT, address, 0, + std::numeric_limits::max()); + read_unlock_index( + &still_locked, LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT, address, 0, std::numeric_limits::max()); for (auto &&lou : still_locked) if (!is_memory_spent(lou.second)) result.push_back(std::move(lou.second)); @@ -596,7 +593,7 @@ std::vector WalletStateBasic::api_get_locked_or_unconfirmed_unspent } // spendable: unspent [0..conf] && !recently unlocked && !spent -// unconfirmed unspent (conf..inf] || recently_unlocked +// unconfirmed: unspent (conf..inf] || recently_unlocked api::Balance WalletStateBasic::get_balance(const std::string &address, Height confirmed_height) const { auto bakey = INDEX_ADDRESS_to_BALANCE + address; @@ -606,15 +603,15 @@ api::Balance WalletStateBasic::get_balance(const std::string &address, Height co seria::from_binary(balance, ba); for_each_in_unspent_index( - address, confirmed_height, std::numeric_limits::max(), [&](const api::Output &output) -> bool { + address, confirmed_height + 1, std::numeric_limits::max(), [&](api::Output &&output) -> bool { if (is_memory_spent(output)) combine_balance(balance, output, 0, -1); else combine_balance(balance, output, 1, -1); return true; - }); + }); - auto recently_unlocked = get_unlocked_outputs(address, confirmed_height, std::numeric_limits::max()); + auto recently_unlocked = get_unlocked_outputs(address, confirmed_height + 1, std::numeric_limits::max()); for (auto &&lou : recently_unlocked) { HeightAmounGi heamgi; heamgi.height = lou.second.height; @@ -627,7 +624,7 @@ api::Balance WalletStateBasic::get_balance(const std::string &address, Height co if (lou.second.height <= confirmed_height) combine_balance(balance, existing_output, 1, -1); } - for (auto &&kit : get_used_key_images()) { + for (auto &&kit : get_mempool_kis_or_pks()) { HeightAmounGi heamgi; bool ki_exists = read_by_keyimage(kit.first, &heamgi); api::Output existing_output; @@ -639,9 +636,11 @@ api::Balance WalletStateBasic::get_balance(const std::string &address, Height co // We commented code below because it requires either iterating all locked index ot all used keyimages // So, we do not account for memory spent locked outputs in unconfirmed balance - // std::map, api::Output> still_locked; - // read_unlock_index(&still_locked, LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT, address, uint32_t(-1), 0xFFFFFFFF); - // read_unlock_index(&still_locked, LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT, address, uint32_t(-1), 0xFFFFFFFF); + // std::map, api::Output> still_locked; + // read_unlock_index(&still_locked, LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT, address, 0, + // std::numeric_limits::max()); + // read_unlock_index(&still_locked, LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT, address, 0, + // std::numeric_limits::max()); // for(auto && lou : still_locked) // if (is_memory_spent(lou.second)) // combine_balance(balance, lou.second, -1, 0); @@ -662,36 +661,31 @@ bool WalletStateBasic::get_transaction(Hash tid, TransactionPrefix *tx, api::Tra return false; std::pair pa; seria::from_binary(pa, data); - for (auto &tr : pa.second.transfers) // TODO - remove after DB version switch - tr.transaction_hash = pa.second.hash; - *tx = std::move(pa.first); - *ptx = std::move(pa.second); + *tx = std::move(pa.first); + *ptx = std::move(pa.second); return true; } -static void parse_lock_key( - const std::string &suffix, uint32_t *clamped_unlocktime, Amount *amount, uint32_t *global_index) { - const char *be = suffix.data(); - const char *en = be + suffix.size(); - *clamped_unlocktime = common::integer_cast(common::read_varint_sqlite4(be, en)); - *amount = common::read_varint_sqlite4(be, en); - *global_index = common::integer_cast(common::read_varint_sqlite4(be, en)); +static void parse_lock_key(const std::string &suffix, BlockOrTimestamp *unl, Amount *amount, size_t *global_index) { + const char *be = suffix.data(); + const char *en = be + suffix.size(); + *unl = common::integer_cast(common::read_varint_sqlite4(be, en)); + *amount = common::read_varint_sqlite4(be, en); + *global_index = common::integer_cast(common::read_varint_sqlite4(be, en)); invariant(en - be == 0, ""); } -// Unique in that it reads (begin, end] interval, not [begin, end) as most other -// funs. That is because block height 312 -// unlocks output with unlock_block_or_timestamp=312 -void WalletStateBasic::read_unlock_index(std::map, api::Output> *add, - const std::string &index_prefix, const std::string &address, uint32_t begin, uint32_t end) const { - if (begin != uint32_t(-1) && begin >= end) // optimization + +void WalletStateBasic::read_unlock_index(std::map, api::Output> *add, + const std::string &index_prefix, const std::string &address, BlockOrTimestamp begin, BlockOrTimestamp end) const { + if (begin >= end) // optimization return; - auto middle = common::write_varint_sqlite4(begin + 1); + auto middle = common::write_varint_sqlite4(begin); for (DB::Cursor cur = m_db.begin(index_prefix, middle); !cur.end(); cur.next()) { - Height height = 0; - Amount amount = 0; - uint32_t global_index; - parse_lock_key(cur.get_suffix(), &height, &amount, &global_index); - if (height > end) + BlockOrTimestamp unl = 0; + Amount amount = 0; + size_t global_index = 0; + parse_lock_key(cur.get_suffix(), &unl, &amount, &global_index); + if (unl >= end) break; api::Output output; seria::from_binary(output, cur.get_value_array()); @@ -704,10 +698,10 @@ void WalletStateBasic::read_unlock_index(std::map, a } } -std::map, api::Output> WalletStateBasic::get_unlocked_outputs(const std::string &address, +std::map, api::Output> WalletStateBasic::get_unlocked_outputs(const std::string &address, Height from_height, Height to_height) const { - std::map, api::Output> unlocked; + std::map, api::Output> unlocked; read_unlock_index(&unlocked, UNLOCKED_INDEX_REALHE_AM_GI_to_OUTPUT, address, from_height, to_height); return unlocked; } @@ -717,12 +711,11 @@ std::vector WalletStateBasic::api_get_unlocked_transfers( auto unlocked = get_unlocked_outputs(address, from_height, to_height); std::map, api::Transfer> transfers; for (auto &unl : unlocked) { - Hash tid = get_am_gi_tid(unl.second.amount, unl.second.index); - api::Transfer &tr = transfers[std::make_pair(tid, unl.second.address)]; + api::Transfer &tr = transfers[std::make_pair(unl.second.transaction_hash, unl.second.address)]; tr.ours = true; tr.amount += unl.second.amount; tr.address = unl.second.address; - tr.transaction_hash = tid; + tr.transaction_hash = unl.second.transaction_hash; tr.outputs.push_back(std::move(unl.second)); } std::vector result; @@ -757,8 +750,10 @@ void WalletStateBasic::modify_balance(const api::Output &output, int locked_op, put_with_undo(bakey2, seria::to_binary(balance2), false); } -static std::map empty_keyimages; -const std::map &WalletStateBasic::get_used_key_images() const { return empty_keyimages; } +static const std::map empty_kis_or_pks; +const std::map &WalletStateBasic::get_mempool_kis_or_pks() const { + return empty_kis_or_pks; +} void WalletStateBasic::unlock(Height now_height, api::Output &&output) { remove_from_lock_index(output); @@ -775,16 +770,14 @@ void WalletStateBasic::unlock(Height now_height, api::Output &&output) { void WalletStateBasic::add_to_lock_index(const api::Output &output, const Hash &tid) { m_log(LEVEL) << " Adding output to lock index, " << format_output(output) << std::endl; - put_am_gi_tid(output.amount, output.index, tid); + // put_am_gi_tid(output.amount, output.index, tid); modify_balance(output, 1, 0); - uint32_t clamped_unlock_time = - static_cast(std::min(output.unlock_block_or_timestamp, 0xFFFFFFFF)); - std::string unkey = m_currency.is_transaction_spend_time_block(output.unlock_block_or_timestamp) + std::string unkey = m_currency.is_block_or_timestamp_block(output.unlock_block_or_timestamp) ? LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT : LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT; - unkey += common::write_varint_sqlite4(clamped_unlock_time) + common::write_varint_sqlite4(output.amount) + - common::write_varint_sqlite4(output.index); + unkey += common::write_varint_sqlite4(output.unlock_block_or_timestamp) + + common::write_varint_sqlite4(output.amount) + common::write_varint_sqlite4(output.index); put_with_undo(unkey, seria::to_binary(output), true); if (output.key_image != KeyImage{}) { unkey = LOCKED_INDEX_KI_AM_GI + DB::to_binary_key(output.key_image.data, sizeof(output.key_image.data)) + @@ -796,13 +789,11 @@ void WalletStateBasic::add_to_lock_index(const api::Output &output, const Hash & void WalletStateBasic::remove_from_lock_index(const api::Output &output) { m_log(LEVEL) << " Removing output from lock index, " << format_output(output) << std::endl; - uint32_t clamped_unlock_time = - static_cast(std::min(output.unlock_block_or_timestamp, 0xFFFFFFFF)); - std::string unkey = m_currency.is_transaction_spend_time_block(output.unlock_block_or_timestamp) + std::string unkey = m_currency.is_block_or_timestamp_block(output.unlock_block_or_timestamp) ? LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT : LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT; - unkey += common::write_varint_sqlite4(clamped_unlock_time) + common::write_varint_sqlite4(output.amount) + - common::write_varint_sqlite4(output.index); + unkey += common::write_varint_sqlite4(output.unlock_block_or_timestamp) + + common::write_varint_sqlite4(output.amount) + common::write_varint_sqlite4(output.index); modify_balance(output, -1, 0); del_with_undo(unkey, true); if (output.key_image != KeyImage{}) { @@ -813,9 +804,9 @@ void WalletStateBasic::remove_from_lock_index(const api::Output &output) { } void WalletStateBasic::unlock(Height now_height, Timestamp now) { - std::map, api::Output> to_unlock; - read_unlock_index(&to_unlock, LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT, std::string(), uint32_t(-1), now_height); - read_unlock_index(&to_unlock, LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT, std::string(), uint32_t(-1), now); + std::map, api::Output> to_unlock; + read_unlock_index(&to_unlock, LOCKED_INDEX_HEIGHT_AM_GI_to_OUTPUT, std::string(), 0, now_height + 1); + read_unlock_index(&to_unlock, LOCKED_INDEX_TIMESTAMP_AM_GI_to_OUTPUT, std::string(), 0, now + 1); if (!to_unlock.empty()) m_log(LEVEL) << "Unlocking for height=" << now_height << ", now=" << now << std::endl; for (auto &&unl : to_unlock) { @@ -833,18 +824,18 @@ bool WalletStateBasic::read_from_unspent_index(const HeightAmounGi &value, api:: return true; } bool WalletStateBasic::for_each_in_unspent_index( - const std::string &address, Height from, Height to, std::function fun) const { + const std::string &address, Height from, Height to, std::function fun) const { auto prefix = address.empty() ? INDEX_HE_AM_GI_to_OUTPUT : INDEX_ADDRESS_HE_AM_GI + address + "/"; - std::string middle = common::write_varint_sqlite4(from + 1); + std::string middle = common::write_varint_sqlite4(from); for (DB::Cursor cur = m_db.begin(prefix, middle); !cur.end(); cur.next()) { const std::string &suf = cur.get_suffix(); const char *be = suf.data(); const char *en = be + suf.size(); Height he = common::integer_cast(common::read_varint_sqlite4(be, en)); Amount am = common::read_varint_sqlite4(be, en); - uint32_t gi = common::integer_cast(common::read_varint_sqlite4(be, en)); + size_t gi = common::integer_cast(common::read_varint_sqlite4(be, en)); invariant(en - be == 0, ""); - if (he > to) + if (he >= to) break; api::Output output; if (!address.empty()) { @@ -853,7 +844,7 @@ bool WalletStateBasic::for_each_in_unspent_index( invariant(output.address == address, "output is in wrong index by address"); } else seria::from_binary(output, cur.get_value_array()); - if (!fun(output)) + if (!fun(std::move(output))) return false; } return true; @@ -883,7 +874,7 @@ void WalletStateBasic::remove_from_unspent_index(const api::Output &output) { del_with_undo(keyun, true); } -bool WalletStateBasic::read_by_keyimage(const KeyImage &ki, HeightAmounGi *value) const { +bool WalletStateBasic::read_by_keyimage(const crypto::EllipticCurvePoint &ki, HeightAmounGi *value) const { auto keyun = INDEX_KEYIMAGE_to_HE_AM_GI + DB::to_binary_key(ki.data, sizeof(ki.data)); BinaryArray ba; if (!m_db.get(keyun, ba)) @@ -891,9 +882,10 @@ bool WalletStateBasic::read_by_keyimage(const KeyImage &ki, HeightAmounGi *value seria::from_binary(*value, ba); return true; } -void WalletStateBasic::update_keyimage(const KeyImage &ki, const HeightAmounGi &value, bool nooverwrite) { - if (ki == KeyImage{}) - return; +void WalletStateBasic::update_keyimage( + const crypto::EllipticCurvePoint &ki, const HeightAmounGi &value, bool nooverwrite) { + // if (ki == KeyImage{}) + // return; auto keyun = INDEX_KEYIMAGE_to_HE_AM_GI + DB::to_binary_key(ki.data, sizeof(ki.data)); put_with_undo(keyun, seria::to_binary(value), nooverwrite); } diff --git a/src/Core/WalletStateBasic.hpp b/src/Core/WalletStateBasic.hpp index ffec9f07..6b5c61ac 100644 --- a/src/Core/WalletStateBasic.hpp +++ b/src/Core/WalletStateBasic.hpp @@ -14,11 +14,10 @@ #include "BlockChainState.hpp" #include "CryptoNote.hpp" #include "Wallet.hpp" -#include "crypto/chacha8.hpp" #include "platform/DB.hpp" #include "rpc_api.hpp" -namespace bytecoin { +namespace cn { class Config; @@ -28,6 +27,7 @@ class IWalletState { virtual Amount add_incoming_output(const api::Output &, const Hash &tid) = 0; // added amount may be lower virtual Amount add_incoming_keyimage(Height block_height, const KeyImage &) = 0; + virtual Amount add_incoming_deterministic_input(Height block_height, Amount am, size_t gi, const PublicKey &pk) = 0; virtual void add_transaction( Height block_height, const Hash &tid, const TransactionPrefix &tx, const api::Transaction &ptx) = 0; }; @@ -36,19 +36,22 @@ class WalletStateBasic : protected IWalletState { public: typedef platform::DB DB; - explicit WalletStateBasic(logging::ILogger &, const Config &, const Currency &, const std::string &cache_name); + explicit WalletStateBasic( + logging::ILogger &, const Config &, const Currency &, const std::string &cache_name, bool is_det_viewonly); const Currency &get_currency() const { return m_currency; }; Hash get_tip_bid() const { return m_tip.hash; } Height get_tip_height() const { return m_tip_height; } const api::BlockHeader &get_tip() const { return m_tip; } + bool read_chain(Height, api::BlockHeader &) const; + std::vector get_sparse_chain() const; // methods used by API bool api_add_unspent(std::vector *result, Amount *total_amount, const std::string &address, - Height height, Amount max_amount = std::numeric_limits::max()) const; + Height confirmed_height, Amount max_amount = std::numeric_limits::max()) const; std::vector api_get_transfers(const std::string &address, Height *from_height, Height *to_height, - bool forward, uint32_t desired_tx_count = std::numeric_limits::max()) const; + bool forward, size_t desired_tx_count = std::numeric_limits::max()) const; virtual std::vector api_get_locked_or_unconfirmed_unspent( const std::string &address, Height height) const; virtual api::Balance get_balance(const std::string &address, Height height) const; @@ -62,9 +65,9 @@ class WalletStateBasic : protected IWalletState { common::BinaryArray value; }; struct HeightAmounGi { - Height height = 0; - Amount amount = 0; - uint32_t global_index = 0; + Height height = 0; + Amount amount = 0; + size_t global_index = 0; }; void db_commit(); @@ -72,6 +75,8 @@ class WalletStateBasic : protected IWalletState { void test_print_everything(const std::string &str); static void combine_balance(api::Balance &balance, const api::Output &output, int locked_op, int spendable_op); + const Config &get_config() const { return m_config; } + protected: const Hash m_genesis_bid; const Config &m_config; @@ -81,7 +86,6 @@ class WalletStateBasic : protected IWalletState { DB m_db; void push_chain(const api::BlockHeader &); - bool read_chain(Height, api::BlockHeader &) const; void pop_chain(); bool empty_chain() const { return m_tip_height + 1 == m_tail_height; } void reset_chain(Height new_tail_height); @@ -89,41 +93,48 @@ class WalletStateBasic : protected IWalletState { void fix_empty_chain(); // push genesis block api::BlockHeader read_chain(Height) const; - bool is_memory_spent(const api::Output &output) const { return get_used_key_images().count(output.key_image) != 0; } - virtual const std::map &get_used_key_images() const; + bool is_memory_spent(const api::Output &output) const { + return get_mempool_kis_or_pks().count(output.key_image) != 0; + } + virtual const std::map &get_mempool_kis_or_pks() const; virtual void on_first_transaction_found(Timestamp ts) {} void unlock(Height now_height, Timestamp now); bool read_from_unspent_index(const HeightAmounGi &value, api::Output *) const; - bool read_by_keyimage(const KeyImage &m, HeightAmounGi *value) const; + bool read_by_keyimage(const crypto::EllipticCurvePoint &ki, HeightAmounGi *value) const; bool try_add_incoming_output(const api::Output &, Amount *confirmed_balance_delta) const; bool try_adding_incoming_keyimage(const KeyImage &, api::Output *spending_output) const; + bool try_adding_deterministic_input(Amount am, size_t gi, api::Output *spending_output) const; // returns true if our keyimage // methods to add incoming tx Amount add_incoming_output(const api::Output &, const Hash &tid) override; // added amount may be lower Amount add_incoming_keyimage(Height block_height, const KeyImage &) override; + Amount add_incoming_deterministic_input(Height block_height, Amount am, size_t gi, const PublicKey &pk) override; void add_transaction(Height, const Hash &tid, const TransactionPrefix &tx, const api::Transaction &ptx) override; std::string format_output(const api::Output &output); private: + bool try_adding_incoming_ki_or_pk(const crypto::EllipticCurvePoint &, api::Output *spending_output) const; + Amount add_incoming_ki_or_pk(Height block_height, const crypto::EllipticCurvePoint &); + Height m_tip_height = -1; Height m_tail_height = 0; api::BlockHeader m_tip; - void version_3_to_4(); - void put_am_gi_tid(Amount am, uint32_t gi, Hash tid); - Hash get_am_gi_tid(Amount am, uint32_t gi) const; + const bool is_det_viewonly; + void put_am_gi_he(Amount am, size_t gi, Height he, const PublicKey &pk); + bool get_am_gi_he(Amount am, size_t gi, Height *he, PublicKey *pk) const; // DB generic undo machinery typedef std::map UndoMap; UndoMap current_undo_map; UndoMap::iterator record_undo(UndoMap &undo_map, const std::string &key); void put_with_undo(const std::string &key, const common::BinaryArray &value, bool nooverwrite); void del_with_undo(const std::string &key, bool mustexist); - void save_db_state(uint32_t state, const UndoMap &undo_map); - void undo_db_state(uint32_t state); + void save_db_state(Height state, const UndoMap &undo_map); + void undo_db_state(Height state); // indices implemenation Amount add_incoming_output(const api::Output &, const Hash &tid, bool just_unlocked); @@ -133,23 +144,22 @@ class WalletStateBasic : protected IWalletState { void remove_from_lock_index(const api::Output &); void unlock(Height now_height, api::Output &&output); - void add_to_unlocked_index(const api::Output &, Height); - void read_unlock_index(std::map, api::Output> *add, const std::string &index_prefix, - const std::string &address, uint32_t begin, uint32_t end) const; - std::map, api::Output> get_unlocked_outputs( + void read_unlock_index(std::map, api::Output> *add, const std::string &index_prefix, + const std::string &address, BlockOrTimestamp begin, BlockOrTimestamp end) const; + std::map, api::Output> get_unlocked_outputs( const std::string &address, Height from_height, Height to_height) const; // add coin/spend coin void add_to_unspent_index(const api::Output &); void remove_from_unspent_index(const api::Output &); bool for_each_in_unspent_index( - const std::string &address, Height from, Height to, std::function fun) const; - void update_keyimage(const KeyImage &m, const HeightAmounGi &value, bool nooverwrite); + const std::string &address, Height from, Height to, std::function fun) const; + void update_keyimage(const crypto::EllipticCurvePoint &m, const HeightAmounGi &value, bool nooverwrite); }; -} // namespace bytecoin +} // namespace cn namespace seria { -void ser_members(bytecoin::WalletStateBasic::HeightAmounGi &v, ISeria &s); -void ser_members(bytecoin::WalletStateBasic::UndoValue &v, seria::ISeria &s); +void ser_members(cn::WalletStateBasic::HeightAmounGi &v, ISeria &s); +void ser_members(cn::WalletStateBasic::UndoValue &v, seria::ISeria &s); } // namespace seria diff --git a/src/Core/WalletSync.cpp b/src/Core/WalletSync.cpp index f378a247..4c92131b 100644 --- a/src/Core/WalletSync.cpp +++ b/src/Core/WalletSync.cpp @@ -5,6 +5,7 @@ #include "Config.hpp" #include "CryptoNoteTools.hpp" #include "TransactionBuilder.hpp" +#include "platform/PreventSleep.hpp" #include "platform/Time.hpp" #include "seria/BinaryInputStream.hpp" #include "seria/BinaryOutputStream.hpp" @@ -14,7 +15,7 @@ constexpr float STATUS_POLL_PERIOD = 0.1f; constexpr float STATUS_ERROR_PERIOD = 5; -using namespace bytecoin; +using namespace cn; WalletSync::WalletSync( logging::ILogger &log, const Config &config, WalletState &wallet_state, std::function state_changed_handler) @@ -30,42 +31,45 @@ WalletSync::WalletSync( , m_wallet_state(wallet_state) , m_commit_timer(std::bind(&WalletSync::db_commit, this)) { advance_sync(); - m_commit_timer.once(m_config.db_commit_period_wallet_cache); + m_commit_timer.once(float(m_config.db_commit_period_wallet_cache)); } +WalletSync::~WalletSync() {} // we have unique_ptr to inoplete type + void WalletSync::db_commit() { m_wallet_state.db_commit(); - m_commit_timer.once(m_config.db_commit_period_wallet_cache); + m_commit_timer.once(float(m_config.db_commit_period_wallet_cache)); } void WalletSync::send_get_status() { - api::bytecoind::GetStatus::Request req; + api::cnd::GetStatus::Request req; req.top_block_hash = m_wallet_state.get_tip_bid(); req.transaction_pool_version = m_wallet_state.get_tx_pool_version(); req.outgoing_peer_count = m_last_node_status.outgoing_peer_count; req.incoming_peer_count = m_last_node_status.incoming_peer_count; req.lower_level_error = m_last_node_status.lower_level_error; - http::RequestData req_header = - json_rpc::create_request(api::bytecoind::url(), api::bytecoind::GetStatus::method(), req); + http::RequestBody req_header = json_rpc::create_request(api::cnd::url(), api::cnd::GetStatus::method(), req); req_header.r.basic_authorization = m_config.bytecoind_authorization; m_sync_request = std::make_unique(m_sync_agent, std::move(req_header), - [&](http::ResponseData &&response) { + [&](http::ResponseBody &&response) { m_sync_request.reset(); if (response.r.status == 504) { // Common for longpoll advance_sync(); } else if (response.r.status == 401) { m_sync_error = "AUTHORIZATION_FAILED"; - m_log(logging::INFO) << "Wrong daemon password - please check --bytecoind-authorization" << std::endl; + m_state_changed_handler(); + m_log(logging::INFO) << "Wrong daemon password - please check --" CRYPTONOTE_NAME "-authorization" + << std::endl; m_status_timer.once(STATUS_ERROR_PERIOD); } else { - api::bytecoind::GetStatus::Response resp; + api::cnd::GetStatus::Response resp; json_rpc::Error error; if (json_rpc::parse_response(response.body, resp, error)) { m_last_node_status = resp; - m_sync_error = std::string(); - m_state_changed_handler(); + // m_sync_error = std::string(); + // m_state_changed_handler(); advance_sync(); } else { m_log(logging::INFO) << "GetStatus request RPC error code=" << error.code @@ -73,12 +77,12 @@ void WalletSync::send_get_status() { m_status_timer.once(STATUS_ERROR_PERIOD); } } - }, + }, [&](std::string err) { m_sync_error = "CONNECTION_FAILED"; - m_status_timer.once(STATUS_ERROR_PERIOD); m_state_changed_handler(); - }); + m_status_timer.once(STATUS_ERROR_PERIOD); + }); } void WalletSync::advance_sync() { @@ -95,13 +99,15 @@ void WalletSync::advance_sync() { if (m_sync_request) return; if (m_last_node_status.top_block_hash != m_wallet_state.get_tip_bid()) { - next_send_hash = Hash{}; // We start sending again after new block + m_next_send_hash = Hash{}; // We start sending again after new block send_get_blocks(); return; } if (send_send_transaction()) return; if (m_last_node_status.transaction_pool_version == m_wallet_state.get_tx_pool_version()) { + m_sync_error = std::string(); + m_state_changed_handler(); m_status_timer.once(STATUS_POLL_PERIOD); return; } @@ -110,34 +116,40 @@ void WalletSync::advance_sync() { void WalletSync::send_sync_pool() { m_log(logging::TRACE) << "Sending SyncMemPool request" << std::endl; - api::bytecoind::SyncMemPool::Request msg; - msg.known_hashes = m_wallet_state.get_tx_pool_hashes(); - http::RequestData req_header; - req_header.r.set_firstline("POST", api::bytecoind::binary_url(), 1, 1); + api::cnd::SyncMemPool::Request msg; + msg.known_hashes = m_wallet_state.get_tx_pool_hashes(); + msg.need_signatures = m_wallet_state.get_wallet().is_det_viewonly(); + http::RequestBody req_header; + req_header.r.set_firstline("POST", api::cnd::binary_url(), 1, 1); req_header.r.basic_authorization = m_config.bytecoind_authorization; - req_header.set_body(json_rpc::create_binary_request_body(api::bytecoind::SyncMemPool::method(), msg)); + req_header.set_body(json_rpc::create_binary_request_body(api::cnd::SyncMemPool::bin_method(), msg)); m_sync_request = std::make_unique(m_sync_agent, std::move(req_header), - [&](http::ResponseData &&response) { + [&](http::ResponseBody &&response) { m_sync_request.reset(); m_log(logging::TRACE) << "Received SyncMemPool response status=" << response.r.status << std::endl; if (response.r.status == 401) { m_sync_error = "AUTHORIZATION_FAILED"; - m_log(logging::INFO) << "Wrong daemon password - please check --bytecoind-authorization" << std::endl; + m_log(logging::INFO) << "Wrong daemon password - please check --" CRYPTONOTE_NAME "d-authorization" + << std::endl; m_status_timer.once(STATUS_ERROR_PERIOD); } else if (response.r.status == 200) { - m_sync_error = "WRONG_BLOCKCHAIN"; - api::bytecoind::SyncMemPool::Response resp; + // m_sync_error = "WRONG_BLOCKCHAIN"; + api::cnd::SyncMemPool::Response resp; json_rpc::Error error; if (json_rpc::parse_binary_response(response.body, resp, error)) { m_last_node_status = resp.status; if (m_wallet_state.sync_with_blockchain(resp)) { m_sync_error = std::string(); advance_sync(); - } else + } else { + m_sync_error = "INCOMPATIBLE_DAEMON_VERSION"; m_status_timer.once(STATUS_ERROR_PERIOD); + } } else { m_log(logging::INFO) << "SyncMemPool request RPC error code=" << error.code << " message=" << error.message << std::endl; + if (error.code == json_rpc::METHOD_NOT_FOUND) + m_sync_error = "INCOMPATIBLE_DAEMON_VERSION"; m_status_timer.once(STATUS_ERROR_PERIOD); } } else { @@ -145,48 +157,56 @@ void WalletSync::send_sync_pool() { m_status_timer.once(STATUS_ERROR_PERIOD); } m_state_changed_handler(); - }, + }, [&](std::string err) { m_log(logging::TRACE) << "SyncMemPool request error " << err << std::endl; m_sync_error = "CONNECTION_FAILED"; m_status_timer.once(STATUS_ERROR_PERIOD); m_state_changed_handler(); - }); + }); // m_log(logging::INFO) << "WalletNode::send_sync_pool" << std::endl; } void WalletSync::send_get_blocks() { m_log(logging::TRACE) << "Sending SyncBlocks request" << std::endl; - api::bytecoind::SyncBlocks::Request msg; - msg.sparse_chain = m_wallet_state.get_sparse_chain(); - msg.first_block_timestamp = m_wallet_state.get_wallet().get_oldest_timestamp(); - msg.need_redundant_data = false; - http::RequestData req_header; - req_header.r.set_firstline("POST", api::bytecoind::binary_url(), 1, 1); + api::cnd::SyncBlocks::Request msg; + msg.sparse_chain = m_wallet_state.get_sparse_chain(); + msg.first_block_timestamp = + (m_wallet_state.get_wallet().get_oldest_timestamp() / m_config.wallet_sync_timestamp_granularity) * + m_config.wallet_sync_timestamp_granularity; + msg.need_redundant_data = false; + msg.need_signatures = m_wallet_state.get_wallet().is_det_viewonly(); + http::RequestBody req_header; + req_header.r.set_firstline("POST", api::cnd::binary_url(), 1, 1); req_header.r.basic_authorization = m_config.bytecoind_authorization; - req_header.set_body(json_rpc::create_binary_request_body(api::bytecoind::SyncBlocks::method(), msg)); + req_header.set_body(json_rpc::create_binary_request_body(api::cnd::SyncBlocks::bin_method(), msg)); m_sync_request = std::make_unique(m_sync_agent, std::move(req_header), - [&](http::ResponseData &&response) { + [&](http::ResponseBody &&response) { m_sync_request.reset(); m_log(logging::TRACE) << "Received SyncBlocks response status=" << response.r.status << std::endl; if (response.r.status == 401) { m_sync_error = "AUTHORIZATION_FAILED"; - m_log(logging::INFO) << "Wrong daemon password - please check --bytecoind-authorization" << std::endl; + m_log(logging::INFO) << "Wrong daemon password - please check --" CRYPTONOTE_NAME "d-authorization" + << std::endl; m_status_timer.once(STATUS_ERROR_PERIOD); } else if (response.r.status == 200) { - m_sync_error = "WRONG_BLOCKCHAIN"; - api::bytecoind::SyncBlocks::Response resp; + // m_sync_error = "WRONG_BLOCKCHAIN"; + api::cnd::SyncBlocks::Response resp; json_rpc::Error error; if (json_rpc::parse_binary_response(response.body, resp, error)) { m_last_node_status = resp.status; if (m_wallet_state.sync_with_blockchain(resp)) { m_sync_error = std::string(); advance_sync(); - } else + } else { + m_sync_error = "INCOMPATIBLE_DAEMON_VERSION"; m_status_timer.once(STATUS_ERROR_PERIOD); + } } else { m_log(logging::INFO) << "SyncBlocks request RPC error code=" << error.code << " message=" << error.message << std::endl; + if (error.code == json_rpc::METHOD_NOT_FOUND) + m_sync_error = "INCOMPATIBLE_DAEMON_VERSION"; m_status_timer.once(STATUS_ERROR_PERIOD); } } else { @@ -194,61 +214,61 @@ void WalletSync::send_get_blocks() { m_status_timer.once(STATUS_ERROR_PERIOD); } m_state_changed_handler(); - }, + }, [&](std::string err) { m_log(logging::TRACE) << "SyncBlocks request error " << err << std::endl; m_sync_error = "CONNECTION_FAILED"; m_status_timer.once(STATUS_ERROR_PERIOD); m_state_changed_handler(); - }); + }); // m_log(logging::INFO) << "WalletNode::send_get_blocks" << std::endl; } bool WalletSync::send_send_transaction() { - api::bytecoind::SendTransaction::Request msg; - msg.binary_transaction = m_wallet_state.get_next_from_sending_queue(&next_send_hash); + api::cnd::SendTransaction::Request msg; + msg.binary_transaction = m_wallet_state.get_next_from_sending_queue(&m_next_send_hash); if (msg.binary_transaction.empty()) return false; - sending_transaction_hash = next_send_hash; - m_log(logging::INFO) << "Sending transaction from payment queue " << sending_transaction_hash << std::endl; - http::RequestData new_request = - json_rpc::create_request(api::bytecoind::url(), api::bytecoind::SendTransaction::method(), msg); + m_sending_transaction_hash = m_next_send_hash; + m_log(logging::INFO) << "Sending transaction from payment queue " << m_sending_transaction_hash << std::endl; + http::RequestBody new_request = json_rpc::create_request(api::cnd::url(), api::cnd::SendTransaction::method(), msg); new_request.r.basic_authorization = m_config.bytecoind_authorization; m_sync_request = std::make_unique(m_sync_agent, std::move(new_request), - [&](http::ResponseData &&response) { - m_sync_request.reset(); - m_log(logging::TRACE) << "Received send_transaction response status=" << response.r.status << std::endl; - if (response.r.status == 401) { - m_sync_error = "AUTHORIZATION_FAILED"; - m_log(logging::INFO) << "Wrong daemon password - please check --bytecoind-authorization" << std::endl; - m_status_timer.once(STATUS_ERROR_PERIOD); - } else if (response.r.status == 200) { - m_sync_error = "SEND_ERROR"; - api::bytecoind::SendTransaction::Response resp; - api::bytecoind::SendTransaction::Error error; - if (json_rpc::parse_response(response.body, resp, error)) { - m_log(logging::INFO) << "Success sending transaction from payment queue with result " - << resp.send_result << std::endl; - m_sync_error = std::string(); - } else { - m_log(logging::INFO) << "Json Error sending transaction from payment queue conflict height=" - << error.conflict_height << " code=" << error.code << " msg=" << error.message - << std::endl; - m_wallet_state.process_payment_queue_send_error(sending_transaction_hash, error); - } - advance_sync(); - } else { - m_log(logging::INFO) << "Error sending transaction from payment queue " << response.body << std::endl; - m_sync_error = response.body; - m_status_timer.once(STATUS_ERROR_PERIOD); - } - m_state_changed_handler(); - }, - [&](std::string err) { - m_log(logging::INFO) << "Error sending transaction from payment queue " << err << std::endl; - m_status_timer.once(STATUS_ERROR_PERIOD); - m_state_changed_handler(); - }); + [&](http::ResponseBody &&response) { + m_sync_request.reset(); + m_log(logging::TRACE) << "Received send_transaction response status=" << response.r.status << std::endl; + if (response.r.status == 401) { + m_sync_error = "AUTHORIZATION_FAILED"; + m_log(logging::INFO) << "Wrong daemon password - please check --" CRYPTONOTE_NAME "d-authorization" + << std::endl; + m_status_timer.once(STATUS_ERROR_PERIOD); + } else if (response.r.status == 200) { + api::cnd::SendTransaction::Response resp; + api::cnd::SendTransaction::Error error; + if (json_rpc::parse_response(response.body, resp, error)) { + m_log(logging::INFO) << "Success sending transaction from payment queue with result " + << resp.send_result << std::endl; + m_sync_error = std::string(); + } else { + m_log(logging::INFO) << "Json Error sending transaction from payment queue conflict height=" + << error.conflict_height << " code=" << error.code << " msg=" << error.message + << std::endl; + m_sync_error = "SEND_ERROR"; + m_wallet_state.process_payment_queue_send_error(m_sending_transaction_hash, error); + } + advance_sync(); + } else { + m_log(logging::INFO) << "Error sending transaction from payment queue " << response.body << std::endl; + m_sync_error = response.body; + m_status_timer.once(STATUS_ERROR_PERIOD); + } + m_state_changed_handler(); + }, + [&](std::string err) { + m_log(logging::INFO) << "Error sending transaction from payment queue " << err << std::endl; + m_status_timer.once(STATUS_ERROR_PERIOD); + m_state_changed_handler(); + }); // m_log(logging::INFO) << "WalletNode::send_get_blocks" << std::endl; return true; } diff --git a/src/Core/WalletSync.hpp b/src/Core/WalletSync.hpp index 25b2d7fe..a66b2618 100644 --- a/src/Core/WalletSync.hpp +++ b/src/Core/WalletSync.hpp @@ -9,13 +9,15 @@ #include "http/Agent.hpp" #include "http/JsonRpc.hpp" -namespace bytecoin { +namespace cn { + +class WalletState; class WalletSync { public: explicit WalletSync(logging::ILogger &, const Config &, WalletState &, std::function state_changed_handler); - - const api::bytecoind::GetStatus::Response &get_last_node_status() const { return m_last_node_status; } + ~WalletSync(); + const api::cnd::GetStatus::Response &get_last_node_status() const { return m_last_node_status; } std::string get_sync_error() const { return m_sync_error; } protected: @@ -23,7 +25,7 @@ class WalletSync { logging::LoggerRef m_log; const Config &m_config; - api::bytecoind::GetStatus::Response m_last_node_status; + api::cnd::GetStatus::Response m_last_node_status; std::string m_sync_error; platform::Timer m_status_timer; http::Agent m_sync_agent; @@ -38,8 +40,8 @@ class WalletSync { std::unique_ptr prevent_sleep; platform::Timer m_commit_timer; - Hash next_send_hash; - Hash sending_transaction_hash; + Hash m_next_send_hash; + Hash m_sending_transaction_hash; void db_commit(); void send_get_status(); @@ -48,4 +50,4 @@ class WalletSync { void send_get_blocks(); }; -} // namespace bytecoin +} // namespace cn diff --git a/src/CryptoNote.cpp b/src/CryptoNote.cpp index 27f60850..edb58d8b 100644 --- a/src/CryptoNote.cpp +++ b/src/CryptoNote.cpp @@ -4,17 +4,19 @@ #include "CryptoNote.hpp" #include "Core/CryptoNoteTools.hpp" #include "Core/TransactionExtra.hpp" +#include "CryptoNoteConfig.hpp" // We access TRANSACTION_VERSION_AMETHYST directly #include "rpc_api.hpp" #include "seria/JsonOutputStream.hpp" // includes below are for proof seria #include "Core/Currency.hpp" #include "common/Base58.hpp" #include "common/Varint.hpp" +#include "seria/BinaryInputStream.hpp" +#include "seria/BinaryOutputStream.hpp" -using namespace bytecoin; +using namespace cn; namespace seria { -enum class SerializationTag2 : uint8_t { Base = 0xff, Key = 0x2 }; void ser(Hash &v, ISeria &s) { s.binary(v.data, sizeof(v.data)); } void ser(KeyImage &v, ISeria &s) { s.binary(v.data, sizeof(v.data)); } @@ -22,147 +24,414 @@ void ser(PublicKey &v, ISeria &s) { s.binary(v.data, sizeof(v.data)); } void ser(SecretKey &v, ISeria &s) { s.binary(v.data, sizeof(v.data)); } void ser(KeyDerivation &v, ISeria &s) { s.binary(v.data, sizeof(v.data)); } void ser(Signature &v, ISeria &s) { s.binary(reinterpret_cast(&v), sizeof(Signature)); } -void ser_members(AccountPublicAddress &v, ISeria &s) { +void ser(crypto::EllipticCurveScalar &v, ISeria &s) { s.binary(v.data, sizeof(v.data)); } + +void ser_members(cn::AccountAddressSimple &v, ISeria &s) { seria_kv("spend", v.spend_public_key, s); seria_kv("view", v.view_public_key, s); } -void ser_members(SendProof &v, ISeria &s, const Currency ¤cy) { +void ser_members(cn::AccountAddressUnlinkable &v, ISeria &s) { + seria_kv("s", v.s, s); + seria_kv("sv", v.sv, s); +} +void ser_members(AccountAddress &v, ISeria &s) { + if (s.is_input()) { + uint8_t type = 0; + s.object_key("type"); + if (dynamic_cast(&s)) { + std::string str_type_tag; + ser(str_type_tag, s); + if (str_type_tag == AccountAddressSimple::str_type_tag()) + type = AccountAddressSimple::type_tag; + else if (str_type_tag == AccountAddressUnlinkable::str_type_tag()) + type = AccountAddressUnlinkable::type_tag; + else if (str_type_tag == AccountAddressUnlinkable::str_type_tag_auditable()) + type = AccountAddressUnlinkable::type_tag_auditable; + else + throw std::runtime_error("Deserialization error - unknown address type " + str_type_tag); + } else + s.binary(&type, 1); + switch (type) { + case AccountAddressSimple::type_tag: { + AccountAddressSimple in{}; + ser_members(in, s); + v = in; + break; + } + case AccountAddressUnlinkable::type_tag: + case AccountAddressUnlinkable::type_tag_auditable: { + AccountAddressUnlinkable in{}; + in.is_auditable = (type == AccountAddressUnlinkable::type_tag_auditable); + ser_members(in, s); + v = in; + break; + } + default: + throw std::runtime_error("Deserialization error - unknown address type " + common::to_string(type)); + } + return; + } + if (v.type() == typeid(AccountAddressSimple)) { + auto &in = boost::get(v); + s.object_key("type"); + if (dynamic_cast(&s)) { + std::string str_type_tag = AccountAddressSimple::str_type_tag(); + ser(str_type_tag, s); + } else { + uint8_t type = AccountAddressSimple::type_tag; + s.binary(&type, 1); + } + ser_members(in, s); + } else if (v.type() == typeid(AccountAddressUnlinkable)) { + auto &in = boost::get(v); + s.object_key("type"); + if (dynamic_cast(&s)) { + std::string str_type_tag = in.is_auditable ? AccountAddressUnlinkable::str_type_tag_auditable() + : AccountAddressUnlinkable::str_type_tag(); + ser(str_type_tag, s); + } else { + uint8_t type = + in.is_auditable ? AccountAddressUnlinkable::type_tag_auditable : AccountAddressUnlinkable::type_tag; + s.binary(&type, 1); + } + ser_members(in, s); + } +} +void ser_members(cn::SendproofKey &v, ISeria &s) { + seria_kv("derivation", v.derivation, s); + seria_kv("signature", v.signature, s); +} +void ser_members(cn::SendproofUnlinkable::Element &v, ISeria &s) { + seria_kv("out_index", v.out_index, s); + seria_kv("q", v.q, s); + seria_kv("signature", v.signature, s); +} +void ser_members(cn::SendproofUnlinkable &v, ISeria &s) { seria_kv("elements", v.elements, s); } +void ser_members(Sendproof &v, ISeria &s, const Currency ¤cy) { std::string addr; if (!s.is_input()) addr = currency.account_address_as_string(v.address); seria_kv("address", addr, s); if (s.is_input() && (!currency.parse_account_address_string(addr, &v.address))) throw api::ErrorAddress(api::ErrorAddress::ADDRESS_FAILED_TO_PARSE, "Failed to parse wallet address", addr); + seria_kv("transaction_hash", v.transaction_hash, s); + seria_kv("message", v.message, s); + seria_kv("amount", v.amount, s); std::string proof; BinaryArray binary_proof; if (!s.is_input()) { - common::append(binary_proof, std::begin(v.derivation.data), std::end(v.derivation.data)); - common::append(binary_proof, std::begin(v.signature.c.data), std::end(v.signature.c.data)); - common::append(binary_proof, std::begin(v.signature.r.data), std::end(v.signature.r.data)); + if (v.address.type() == typeid(AccountAddressSimple)) { + auto &var = boost::get(v.proof); + binary_proof = seria::to_binary(var); + } else if (v.address.type() == typeid(AccountAddressUnlinkable)) { + auto &var = boost::get(v.proof); + binary_proof = seria::to_binary(var); + } else + throw api::ErrorAddress(api::ErrorAddress::ADDRESS_FAILED_TO_PARSE, "Unknown address type", addr); proof = common::base58::encode(binary_proof); } seria_kv("proof", proof, s); if (s.is_input()) { - if (!common::base58::decode(proof, &binary_proof) || - binary_proof.size() != sizeof(v.derivation.data) + sizeof(v.signature.c) + sizeof(v.signature.r)) + if (!common::base58::decode(proof, &binary_proof)) throw std::runtime_error("Wrong proof format - " + proof); - memmove(v.derivation.data, binary_proof.data(), sizeof(v.derivation)); - memmove(v.signature.c.data, binary_proof.data() + sizeof(v.derivation), sizeof(v.signature.c)); - memmove(v.signature.r.data, - binary_proof.data() + sizeof(v.derivation) + sizeof(v.signature.c), - sizeof(v.signature.r)); + if (v.address.type() == typeid(AccountAddressSimple)) { + SendproofKey sp; + seria::from_binary(sp, binary_proof); + v.proof = sp; + } else if (v.address.type() == typeid(AccountAddressUnlinkable)) { + SendproofUnlinkable sp; + seria::from_binary(sp, binary_proof); + v.proof = sp; + } else + throw api::ErrorAddress(api::ErrorAddress::ADDRESS_FAILED_TO_PARSE, "Unknown address type", addr); } - seria_kv("transaction_hash", v.transaction_hash, s); - seria_kv("message", v.message, s); - seria_kv("amount", v.amount, s); } void ser_members(TransactionInput &v, ISeria &s) { if (s.is_input()) { - uint8_t tag = 0; + uint8_t type = 0; s.object_key("type"); if (dynamic_cast(&s)) { - std::string str_tag; - ser(str_tag, s); - if (str_tag == "coinbase") - tag = (uint8_t)SerializationTag2::Base; - else if (str_tag == "key") - tag = (uint8_t)SerializationTag2::Key; + std::string str_type_tag; + ser(str_type_tag, s); + if (str_type_tag == InputCoinbase::str_type_tag()) + type = InputCoinbase::type_tag; + else if (str_type_tag == InputKey::str_type_tag()) + type = InputKey::type_tag; + else + throw std::runtime_error("Deserialization error - unknown input type " + str_type_tag); } else - s.binary(&tag, 1); - switch ((SerializationTag2)tag) { - case SerializationTag2::Base: { - CoinbaseInput in{}; + s.binary(&type, 1); + switch (type) { + case InputCoinbase::type_tag: { + InputCoinbase in{}; ser_members(in, s); v = in; break; } - case SerializationTag2::Key: { - KeyInput in{}; + case InputKey::type_tag: { + InputKey in{}; ser_members(in, s); v = in; break; } default: - throw std::runtime_error("Deserialization error - unknown input tag"); + throw std::runtime_error("Deserialization error - unknown input type " + common::to_string(type)); } return; } - if (v.type() == typeid(CoinbaseInput)) { - CoinbaseInput &in = boost::get(v); - uint8_t tag = (uint8_t)SerializationTag2::Base; + if (v.type() == typeid(InputCoinbase)) { + auto &in = boost::get(v); + uint8_t type = InputCoinbase::type_tag; s.object_key("type"); if (dynamic_cast(&s)) { - std::string str_tag = "coinbase"; - ser(str_tag, s); + std::string str_type_tag = InputCoinbase::str_type_tag(); + ser(str_type_tag, s); } else - s.binary(&tag, 1); + s.binary(&type, 1); ser_members(in, s); - } else if (v.type() == typeid(KeyInput)) { - KeyInput &in = boost::get(v); - uint8_t tag = (uint8_t)SerializationTag2::Key; + } else if (v.type() == typeid(InputKey)) { + auto &in = boost::get(v); + uint8_t type = InputKey::type_tag; s.object_key("type"); if (dynamic_cast(&s)) { - std::string str_tag = "key"; - ser(str_tag, s); + std::string str_type_tag = InputKey::str_type_tag(); + ser(str_type_tag, s); } else - s.binary(&tag, 1); + s.binary(&type, 1); ser_members(in, s); } } -void ser_members(TransactionOutputTarget &v, ISeria &s) { +void ser_members(TransactionOutput &v, ISeria &s, bool is_tx_amethyst) { if (s.is_input()) { - uint8_t tag = 0; + Amount amount = 0; + if (!is_tx_amethyst) + seria_kv("amount", amount, s); + uint8_t type = 0; s.object_key("type"); if (dynamic_cast(&s)) { - std::string str_tag; - ser(str_tag, s); - if (str_tag == "key") - tag = (uint8_t)SerializationTag2::Key; + std::string str_type_tag; + ser(str_type_tag, s); + if (str_type_tag == OutputKey::str_type_tag()) + type = OutputKey::type_tag; + else if (str_type_tag == OutputKey::str_type_tag_auditable()) + type = OutputKey::type_tag_auditable; + else + throw std::runtime_error("Deserialization error - unknown output type " + str_type_tag); } else - s.binary(&tag, 1); - switch ((SerializationTag2)tag) { - case SerializationTag2::Key: { - KeyOutput in{}; - ser_members(in, s); + s.binary(&type, 1); + switch (type) { + case OutputKey::type_tag: + case OutputKey::type_tag_auditable: { + OutputKey in{}; + in.amount = amount; + in.is_auditable = (type == OutputKey::type_tag_auditable); + ser_members(in, s, is_tx_amethyst); v = in; break; } default: - throw std::runtime_error("Deserialization error - unknown output tag"); + throw std::runtime_error("Deserialization error - unknown output type " + common::to_string(type)); } return; } - if (v.type() == typeid(KeyOutput)) { - KeyOutput &in = boost::get(v); - uint8_t tag = (uint8_t)SerializationTag2::Key; + if (v.type() == typeid(OutputKey)) { + auto &in = boost::get(v); + if (!is_tx_amethyst) + seria_kv("amount", in.amount, s); s.object_key("type"); if (dynamic_cast(&s)) { - std::string str_tag = "key"; - ser(str_tag, s); - } else - s.binary(&tag, 1); - ser_members(in, s); + std::string str_type_tag = + in.is_auditable ? OutputKey::str_type_tag_auditable() : OutputKey::str_type_tag(); + ser(str_type_tag, s); + } else { + uint8_t type = in.is_auditable ? OutputKey::type_tag_auditable : OutputKey::type_tag; + s.binary(&type, 1); + } + ser_members(in, s, is_tx_amethyst); } } -void ser_members(TransactionOutput &v, ISeria &s) { - seria_kv("amount", v.amount, s); - ser_members(v.target, s); - // seria_kv("target", v.target, s); -} -void ser_members(CoinbaseInput &v, ISeria &s) { seria_kv("height", v.height, s); } -void ser_members(KeyInput &v, ISeria &s) { +void ser_members(InputCoinbase &v, ISeria &s) { seria_kv("height", v.height, s); } +void ser_members(InputKey &v, ISeria &s) { seria_kv("amount", v.amount, s); seria_kv("output_indexes", v.output_indexes, s); seria_kv("key_image", v.key_image, s); } +void ser_members(cn::RingSignatures &v, ISeria &s) { seria_kv("signatures", v.signatures, s); } +void ser_members(cn::RingSignature3 &v, ISeria &s) { + seria_kv("c0", v.c0, s); + seria_kv("r", v.r, s); +} + +enum { ring_signature_blank_tag = 0xff, ring_signature_normal_tag = 1, ring_signature_halfsize_tag = 2 }; +const char ring_signature_blank_tag_str[] = "blank"; +const char ring_signature_normal_tag_str[] = "normal"; +const char ring_signature_halfsize_tag_str[] = "halfsize"; + +// Usually signatures are parsed in the context of transaction where type is known +// but sometimes we need to send just signatures, so we need to seria them independently + +void ser_members(cn::TransactionSignatures &v, ISeria &s) { + if (s.is_input()) { + uint8_t type = 0; + s.object_key("type"); + if (dynamic_cast(&s)) { + std::string str_type_tag; + ser(str_type_tag, s); + if (str_type_tag == ring_signature_blank_tag_str) + type = ring_signature_blank_tag; + else if (str_type_tag == ring_signature_normal_tag_str) + type = ring_signature_normal_tag; + else if (str_type_tag == ring_signature_halfsize_tag_str) + type = ring_signature_halfsize_tag; + else + throw std::runtime_error("Deserialization error - unknown ring signature type " + str_type_tag); + } else + s.binary(&type, 1); + switch (type) { + case ring_signature_blank_tag: { + v = boost::blank{}; + break; + } + case ring_signature_normal_tag: { + RingSignatures ring_sig; + ser_members(ring_sig, s); + v = ring_sig; + break; + } + case ring_signature_halfsize_tag: { + RingSignature3 ring_sig; + ser_members(ring_sig, s); + v = ring_sig; + break; + } + default: + throw std::runtime_error("Deserialization error - unknown ring signature type " + common::to_string(type)); + } + return; + } + if (v.type() == typeid(boost::blank)) { + s.object_key("type"); + if (dynamic_cast(&s)) { + std::string str_type_tag = ring_signature_blank_tag_str; + ser(str_type_tag, s); + } else { + uint8_t type = ring_signature_blank_tag; + s.binary(&type, 1); + } + } else if (v.type() == typeid(RingSignatures)) { + auto &in = boost::get(v); + s.object_key("type"); + if (dynamic_cast(&s)) { + std::string str_type_tag = ring_signature_normal_tag_str; + ser(str_type_tag, s); + } else { + uint8_t type = ring_signature_normal_tag; + s.binary(&type, 1); + } + ser_members(in, s); + } else if (v.type() == typeid(RingSignature3)) { + auto &in = boost::get(v); + s.object_key("type"); + if (dynamic_cast(&s)) { + std::string str_type_tag = ring_signature_halfsize_tag_str; + ser(str_type_tag, s); + } else { + uint8_t type = ring_signature_halfsize_tag; + s.binary(&type, 1); + } + ser_members(in, s); + } else + throw std::runtime_error("Serialization error - unknown ring signature type"); +} -void ser_members(KeyOutput &v, ISeria &s) { seria_kv("public_key", v.public_key, s); } +// Serializing in the context of transaction - sizes and types are known from transaction prefix +void ser_members(cn::TransactionSignatures &v, ISeria &s, const TransactionPrefix &prefix) { + const bool is_base = (prefix.inputs.size() == 1) && (prefix.inputs[0].type() == typeid(InputCoinbase)); + if (is_base) + return; // No signatures in base transaction + size_t sig_size = prefix.inputs.size(); + const bool is_tx_amethyst = (prefix.version >= parameters::TRANSACTION_VERSION_AMETHYST); + + s.object_key("signatures"); + if (is_tx_amethyst) { + s.begin_object(); + if (s.is_input()) + v = RingSignature3{}; + auto &sigs = boost::get(v); + if (s.is_input()) + sigs.r.resize(sig_size); + s.object_key("c0"); + ser(sigs.c0, s); + s.object_key("r"); + s.begin_array(sig_size, true); + for (size_t i = 0; i < sig_size; ++i) { + invariant(prefix.inputs[i].type() == typeid(InputKey), + "Serialization error: input type wrong for transaction version"); + size_t signature_size = boost::get(prefix.inputs[i]).output_indexes.size(); + if (s.is_input()) { + sigs.r[i].resize(signature_size); + s.begin_array(signature_size, true); + for (crypto::EllipticCurveScalar &sig : sigs.r[i]) { + ser(sig, s); + } + s.end_array(); + } else { + invariant(signature_size == sigs.r[i].size(), "Serialization error: unexpected signatures size"); + s.begin_array(signature_size, true); + for (crypto::EllipticCurveScalar &sig : sigs.r[i]) { + ser(sig, s); + } + s.end_array(); + } + } + s.end_array(); + s.end_object(); + } else { + s.begin_array(sig_size, true); + if (s.is_input()) + v = RingSignatures{}; + auto &sigs = boost::get(v); + if (s.is_input()) + sigs.signatures.resize(sig_size); + for (size_t i = 0; i < sig_size; ++i) { + invariant(prefix.inputs[i].type() == typeid(InputKey), + "Serialization error: input type wrong for transaction version"); + size_t signature_size = boost::get(prefix.inputs[i]).output_indexes.size(); + if (s.is_input()) { + sigs.signatures[i].resize(signature_size); + s.begin_array(signature_size, true); + for (crypto::Signature &sig : sigs.signatures[i]) { + ser(sig, s); + } + s.end_array(); + } else { + invariant( + signature_size == sigs.signatures[i].size(), "Serialization error: unexpected signatures size"); + s.begin_array(signature_size, true); + for (crypto::Signature &sig : sigs.signatures[i]) { + ser(sig, s); + } + s.end_array(); + } + } + s.end_array(); + } +} + +void ser_members(OutputKey &v, ISeria &s, bool is_tx_amethyst) { + if (is_tx_amethyst) // We moved amount inside variant part in amethyst + seria_kv("amount", v.amount, s); + seria_kv("public_key", v.public_key, s); + if (is_tx_amethyst) + seria_kv("encrypted_secret", v.encrypted_secret, s); +} void ser_members(TransactionPrefix &v, ISeria &s) { seria_kv("version", v.version, s); + const bool is_tx_amethyst = (v.version >= parameters::TRANSACTION_VERSION_AMETHYST); seria_kv("unlock_block_or_timestamp", v.unlock_block_or_timestamp, s); seria_kv("inputs", v.inputs, s); - seria_kv("outputs", v.outputs, s); + seria_kv("outputs", v.outputs, s, is_tx_amethyst); seria_kv("extra", v.extra, s); } void ser_members(BaseTransaction &v, ISeria &s) { @@ -173,67 +442,32 @@ void ser_members(BaseTransaction &v, ISeria &s) { } } -static size_t get_signatures_count(const TransactionInput &input) { - struct txin_signature_size_visitor : public boost::static_visitor { - size_t operator()(const CoinbaseInput &) const { return 0; } - size_t operator()(const KeyInput &txin) const { return txin.output_indexes.size(); } - }; - return boost::apply_visitor(txin_signature_size_visitor(), input); -} +// static size_t get_signatures_count(const TransactionInput &input) { +// struct txin_signature_size_visitor : public boost::static_visitor { +// size_t operator()(const InputCoinbase &) const { return 0; } +// size_t operator()(const InputKey &txin) const { return txin.output_indexes.size(); } +// }; +// return boost::apply_visitor(txin_signature_size_visitor(), input); +//} void ser_members(Transaction &v, ISeria &s) { ser_members(static_cast(v), s); - - bool is_base = (v.inputs.size() == 1) && (v.inputs[0].type() == typeid(CoinbaseInput)); - size_t sig_size = is_base ? 0 : v.inputs.size(); - - if (s.is_input()) - v.signatures.resize(sig_size); - - if (sig_size && v.inputs.size() != v.signatures.size()) - throw std::runtime_error("Serialization error: unexpected signatures size"); - - s.object_key("signatures"); - s.begin_array(sig_size, true); - for (size_t i = 0; i < sig_size; ++i) { - size_t signature_size = get_signatures_count(v.inputs[i]); - if (!s.is_input()) { - if (signature_size != v.signatures[i].size()) - throw std::runtime_error("Serialization error: unexpected signatures size"); - s.begin_array(signature_size, true); - for (crypto::Signature &sig : v.signatures[i]) { - ser(sig, s); - } - s.end_array(); - } else { - std::vector signatures(signature_size); - s.begin_array(signature_size, true); - for (crypto::Signature &sig : signatures) { - ser(sig, s); - } - s.end_array(); - v.signatures[i] = std::move(signatures); - } - } - s.end_array(); + ser_members(v.signatures, s, static_cast(v)); } -void ser_members(ParentBlock &v, ISeria &s, BlockSeriaType seria_type) { + +void ser_members(RootBlock &v, ISeria &s, BlockSeriaType seria_type) { seria_kv("major_version", v.major_version, s); seria_kv("minor_version", v.minor_version, s); seria_kv("timestamp", v.timestamp, s); seria_kv("previous_block_hash", v.previous_block_hash, s); - unsigned char nonce_data[4]; - common::uint_le_to_bytes(nonce_data, 4, v.nonce); s.object_key("nonce"); - s.binary(nonce_data, 4); - if (s.is_input()) - v.nonce = common::uint_le_from_bytes(nonce_data, 4); + s.binary(v.nonce, 4); if (seria_type == BlockSeriaType::BLOCKHASH || seria_type == BlockSeriaType::LONG_BLOCKHASH) { - Hash miner_tx_hash = get_base_transaction_hash(v.base_transaction); + Hash miner_tx_hash = get_root_block_base_transaction_hash(v.base_transaction); Hash merkle_root = crypto::tree_hash_from_branch( - v.base_transaction_branch.data(), v.base_transaction_branch.size(), miner_tx_hash, nullptr); + v.base_transaction_branch.data(), v.base_transaction_branch.size(), miner_tx_hash, nullptr); seria_kv("merkle_root", merkle_root, s); } @@ -244,7 +478,7 @@ void ser_members(ParentBlock &v, ISeria &s, BlockSeriaType seria_type) { if (seria_type == BlockSeriaType::LONG_BLOCKHASH) return; - size_t branch_size = crypto::coinbase_tree_depth(v.transaction_count); + size_t branch_size = crypto_coinbase_tree_depth(v.transaction_count); if (!s.is_input()) { if (v.base_transaction_branch.size() != branch_size) throw std::runtime_error("Wrong miner transaction branch size"); @@ -283,90 +517,140 @@ void ser_members(ParentBlock &v, ISeria &s, BlockSeriaType seria_type) { } s.end_array(); } -void ser_members(BlockHeader &v, ISeria &s, BlockSeriaType seria_type, BlockBodyProxy body_proxy) { - if (v.major_version == 1 || seria_type != BlockSeriaType::LONG_BLOCKHASH) { +/* + We commented code below (for now) which uses bitmask to save space on enconding + zero hashes (space saved depends on how many collisions in coins currency ids) + + size_t length = v.cm_merkle_branch.size(); + seria_kv("cm_merkle_branch_length", length, s); + std::vector mask((length + 7) / 8); + if (s.is_input()) { + v.cm_merkle_branch.resize(length); + s.object_key("cm_merkle_branch_mask"); + s.binary(mask.data(), mask.size()); + size_t non_zero_count = 0; + for (size_t i = 0; i != length; ++i) + if ((mask.at(i / 8) & (1U << (i % 8))) != 0) + non_zero_count += 1; + s.object_key("cm_merkle_branch"); + s.begin_array(non_zero_count, true); + for (size_t i = 0; i != length; ++i) { + if ((mask.at(i / 8) & (1U << (i % 8))) != 0) + ser(v.cm_merkle_branch.at(i), s); + else + v.cm_merkle_branch.at(i) = Hash{}; + } + s.end_array(); + } else { + std::vector non_zero_hashes; + for (size_t i = 0; i != length; ++i) + if (v.cm_merkle_branch.at(i) != Hash{}) { + mask.at(i / 8) |= (1U << (i % 8)); + non_zero_hashes.push_back(v.cm_merkle_branch.at(i)); + } + s.object_key("cm_merkle_branch_mask"); + s.binary(mask.data(), mask.size()); + s.object_key("cm_merkle_branch"); + size_t non_zero_count = non_zero_hashes.size(); + s.begin_array(non_zero_count, true); + for (Hash &hash : non_zero_hashes) { + ser(hash, s); + } + s.end_array(); + } +*/ +void ser_members(crypto::CMBranchElement &v, ISeria &s) { + s.object_key("depth"); + s.binary(&v.depth, 1); + seria_kv("hash", v.hash, s); +} + +void ser_members(BlockHeader &v, + ISeria &s, + BlockSeriaType seria_type, + BlockBodyProxy body_proxy, + const crypto::Hash &cm_path) { + if (seria_type == BlockSeriaType::NORMAL) { seria_kv("major_version", v.major_version, s); seria_kv("minor_version", v.minor_version, s); + if (v.major_version == 1) { + seria_kv("timestamp", v.timestamp, s); + seria_kv("previous_block_hash", v.previous_block_hash, s); + v.nonce.resize(4); + s.object_key("nonce"); + s.binary(v.nonce.data(), 4); + return; + } + if (v.is_merge_mined()) { + seria_kv("previous_block_hash", v.previous_block_hash, s); + seria_kv("root_block", v.root_block, s); + if (s.is_input()) { + v.nonce.assign(std::begin(v.root_block.nonce), std::end(v.root_block.nonce)); + v.timestamp = v.root_block.timestamp; + } + return; + } +#if bytecoin_ALLOW_CM + if (v.is_cm_mined()) { + seria_kv("timestamp", v.timestamp, s); + seria_kv("previous_block_hash", v.previous_block_hash, s); + seria_kv("nonce", v.nonce, s); + seria_kv("cm_merkle_branch", v.cm_merkle_branch, s); + return; + } +#endif + throw std::runtime_error("Unknown block major version " + common::to_string(v.major_version)); } if (v.major_version == 1) { + seria_kv("major_version", v.major_version, s); + seria_kv("minor_version", v.minor_version, s); seria_kv("timestamp", v.timestamp, s); seria_kv("previous_block_hash", v.previous_block_hash, s); - unsigned char nonce_data[4]; - common::uint_le_to_bytes(nonce_data, 4, v.nonce); + invariant(v.nonce.size() == 4, ""); s.object_key("nonce"); - s.binary(nonce_data, 4); - if (s.is_input()) - v.nonce = common::uint_le_from_bytes(nonce_data, 4); - if (seria_type != BlockSeriaType::NORMAL) - seria_kv("body_proxy", body_proxy, s); + s.binary(v.nonce.data(), 4); + seria_kv("body_proxy", body_proxy, s); return; } if (v.is_merge_mined()) { - if (seria_type != BlockSeriaType::LONG_BLOCKHASH) { - seria_kv("previous_block_hash", v.previous_block_hash, s); - if (seria_type != BlockSeriaType::NORMAL) - seria_kv("body_proxy", body_proxy, s); + if (seria_type == BlockSeriaType::LONG_BLOCKHASH) { + seria_kv("root_block", v.root_block, s, seria_type); + return; } - // auto parent_block_serializer = make_parent_block_serializer(v, false, false); - if (seria_type != BlockSeriaType::PREHASH) { - s.object_key("parent_block"); - s.begin_object(); - ser_members(v.parent_block, s, seria_type); - s.end_object(); - if (s.is_input()) { - v.nonce = v.parent_block.nonce; - v.timestamp = v.parent_block.timestamp; - } + seria_kv("major_version", v.major_version, s); + seria_kv("minor_version", v.minor_version, s); + seria_kv("previous_block_hash", v.previous_block_hash, s); + seria_kv("body_proxy", body_proxy, s); + if (seria_type != BlockSeriaType::PREHASH) { // BLOCKHASH + seria_kv("root_block", v.root_block, s, seria_type); } - // seria_kv("parent_block", parent_block_serializer, s); return; } #if bytecoin_ALLOW_CM if (v.is_cm_mined()) { + if (seria_type == BlockSeriaType::LONG_BLOCKHASH) { + Hash cm_merkle_root = crypto::tree_hash_from_cm_branch( + v.cm_merkle_branch, get_auxiliary_block_header_hash(v, body_proxy), cm_path); + // We should not allow adding merkle_root_hash twice to the long_hashing_array, so more + // flexible "pre_nonce" | root | "post_nonce" would be bad (allow mining of sidechains) + // We select "nonce" to be first, because this would allow compatibility with some weird + // ASICs which select algorithm based on block major version. Nonce could be made to contain + // all required bytes and be of appropriate length (39+4+1 bytes for full binary compatibilty) + s.object_key("nonce"); + s.binary(v.nonce.data(), v.nonce.size()); + seria_kv("cm_merkle_root", cm_merkle_root, s); + return; + } + // Any participating currency must have prehash taken from something with length != 65 bytes + seria_kv("major_version", v.major_version, s); + seria_kv("minor_version", v.minor_version, s); seria_kv("timestamp", v.timestamp, s); seria_kv("previous_block_hash", v.previous_block_hash, s); - if (seria_type != BlockSeriaType::PREHASH && seria_type != BlockSeriaType::LONG_BLOCKHASH) { - seria_kv("cm_nonce", v.cm_nonce, s); - size_t length = v.cm_merkle_branch.size(); - seria_kv("cm_merkle_branch_length", length, s); - std::vector mask((length + 7) / 8); - if (s.is_input()) { - v.cm_merkle_branch.resize(length); - s.object_key("cm_merkle_branch_mask"); - s.binary(mask.data(), mask.size()); - size_t non_zero_count = 0; - for (size_t i = 0; i != length; ++i) - if ((mask.at(i / 8) & (1 << (i % 8))) != 0) - non_zero_count += 1; - s.object_key("cm_merkle_branch"); - s.begin_array(non_zero_count, true); - for (size_t i = 0; i != length; ++i) { - if ((mask.at(i / 8) & (1 << (i % 8))) != 0) - ser(v.cm_merkle_branch.at(i), s); - else - v.cm_merkle_branch.at(i) = Hash{}; - } - s.end_array(); - } else { - std::vector non_zero_hashes; - for (size_t i = 0; i != length; ++i) - if (v.cm_merkle_branch.at(i) != Hash{}) { - mask.at(i / 8) |= 1 << (i % 8); - non_zero_hashes.push_back(v.cm_merkle_branch.at(i)); - } - s.object_key("cm_merkle_branch_mask"); - s.binary(mask.data(), mask.size()); - s.object_key("cm_merkle_branch"); - size_t non_zero_count = non_zero_hashes.size(); - s.begin_array(non_zero_count, true); - for (Hash &hash : non_zero_hashes) { - ser(hash, s); - } - s.end_array(); - } + seria_kv("body_proxy", body_proxy, s); + if (seria_type != BlockSeriaType::PREHASH) { // BLOCKHASH + seria_kv("nonce", v.nonce, s); + seria_kv("cm_merkle_branch", v.cm_merkle_branch, s); } - if (seria_type != BlockSeriaType::NORMAL) - seria_kv("body_proxy", body_proxy, s); return; } #endif @@ -383,13 +667,13 @@ void ser_members(BlockTemplate &v, ISeria &s) { } void ser_members(RawBlock &v, ISeria &s) { seria_kv("block", v.block, s); - seria_kv("transactions", v.transactions, s); + seria_kv("txs", v.transactions, s); // Name important for P2P kv-binary } void ser_members(Block &v, ISeria &s) { seria_kv("header", v.header, s); seria_kv("transactions", v.transactions, s); } -void ser_members(SWCheckpoint &v, ISeria &s) { +void ser_members(HardCheckpoint &v, ISeria &s) { seria_kv("height", v.height, s); seria_kv("hash", v.hash, s); } diff --git a/src/CryptoNote.hpp b/src/CryptoNote.hpp index 4f541b52..85d47782 100644 --- a/src/CryptoNote.hpp +++ b/src/CryptoNote.hpp @@ -13,18 +13,18 @@ // We define here, as CryptoNoteConfig.h is never included anywhere anymore #define bytecoin_ALLOW_DEBUG_COMMANDS 1 -#define bytecoin_ALLOW_CM 0 +#define bytecoin_ALLOW_CM 1 -#define bytecoin_NEWP2P 0 - -namespace bytecoin { +namespace cn { using crypto::Hash; -using crypto::PublicKey; -using crypto::SecretKey; -using crypto::KeyPair; using crypto::KeyDerivation; using crypto::KeyImage; +using crypto::KeyPair; +using crypto::PublicKey; +using crypto::RingSignature; +using crypto::RingSignature3; +using crypto::SecretKey; using crypto::Signature; using common::BinaryArray; @@ -39,30 +39,37 @@ typedef uint64_t BlockOrTimestamp; // Height or Timestamp, 32-bit is enough, but historically we already have several very large values in blockchain typedef int64_t SignedAmount; -struct CoinbaseInput { +struct InputCoinbase { Height height = 0; + enum { type_tag = 0xff }; + static std::string str_type_tag() { return "coinbase"; } }; -struct KeyInput { +struct InputKey { Amount amount = 0; - std::vector output_indexes; + std::vector output_indexes; KeyImage key_image; + enum { type_tag = 2 }; + static std::string str_type_tag() { return "key"; } }; -struct KeyOutput { +struct OutputKey { + Amount amount = 0; PublicKey public_key; + Hash encrypted_secret; // serialized only in amethyst + bool is_auditable = false; + enum { type_tag = 2, type_tag_auditable = 32 + 2 }; // we treat it similar to a flag + // type_tag_auditable is only allowed in amethyst + static std::string str_type_tag() { return "key"; } + static std::string str_type_tag_auditable() { return "key_auditable"; } }; -typedef boost::variant TransactionInput; +typedef boost::variant TransactionInput; -typedef boost::variant TransactionOutputTarget; +typedef boost::variant TransactionOutput; -struct TransactionOutput { - Amount amount = 0; - // Freaking stupidity - why excess indirection with amount left outside variant part? - // We cannot fix it easily. Will have to switch transaction version - TransactionOutputTarget target; -}; +// Beware - amount is serialized before variant tag. We cannot fix it easily without advancing transaction version +// We've broken compatibility in amethyst, bringing amount inside variant part struct TransactionPrefix { uint8_t version = 0; @@ -72,7 +79,11 @@ struct TransactionPrefix { BinaryArray extra; }; -typedef std::vector> TransactionSignatures; +struct RingSignatures { + std::vector signatures; +}; + +typedef boost::variant TransactionSignatures; struct Transaction : public TransactionPrefix { TransactionSignatures signatures; @@ -80,13 +91,13 @@ struct Transaction : public TransactionPrefix { struct BaseTransaction : public TransactionPrefix {}; // has 'ignored' field during seria -struct ParentBlock { // or Merge Mining Root, not to be confused with previous block! +struct RootBlock { // when block is merge mined uint8_t major_version = 0; uint8_t minor_version = 0; Timestamp timestamp = 0; Hash previous_block_hash; - uint32_t nonce = 0; - uint32_t transaction_count = 0; + uint8_t nonce[4]{}; // 4 bytes is more convenient than uint32_t + size_t transaction_count = 0; std::vector base_transaction_branch; BaseTransaction base_transaction; std::vector blockchain_branch; @@ -95,20 +106,20 @@ struct ParentBlock { // or Merge Mining Root, not to be confused with previous struct BlockHeader { uint8_t major_version = 0; uint8_t minor_version = 0; // Not version at all, used for hard fork voting - uint32_t nonce = 0; Timestamp timestamp = 0; Hash previous_block_hash; - ParentBlock parent_block; // For block with is_merge_mined() true - BinaryArray cm_nonce; // For blocks with CM (V104) - std::vector cm_merkle_branch; // For blocks with CM (V104) + BinaryArray nonce; // 4 bytes, except in blocks with is_cm_mined() (variable-length there) + + RootBlock root_block; // For block with is_merge_mined() true + std::vector cm_merkle_branch; // For blocks with is_cm_mined() true - bool is_merge_mined() const { return major_version == 2 || major_version == 3; } - bool is_cm_mined() const { return major_version == 104; } + bool is_merge_mined() const { return major_version == 2 || major_version == 3 || major_version == 4; } + bool is_cm_mined() const { return major_version == 5; } }; struct BlockBodyProxy { Hash transactions_merkle_root; - uint32_t transaction_count = 0; + size_t transaction_count = 0; }; // BlockHeader + (BlockBodyProxy | BlockBody) are enough to calc POW consensus // BlockBody is std::vector where coinbase is the first one @@ -120,27 +131,47 @@ struct BlockTemplate : public BlockHeader { enum BlockSeriaType { NORMAL, PREHASH, BLOCKHASH, LONG_BLOCKHASH }; -struct AccountPublicAddress { +struct AccountAddressSimple { PublicKey spend_public_key; PublicKey view_public_key; + enum { type_tag = 0 }; + static std::string str_type_tag() { return "simple"; } }; -struct SendProof { // proofing that some tx actually sent amount to particular address - Hash transaction_hash; - AccountPublicAddress address; - Amount amount = 0; - std::string message; +struct AccountAddressUnlinkable { + PublicKey s; + PublicKey sv; + bool is_auditable = false; + enum { type_tag = 1, type_tag_auditable = 2 }; + static std::string str_type_tag() { return "unlinkable"; } + static std::string str_type_tag_auditable() { return "unlinkable_auditable"; } +}; + +typedef boost::variant AccountAddress; + +struct SendproofKey { KeyDerivation derivation; Signature signature; // pair of derivation and signature form a proof of only fact that creator knows transaction private key and // he or she wished to include public view key of address into proof. To further check, look up tx_hash in // main chain and sum amounts of outputs which have spend keys corresponding to address public spend key + // For unlinkable addresses +}; +struct SendproofUnlinkable { + struct Element { + size_t out_index = 0; + PublicKey q; + Signature signature; + }; + std::vector elements; }; -struct AccountKeys { - AccountPublicAddress address; - SecretKey spend_secret_key; - SecretKey view_secret_key; +struct Sendproof { // proofing that some tx actually sent amount to particular address + Hash transaction_hash; + AccountAddress address; + Amount amount = 0; + std::string message; + boost::variant proof; }; struct RawBlock { @@ -153,18 +184,20 @@ class Block { BlockTemplate header; std::vector transactions; - bool from_raw_block(const RawBlock &); - bool to_raw_block(RawBlock &) const; + Block() = default; + explicit Block(const RawBlock &rb); + // bool from_raw_block(const RawBlock &); + // bool to_raw_block(RawBlock &) const; }; -struct SWCheckpoint { +struct HardCheckpoint { Height height = 0; Hash hash; }; struct Checkpoint { Height height = 0; Hash hash; - uint32_t key_id = 0; + size_t key_id = 0; uint64_t counter = 0; Hash get_message_hash() const; bool is_enabled() const { return counter != std::numeric_limits::max(); } @@ -174,55 +207,74 @@ struct SignedCheckpoint : public Checkpoint { }; // Predicates for using in maps, sets, etc -inline bool operator==(const AccountPublicAddress &a, const AccountPublicAddress &b) { +inline bool operator==(const AccountAddressSimple &a, const AccountAddressSimple &b) { return std::tie(a.view_public_key, a.spend_public_key) == std::tie(b.view_public_key, b.spend_public_key); } -inline bool operator!=(const AccountPublicAddress &a, const AccountPublicAddress &b) { return !operator==(a, b); } -inline bool operator<(const AccountPublicAddress &a, const AccountPublicAddress &b) { +inline bool operator!=(const AccountAddressSimple &a, const AccountAddressSimple &b) { return !operator==(a, b); } +inline bool operator<(const AccountAddressSimple &a, const AccountAddressSimple &b) { return std::tie(a.view_public_key, a.spend_public_key) < std::tie(b.view_public_key, b.spend_public_key); } -class Currency; // For ser_members of bytecoin::SendProof -} // namespace bytecoin +inline bool operator==(const AccountAddressUnlinkable &a, const AccountAddressUnlinkable &b) { + return std::tie(a.s, a.sv) == std::tie(b.s, b.sv); +} +inline bool operator!=(const AccountAddressUnlinkable &a, const AccountAddressUnlinkable &b) { + return !operator==(a, b); +} +inline bool operator<(const AccountAddressUnlinkable &a, const AccountAddressUnlinkable &b) { + return std::tie(a.s, a.sv) < std::tie(b.s, b.sv); +} + +class Currency; // For ser_members of cn::Sendproof +} // namespace cn // Serialization is part of CryptoNote standard, not problem to put it here namespace seria { class ISeria; -void ser(bytecoin::Hash &v, ISeria &s); -void ser(bytecoin::KeyImage &v, ISeria &s); -void ser(bytecoin::PublicKey &v, ISeria &s); -void ser(bytecoin::SecretKey &v, ISeria &s); -void ser(bytecoin::KeyDerivation &v, ISeria &s); -void ser(bytecoin::Signature &v, ISeria &s); - -void ser_members(bytecoin::AccountPublicAddress &v, ISeria &s); -void ser_members(bytecoin::SendProof &v, ISeria &s, const bytecoin::Currency &); -void ser_members(bytecoin::TransactionInput &v, ISeria &s); -void ser_members(bytecoin::TransactionOutput &v, ISeria &s); -void ser_members(bytecoin::TransactionOutputTarget &v, ISeria &s); - -void ser_members(bytecoin::CoinbaseInput &v, ISeria &s); -void ser_members(bytecoin::KeyInput &v, ISeria &s); - -void ser_members(bytecoin::KeyOutput &v, ISeria &s); - -void ser_members(bytecoin::TransactionPrefix &v, ISeria &s); -void ser_members(bytecoin::BaseTransaction &v, ISeria &s); -void ser_members(bytecoin::Transaction &v, ISeria &s); - -void ser_members( - bytecoin::ParentBlock &v, ISeria &s, bytecoin::BlockSeriaType seria_type = bytecoin::BlockSeriaType::NORMAL); -void ser_members(bytecoin::BlockHeader &v, ISeria &s, - bytecoin::BlockSeriaType seria_type = bytecoin::BlockSeriaType::NORMAL, - bytecoin::BlockBodyProxy body_proxy = bytecoin::BlockBodyProxy{}); -void ser_members(bytecoin::BlockBodyProxy &v, ISeria &s); -void ser_members(bytecoin::BlockTemplate &v, ISeria &s); - -void ser_members(bytecoin::RawBlock &v, ISeria &s); -void ser_members(bytecoin::Block &v, ISeria &s); - -void ser_members(bytecoin::SWCheckpoint &v, ISeria &s); -void ser_members(bytecoin::Checkpoint &v, ISeria &s); -void ser_members(bytecoin::SignedCheckpoint &v, ISeria &s); -} +void ser(cn::Hash &v, ISeria &s); +void ser(cn::KeyImage &v, ISeria &s); +void ser(cn::PublicKey &v, ISeria &s); +void ser(cn::SecretKey &v, ISeria &s); +void ser(cn::KeyDerivation &v, ISeria &s); +void ser(cn::Signature &v, ISeria &s); +void ser(crypto::EllipticCurveScalar &v, ISeria &s); + +void ser_members(cn::AccountAddressSimple &v, ISeria &s); +void ser_members(cn::AccountAddressUnlinkable &v, ISeria &s); +void ser_members(cn::AccountAddress &v, ISeria &s); +void ser_members(cn::SendproofKey &v, ISeria &s); +void ser_members(cn::SendproofUnlinkable::Element &v, ISeria &s); +void ser_members(cn::SendproofUnlinkable &v, ISeria &s); +void ser_members(cn::Sendproof &v, ISeria &s, const cn::Currency &); +void ser_members(cn::TransactionInput &v, ISeria &s); +void ser_members(cn::TransactionOutput &v, ISeria &s, bool is_tx_amethyst); + +void ser_members(cn::InputCoinbase &v, ISeria &s); +void ser_members(cn::InputKey &v, ISeria &s); +void ser_members(cn::RingSignatures &v, ISeria &s); +void ser_members(cn::RingSignature3 &v, ISeria &s); +void ser_members(cn::TransactionSignatures &v, ISeria &s); +void ser_members(cn::TransactionSignatures &v, ISeria &s, const cn::TransactionPrefix &prefix); + +void ser_members(cn::OutputKey &v, ISeria &s, bool is_tx_amethyst); + +void ser_members(cn::TransactionPrefix &v, ISeria &s); +void ser_members(cn::BaseTransaction &v, ISeria &s); +void ser_members(cn::Transaction &v, ISeria &s); + +void ser_members(cn::RootBlock &v, ISeria &s, cn::BlockSeriaType seria_type = cn::BlockSeriaType::NORMAL); +void ser_members(crypto::CMBranchElement &v, ISeria &s); + +void ser_members(cn::BlockHeader &v, ISeria &s, cn::BlockSeriaType seria_type = cn::BlockSeriaType::NORMAL, + cn::BlockBodyProxy body_proxy = cn::BlockBodyProxy{}, const crypto::Hash &cm_path = crypto::Hash{}); +void ser_members(cn::BlockBodyProxy &v, ISeria &s); +void ser_members(cn::BlockTemplate &v, ISeria &s); + +void ser_members(cn::RawBlock &v, ISeria &s); +void ser_members(cn::Block &v, ISeria &s); + +void ser_members(cn::HardCheckpoint &v, ISeria &s); +void ser_members(cn::Checkpoint &v, ISeria &s); +void ser_members(cn::SignedCheckpoint &v, ISeria &s); +} // namespace seria diff --git a/src/CryptoNoteConfig.hpp b/src/CryptoNoteConfig.hpp index cf56ddd8..8f930bfa 100644 --- a/src/CryptoNoteConfig.hpp +++ b/src/CryptoNoteConfig.hpp @@ -8,87 +8,112 @@ #include #include "CryptoNote.hpp" #include "common/StringTools.hpp" +#include "p2p/P2pProtocolTypes.hpp" + +#ifndef CRYPTONOTE_NAME +#error CRYPTONOTE_NAME must be defined before compiling project +#endif // All values below should only be used in code through Currency and Config classes, never directly. // This approach allows unlimited customization through config file/command line parameters // Never include this header into other headers -namespace bytecoin { -namespace parameters { +namespace cn { namespace parameters { + +// Magics +const char GENESIS_COINBASE_TX_HEX[] = + "010a01ff0001ffffffffffff0f029b2e4c0281c0b02e7c53291a94d1d0cbff8883f8024f5142ee494ffbbd08807121013c086a48c15fb637a96991bc6d53caf77068b5ba6eeb3c82357228c49790584a"; +// Technically, we should not have predefined genesis block, first hard checkpoint is enough. This is bitcoin legacy. +constexpr UUID BYTECOIN_NETWORK = common::pfh("11100111110001011011001210110110"); // Bender's nightmare + +const Height UPGRADE_HEIGHT_V2 = 546603; +const Height UPGRADE_HEIGHT_V3 = 985549; +const Height KEY_IMAGE_SUBGROUP_CHECKING_HEIGHT = 1267000; -const Height MAX_BLOCK_NUMBER = 500000000; -const uint32_t MAX_BLOCK_BLOB_SIZE = 500000000; -const uint32_t MAX_TX_SIZE = 1000000000; -const uint64_t PUBLIC_ADDRESS_BASE58_PREFIX = 6; // addresses start with "2" -const Height MINED_MONEY_UNLOCK_WINDOW = 10; -const Timestamp BLOCK_FUTURE_TIME_LIMIT = 60 * 60 * 2; +// Radical simplification of consensus rules starts from versions +const uint8_t BLOCK_VERSION_AMETHYST = 4; +const uint8_t TRANSACTION_VERSION_AMETHYST = 4; -const Height BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW = 60; +const size_t MINIMUM_ANONYMITY_V1_3 = 0; +const size_t MINIMUM_ANONYMITY = 3; -// MONEY_SUPPLY - total number coins to be generated +// Emission and formats const Amount MONEY_SUPPLY = std::numeric_limits::max(); const unsigned EMISSION_SPEED_FACTOR = 18; static_assert(EMISSION_SPEED_FACTOR <= 8 * sizeof(uint64_t), "Bad EMISSION_SPEED_FACTOR"); -const Height REWARD_BLOCKS_WINDOW = 100; +const size_t DISPLAY_DECIMAL_POINT = 8; +const Amount MIN_DUST_THRESHOLD = 1000000; // Everything smaller will be split in groups of 3 digits +const Amount MAX_DUST_THRESHOLD = 30000000000000000; // Everything larger is dust because very few coins +const Amount SELF_DUST_THRESHOLD = 1000; // forfeit outputs smaller than this in a change -// size of block (bytes) after which reward for block calculated using block size -const size_t MINIMUM_SIZE_MEDIAN = 100000; -const size_t MINIMUM_SIZE_MEDIAN_V2 = 20000; -const size_t MINIMUM_SIZE_MEDIAN_V1 = 10000; -const size_t COINBASE_BLOB_RESERVED_SIZE = 600; -const size_t DISPLAY_DECIMAL_POINT = 8; -const Amount DEFAULT_DUST_THRESHOLD = 1000000; // pow(10, 6) -const Amount SELF_DUST_THRESHOLD = 1000; // pow(10, 3) +const BinaryArray ADDRESS_BASE58_PREFIX{6}; // legacy addresses start with "2" +const BinaryArray ADDRESS_BASE58_PREFIX_UNLINKABLE{0xce, 0xf5, 0xe2}; // addresses start with "bcn1" +const BinaryArray ADDRESS_BASE58_PREFIX_AUDITABLE_UNLINKABLE{0xce, 0xf5, 0xe4}; // addresses start with "bcn2" +const char BLOCKS_FILENAME[] = "blocks.bin"; +const char BLOCKINDEXES_FILENAME[] = "blockindexes.bin"; -const Timestamp DIFFICULTY_TARGET = 120; +// Difficulty and rewards +const Timestamp DIFFICULTY_TARGET = 120; +const Height EXPECTED_NUMBER_OF_BLOCKS_PER_DAY = 24 * 60 * 60 / DIFFICULTY_TARGET; -const Difficulty MINIMUM_DIFFICULTY_V1 = 1; +const Difficulty MINIMUM_DIFFICULTY_V1 = 1; // Genesis and some first blocks in main net const Difficulty MINIMUM_DIFFICULTY = 100000; -const Height DIFFICULTY_CUT = 60; // out-of-family timestamps to cut after sorting -const Height DIFFICULTY_LAG = 15; // skip last blocks for difficulty calcs (against lowering difficulty attack) +const Height DIFFICULTY_WINDOW = 720; +const Height DIFFICULTY_CUT = 60; // out-of-family timestamps to cut after sorting +const Height DIFFICULTY_LAG = 15; // skip last blocks for difficulty calcs (against lowering difficulty attack) + +static_assert(DIFFICULTY_WINDOW >= 2, "Bad DIFFICULTY_WINDOW"); +static_assert(2 * DIFFICULTY_CUT <= DIFFICULTY_WINDOW - 2, "Bad DIFFICULTY_WINDOW or DIFFICULTY_CUT"); + +// Upgrade voting +const Height UPGRADE_VOTING_PERCENT = 90; +const Height UPGRADE_VOTING_WINDOW = EXPECTED_NUMBER_OF_BLOCKS_PER_DAY; +const Height UPGRADE_WINDOW = EXPECTED_NUMBER_OF_BLOCKS_PER_DAY * 7; // Delay after voting +static_assert(60 <= UPGRADE_VOTING_PERCENT && UPGRADE_VOTING_PERCENT <= 100, "Bad UPGRADE_VOTING_PERCENT"); +static_assert(UPGRADE_VOTING_WINDOW > 1, "Bad UPGRADE_VOTING_WINDOW"); -const uint32_t MAX_BLOCK_SIZE_INITIAL = 20 * 1024; -const uint32_t MAX_BLOCK_SIZE_GROWTH_PER_YEAR = 100 * 1024; +// Timestamps +const Timestamp BLOCK_FUTURE_TIME_LIMIT = 60 * 60 * 2; +const Height BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW_V1_3 = 60; +const Height BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW = 59; +static_assert(BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW % 2 == 1, + "This window must be uneven for median timestamp to grow monotonically"); -// After next hardfork remove settings below +// Locking by timestamp and by block +const Height MAX_BLOCK_NUMBER = 500000000; + +// Legacy pre amethyst locking constants const Height LOCKED_TX_ALLOWED_DELTA_BLOCKS = 1; + constexpr Timestamp LOCKED_TX_ALLOWED_DELTA_SECONDS(Timestamp difficulty_target) { return difficulty_target * LOCKED_TX_ALLOWED_DELTA_BLOCKS; } -const Height UPGRADE_HEIGHT_V2 = 546603; -const Height UPGRADE_HEIGHT_V3 = 985549; -const Height KEY_IMAGE_SUBGROUP_CHECKING_HEIGHT = 1267000; // TODO - after fork remove, check subgroup if version >= 4 +const Height MINED_MONEY_UNLOCK_WINDOW = 10; -// const uint32_t UPGRADE_VOTING_WINDOW = EXPECTED_NUMBER_OF_BLOCKS_PER_DAY; // blocks -// const uint32_t UPGRADE_WINDOW = EXPECTED_NUMBER_OF_BLOCKS_PER_DAY*7; // blocks -// static_assert(UPGRADE_VOTING_WINDOW > 1, "Bad UPGRADE_VOTING_WINDOW"); +// Size limits +const size_t MAX_HEADER_SIZE = 2048; +const size_t BLOCK_CAPACITY_VOTE_MIN = 100 * 1000; // min block size +const size_t BLOCK_CAPACITY_VOTE_MAX = 2000 * 1000; // max block size +static_assert(BLOCK_CAPACITY_VOTE_MAX >= BLOCK_CAPACITY_VOTE_MIN, "Bad TRANSACTIONS_SIZE_VOTE"); +const Height BLOCK_CAPACITY_VOTE_WINDOW = 11; -const char BLOCKS_FILENAME[] = "blocks.bin"; -const char BLOCKINDEXES_FILENAME[] = "blockindexes.bin"; -} // parameters - -const char CRYPTONOTE_NAME[] = "bytecoin"; +// Legacy pre amethyst size limits +const size_t MINIMUM_SIZE_MEDIAN_V3 = 100000; +const size_t MINIMUM_SIZE_MEDIAN_V2 = 20000; +const size_t MINIMUM_SIZE_MEDIAN_V1 = 10000; -const uint8_t CURRENT_TRANSACTION_VERSION = 1; - -const size_t BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT = 10000; // by default, blocks ids count in synchronizing -const size_t BLOCKS_SYNCHRONIZING_DEFAULT_COUNT = 100; // by default, blocks count in blocks downloading -const size_t COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT = 1000; +const Height MEIDAN_BLOCK_SIZE_WINDOW = 100; +const size_t MAX_BLOCK_SIZE_INITIAL = 20 * 1024; // block transactions size +const size_t MAX_BLOCK_SIZE_GROWTH_PER_YEAR = 100 * 1024; // block transactions size +// P2p ports, not strictly part of consensus const uint16_t P2P_DEFAULT_PORT = 8080; const uint16_t RPC_DEFAULT_PORT = 8081; const uint16_t WALLET_RPC_DEFAULT_PORT = 8070; -// const size_t P2P_CONNECTION_MAX_WRITE_BUFFER_SIZE = 32 * 1024 * 1024; // 32 Mb -// const uint32_t P2P_DEFAULT_HANDSHAKE_INTERVAL = 60; // seconds -// const uint32_t P2P_DEFAULT_PACKET_MAX_SIZE = 50000000; // 50000000 bytes maximum packet size -const uint32_t P2P_DEFAULT_PEERS_IN_HANDSHAKE = 250; -// const uint32_t P2P_DEFAULT_CONNECTION_TIMEOUT = 5000; // 5 seconds -// const uint32_t P2P_DEFAULT_PING_CONNECTION_TIMEOUT = 2000; // 2 seconds -// const uint32_t P2P_DEFAULT_INVOKE_TIMEOUT = 60 * 2 * 1000; // 2 minutes -// const uint32_t P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT = 5000; // 5 seconds +// We do not want runtime conversion, so compile-time converter constexpr PublicKey P2P_STAT_TRUSTED_PUBLIC_KEY = common::pfh("E29507CA55455F37A3B783EE2C5123B8B6A34A0C5CAAE050922C6254161480C1"); @@ -123,7 +148,7 @@ const char *const SEED_NODES_STAGENET[] = { "207.246.127.160:10080", "108.61.174.232:10080", "45.32.156.183:10080", "45.76.29.96:10080"}; // testnet will have no seed nodes -constexpr const SWCheckpoint CHECKPOINTS[] = { +constexpr const HardCheckpoint CHECKPOINTS[] = { {79000, common::pfh("cae33204e624faeb64938d80073bb7bbacc27017dc63f36c5c0f313cad455a02")}, {140000, common::pfh("993059fb6ab92db7d80d406c67a52d9c02d873ca34b6290a12b744c970208772")}, {200000, common::pfh("a5f74c7542077df6859f48b5b1f9c3741f29df38f91a47e14c94b5696e6c3073")}, @@ -186,9 +211,10 @@ constexpr const SWCheckpoint CHECKPOINTS[] = { {1560000, common::pfh("1a28c09c74b4b1ad97e4d65b99f97e62aa4f225be5b33017efc07c5c708b83ef")}, {1579000, common::pfh("debfa79d14ff49dc7e8c24e5e27a22f9a67819124a7dcd187c67493a969044be")}, {1605000, common::pfh("a34a41f2b5091f28f234b55a6255a9727fed355ca41233d59f779b2f87d1a359")}, - {1628000, common::pfh("4e7b55e53402c71c45cb97f8ed78ed3f128c802008c83b0153aa52c30b740c68")}}; - -constexpr const SWCheckpoint CHECKPOINTS_STAGENET[] = { - {450, common::pfh("c69823a6b3e0c1f724411e697219a9d31a2df900cb49bb0488b1a91a9989a805")}}; + {1628000, common::pfh("4e7b55e53402c71c45cb97f8ed78ed3f128c802008c83b0153aa52c30b740c68")}, + {1670000, common::pfh("58770b800108c72512a386783fd0a4326c74dc9f99b538337a195945b89a9a6f")}}; -} // CryptoNote +constexpr const HardCheckpoint CHECKPOINTS_STAGENET[] = { + {450, common::pfh("c69823a6b3e0c1f724411e697219a9d31a2df900cb49bb0488b1a91a9989a805")}, + {30000, common::pfh("4a3b02206d120bab6c3bef4a7bcbc1934b5327c27c181d790f4db407dc92c640")}}; +}} // namespace cn::parameters diff --git a/src/common/BIPs.cpp b/src/common/BIPs.cpp new file mode 100644 index 00000000..fd659885 --- /dev/null +++ b/src/common/BIPs.cpp @@ -0,0 +1,287 @@ +// Copyright (c) 2012-2018, The CryptoNote developers, The Bytecoin developers. +// Licensed under the GNU Lesser General Public License. See LICENSE for details. + +#include "BIPs.hpp" +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common/Invariant.hpp" +#include "common/StringTools.hpp" +#include "common/Varint.hpp" +#include "common/Words.hpp" + +struct EC_GROUPw { + EC_GROUP *pgroup = nullptr; + EC_GROUPw() : pgroup(EC_GROUP_new_by_curve_name(NID_secp256k1)) {} + ~EC_GROUPw() { + EC_GROUP_free(pgroup); + pgroup = nullptr; + } +}; +struct BIGNUMw { + BIGNUM *pbn = nullptr; + BIGNUMw() : pbn(BN_new()) {} + BIGNUMw(const unsigned char *data, size_t size) : pbn(BN_bin2bn(data, static_cast(size), nullptr)) {} + ~BIGNUMw() { + BN_free(pbn); + pbn = nullptr; + } +}; +struct BN_CTXw { + BN_CTX *ctx = nullptr; + BN_CTXw() : ctx(BN_CTX_new()) {} + ~BN_CTXw() { + BN_CTX_free(ctx); + ctx = nullptr; + } +}; +struct EC_POINTw { + EC_POINT *p = nullptr; + explicit EC_POINTw(EC_GROUP *pgroup) : p(EC_POINT_new(pgroup)) {} + ~EC_POINTw() { + EC_POINT_free(p); + p = nullptr; + } +}; + +using namespace cn; + +const bool debug_print = false; + +void Bip32Key::make_pub() { + EC_GROUPw group; + BIGNUMw priv_bn(priv_key.data(), priv_key.size()); + + EC_POINTw pkey(group.pgroup); + invariant(EC_POINT_mul(group.pgroup, pkey.p, priv_bn.pbn, nullptr, nullptr, nullptr), "EC_POINT_mul failed"); + unsigned char pub_buf[128]{}; + size_t si = + EC_POINT_point2oct(group.pgroup, pkey.p, POINT_CONVERSION_COMPRESSED, pub_buf, sizeof(pub_buf), nullptr); + invariant(si != 0, "EC_POINT_point2oct failed"); + pub_key.assign(pub_buf, pub_buf + si); + if (debug_print) + std::cout << " pub_key=" << common::to_hex(pub_key) << std::endl; +} + +Bip32Key Bip32Key::create_master_key(const std::string &bip39_mnemonic, const std::string &passphrase) { + Bip32Key result; + unsigned char bip39_seed[64]; + std::string hmac_salt = "mnemonic" + passphrase; + std::string bitcoin_seed = "Bitcoin seed"; + PKCS5_PBKDF2_HMAC(bip39_mnemonic.data(), static_cast(bip39_mnemonic.size()), + reinterpret_cast(hmac_salt.data()), static_cast(hmac_salt.size()), 2048, EVP_sha512(), 64, + bip39_seed); + auto master = HMAC( + EVP_sha512(), bitcoin_seed.data(), static_cast(bitcoin_seed.size()), bip39_seed, 64, nullptr, nullptr); + if (debug_print) { + std::cout << "bip39 seed=" << common::to_hex(bip39_seed, 64) << std::endl; + std::cout << "bip39 master chain code=" << common::to_hex(master + 32, 32) << std::endl; + std::cout << "bip39 master key=" << common::to_hex(master, 32) << std::endl; + } + result.chain_code.assign(master + 32, master + 64); + result.priv_key.assign(master, master + 32); + result.make_pub(); + return result; +} + +static const char *utf8_whitespaces[] = { // Control characters + "\x01", "\x02", "\x03", "\x04", "\x05", "\x06", "\x07", "\x08", "\x0E", "\x0F", "\x10", "\x11", "\x12", "\x13", + "\x14", "\x15", "\x16", "\x17", "\x18", "\x19", "\x1A", "\x1B", "\x1C", "\x1D", "\x1E", "\x1F", + // Unicode whitespace characters with WSpace=Y + "\x09", "\x0A", "\x0B", "\x0C", "\x0D", "\x20", + "\xC2\x85", // (U+0085) + "\xC2\xA0", // (U+00A0) + "\xE1\x9A\x80", // (U+1680) + "\xE2\x80\x80", "\xE2\x80\x81", "\xE2\x80\x82", "\xE2\x80\x83", // (U+2000 - U+200A) + "\xE2\x80\x84", "\xE2\x80\x85", "\xE2\x80\x86", "\xE2\x80\x87", "\xE2\x80\x88", "\xE2\x80\x89", "\xE2\x80\x8A", + "\xE2\x80\xA8", // (U+2028) + "\xE2\x80\xA9", // (U+2029) + "\xE2\x80\xAF", // (U+202F) + "\xE2\x81\x9F", // (U+205F) + "\xE3\x80\x80", // (U+3000) + // Unicode whitespace characters without WSpace=Y + "\xE1\xA0\x8E", // (U+180E) + "\xE2\x80\x8B", // (U+200B) + "\xE2\x80\x8C", // (U+200C) + "\xE2\x80\x8D", // (U+200D) + "\xE2\x81\xA0", // (U+2060) + "\xEF\xBB\xBF"}; + +static size_t find_any(const std::string &str, std::string *found) { + size_t best = std::string::npos; + for (const auto wh : utf8_whitespaces) { + size_t pos = str.find(wh); + if (pos < best) { + *found = wh; + best = pos; + } + } + return best; +} + +static_assert(common::WORDS_COUNT == 2048, "BIP39 wordlist should be 2048 words"); + +std::string Bip32Key::create_random_bip39_mnemonic(size_t bits) { + std::string result; + if (bits % 32 != 0) + throw Exception("Mnemonic bits must be multiple of 32"); + if (bits < 128 || bits > 256) + throw Exception("Mnemonic bits must be between 128 and 256"); + const size_t cs_bits = bits / 32; + const size_t should_be_words = (bits + cs_bits) / 11; + std::vector ent_data(bits / 8); + crypto::generate_random_bytes(ent_data.data(), ent_data.size()); + + unsigned char hash[SHA256_DIGEST_LENGTH]; + SHA256_CTX sha256; + SHA256_Init(&sha256); + SHA256_Update(&sha256, ent_data.data(), ent_data.size()); + SHA256_Final(hash, &sha256); + const size_t crc = size_t(hash[0]) >> (8 - cs_bits); + bool crc_added = false; + + size_t remaining_value_bits = 0; + size_t remaining_value = 0; + + size_t words_added = 0; + while (words_added < should_be_words) { + if (remaining_value_bits >= 11) { + size_t word_num = remaining_value >> (remaining_value_bits - 11); + if (word_num >= common::WORDS_COUNT) + throw Exception("Mnemonic creation error - word outside of list"); + if (!result.empty()) + result += " "; + result += common::raw_words[word_num]; + words_added += 1; + remaining_value &= (1 << (remaining_value_bits - 11)) - 1; + remaining_value_bits -= 11; + continue; + } + if (!ent_data.empty()) { + remaining_value <<= 8; + remaining_value |= ent_data.at(0); + remaining_value_bits += 8; + ent_data.erase(ent_data.begin()); + continue; + } + if (!crc_added) { + remaining_value <<= cs_bits; + remaining_value |= crc; + remaining_value_bits += cs_bits; + crc_added = true; + continue; + } + throw Exception("Mnemonic creation error - run out of entropy"); + } + return check_bip39_mnemonic(result); +} + +std::string Bip32Key::check_bip39_mnemonic(const std::string &bip39_mnemonic) { + std::string str = bip39_mnemonic; + // Not the fastest way to split into words by set of strings + std::vector word_bits; + std::string result; + while (!str.empty()) { + std::string found; + auto wpos = find_any(str, &found); + if (wpos == 0) { + str.erase(str.begin(), str.begin() + found.size()); + continue; + } + auto word = str.substr(0, wpos); + const char *const *begin = common::raw_words; + const char *const *end = begin + common::WORDS_COUNT; + auto fou = std::lower_bound(begin, end, word.c_str(), + [](const char *left, const char *right) -> bool { return strcmp(left, right) < 0; }); + if (fou == end || std::string(*fou) != word) + throw Exception("Mnemonic word '" + word + "' not in the list"); + if (!result.empty()) + result += " "; + result += word; + word_bits.push_back(fou - common::raw_words); + if (wpos <= str.size()) + str.erase(str.begin(), str.begin() + wpos); + else + str.clear(); + } + if (word_bits.size() % 3 != 0) + throw Exception("Mnemonic word count is not multiple of 3"); + if (word_bits.size() > 24) + throw Exception("Mnemonic too many words (max 24)"); + const size_t cs_bits = word_bits.size() / 3; + const size_t ent_bytes = 4 * cs_bits; + common::BinaryArray ent_data; + size_t remaining_value_bits = 0; + size_t remaining_value = 0; + while (ent_data.size() < ent_bytes) { + if (remaining_value_bits >= 8) { + ent_data.push_back(static_cast((remaining_value >> (remaining_value_bits - 8)) & 0xFF)); + remaining_value &= (1 << (remaining_value_bits - 8)) - 1; + remaining_value_bits -= 8; + continue; + } + if (word_bits.empty()) + break; + remaining_value <<= 11; + remaining_value |= word_bits.front(); + remaining_value_bits += 11; + word_bits.erase(word_bits.begin()); + } + // std::cout << common::to_hex(ent_data) << std::endl; + if (remaining_value_bits != cs_bits) + throw Exception("Mnemonic invalid format"); + unsigned char hash[SHA256_DIGEST_LENGTH]; + SHA256_CTX sha256; + SHA256_Init(&sha256); + SHA256_Update(&sha256, ent_data.data(), ent_data.size()); + SHA256_Final(hash, &sha256); + const size_t crc = size_t(hash[0]) >> (8 - cs_bits); + if (crc != remaining_value) + throw Exception("Mnemonic wrong CRC"); + return result; +} + +Bip32Key Bip32Key::derive_key(uint32_t child_num) const { + EC_GROUPw group; + BN_CTXw bn_ctx; + unsigned char numbuf[4]{}; + common::uint_be_to_bytes(numbuf, 4, child_num); + Bip32Key result; + result.child_num = child_num; + common::BinaryArray buf; + if (child_num >= 0x80000000U) { + buf.push_back(0); + common::append(buf, priv_key.begin(), priv_key.end()); + } else { + common::append(buf, pub_key.begin(), pub_key.end()); + } + common::append(buf, numbuf, numbuf + 4); + auto master = HMAC(EVP_sha512(), chain_code.data(), static_cast(chain_code.size()), buf.data(), + static_cast(buf.size()), nullptr, nullptr); + result.chain_code.assign(master + 32, master + 64); + if (debug_print) + std::cout << "chain code=" << common::to_hex(result.chain_code) << std::endl; + BIGNUMw priv_bn(master, 32); + BIGNUMw priv_bn2(priv_key.data(), priv_key.size()); + BIGNUMw priv_order; + invariant(BN_add(priv_bn.pbn, priv_bn.pbn, priv_bn2.pbn), "BN_add failed"); + + invariant(EC_GROUP_get_order(group.pgroup, priv_order.pbn, bn_ctx.ctx), "EC_GROUP_get_order failed"); + invariant(BN_mod(priv_bn.pbn, priv_bn.pbn, priv_order.pbn, bn_ctx.ctx), "BN_mod failed"); + result.priv_key.resize(32); + invariant(BN_bn2binpad(priv_bn.pbn, result.priv_key.data(), 32), "BN_bn2binpad failed"); + if (debug_print) + std::cout << " priv_key=" << common::to_hex(result.priv_key) << std::endl; + result.make_pub(); + + return result; +} diff --git a/src/common/BIPs.hpp b/src/common/BIPs.hpp new file mode 100644 index 00000000..5a512965 --- /dev/null +++ b/src/common/BIPs.hpp @@ -0,0 +1,36 @@ +// Copyright (c) 2012-2018, The CryptoNote developers, The Bytecoin developers. +// Licensed under the GNU Lesser General Public License. See LICENSE for details. + +#pragma once + +#include +#include +#include "common/BinaryArray.hpp" + +namespace cn { + +class Bip32Key { + common::BinaryArray priv_key; + common::BinaryArray pub_key; + common::BinaryArray chain_code; + uint32_t child_num = 0; + + void make_pub(); + Bip32Key() = default; + +public: + class Exception : public std::runtime_error { + public: + explicit Exception(const std::string &what) : std::runtime_error(what) {} + }; + static std::string create_random_bip39_mnemonic(size_t bits); + static std::string check_bip39_mnemonic(const std::string &bip39_mnemonic); // normalizes mnemonic + static Bip32Key create_master_key(const std::string &bip39_mnemonic, const std::string &passphrase); + Bip32Key derive_key(uint32_t child_num) const; + uint32_t get_child_num() const { return child_num; } + const common::BinaryArray get_chain_code() const { return chain_code; } + const common::BinaryArray get_priv_key() const { return priv_key; } + const common::BinaryArray get_pub_key() const { return pub_key; } +}; + +} // namespace cn diff --git a/src/common/Base58.cpp b/src/common/Base58.cpp index ea99f187..762b83cc 100644 --- a/src/common/Base58.cpp +++ b/src/common/Base58.cpp @@ -3,84 +3,97 @@ #include "Base58.hpp" -#include #include +#include +#include #include +#include "CRC32.hpp" +#include "Invariant.hpp" +#include "StringTools.hpp" #include "Varint.hpp" #include "crypto/hash.hpp" #include "crypto/int-util.h" -namespace common { -namespace base58 { +namespace common { namespace base58 { namespace { const char alphabet[] = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; const size_t alphabet_size = sizeof(alphabet) - 1; +const size_t full_block_size = 8; +const size_t full_encoded_block_size = 11; const size_t encoded_block_sizes[] = {0, 2, 3, 5, 6, 7, 9, 10, 11}; -const size_t full_block_size = sizeof(encoded_block_sizes) / sizeof(encoded_block_sizes[0]) - 1; -const size_t full_encoded_block_size = encoded_block_sizes[full_block_size]; -const size_t addr_checksum_size = 4; - -struct reverse_alphabet { - reverse_alphabet() { - base = *std::min_element(alphabet, alphabet + alphabet_size); - auto top = *std::max_element(alphabet, alphabet + alphabet_size); - m_data.resize(top - base + 1, -1); - - for (size_t i = 0; i < alphabet_size; ++i) { - size_t idx = static_cast(alphabet[i] - base); - m_data[idx] = static_cast(i); - } - } +const int decoded_block_sizes[] = {0, -1, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8}; +const uint64_t java_hi_parts[] = {0, 0, 0, 0, 0, 0, 8, 514, 29817, 1729386, 100304420}; +const uint64_t java_lo_parts[] = { + 1, 58, 3364, 195112, 11316496, 656356768, 3708954176, 370977408, 41853184, 2427484672, 3355157504}; +const int8_t reverse_alphabet_table[] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, -1, -1, -1, -1, -1, -1, -1, 9, 10, 11, 12, 13, 14, 15, 16, -1, 17, 18, 19, 20, 21, -1, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, -1, -1, -1, -1, -1, -1, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, -1, + 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57}; + +const size_t addr_checksum_size = 4; + +int reverse_alphabet(char letter) { + auto idx = static_cast(letter); + return idx < sizeof(reverse_alphabet_table) ? reverse_alphabet_table[idx] : -1; +} - int operator()(char letter) const { - size_t idx = static_cast(letter - base); - return idx < m_data.size() ? m_data[idx] : -1; - } +/*struct reverse_alphabet { + reverse_alphabet() { +// base = *std::min_element(alphabet, alphabet + alphabet_size); + auto top = *std::max_element(alphabet, alphabet + alphabet_size); + m_data.resize(top + 1, -1); + + for (size_t i = 0; i < alphabet_size; ++i) { + auto idx = static_cast(alphabet[i]); + m_data[idx] = static_cast(i); + } + for(auto d : m_data) + std::cout << int(d) << ", "; + std::cout << std::endl; + } + + int operator()(char letter) const { + auto idx = static_cast(letter); + return idx < m_data.size() ? m_data[idx] : -1; + } - static reverse_alphabet instance; + static reverse_alphabet instance; private: - std::vector m_data; - char base = 0; + std::vector m_data; +// char base = 0; }; reverse_alphabet reverse_alphabet::instance; struct decoded_block_sizes { - decoded_block_sizes() { - m_data.resize(encoded_block_sizes[full_block_size] + 1, -1); - for (size_t i = 0; i <= full_block_size; ++i) { - m_data[encoded_block_sizes[i]] = static_cast(i); - } - } + decoded_block_sizes() { + m_data.resize(encoded_block_sizes[full_block_size] + 1, -1); + for (size_t i = 0; i <= full_block_size; ++i) { + m_data[encoded_block_sizes[i]] = static_cast(i); + } + } - int operator()(size_t encoded_block_size) const { - assert(encoded_block_size <= full_encoded_block_size); - return m_data[encoded_block_size]; - } + int operator()(size_t encoded_block_size) const { + invariant(encoded_block_size <= full_encoded_block_size, ""); + return m_data[encoded_block_size]; + } - static decoded_block_sizes instance; + static decoded_block_sizes instance; private: - std::vector m_data; // {0, -1, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8}; + std::vector m_data; // {0, -1, 1, 2, -1, 3, 4, 5, -1, 6, 7, 8}; }; -decoded_block_sizes decoded_block_sizes::instance; +decoded_block_sizes decoded_block_sizes::instance;*/ void encode_block(const uint8_t *block, size_t size, char *res) { - assert(1 <= size && size <= full_block_size); - - // uint64_t num = uint_be_from_bytes(block, size); - // size_t i = encoded_block_sizes[size] - 1; - // while (num > 0) { - // uint64_t remainder = num % alphabet_size; - // num /= alphabet_size; - // res[i] = alphabet[remainder]; - // --i; - // } - uint64_t num = uint_be_from_bytes(block, size); + invariant(size >= 1 && size <= full_block_size, ""); + + auto num = uint_be_from_bytes(block, size); for (size_t i = encoded_block_sizes[size]; i-- > 0;) { uint64_t remainder = num % alphabet_size; num /= alphabet_size; @@ -88,22 +101,23 @@ void encode_block(const uint8_t *block, size_t size, char *res) { } } -bool decode_block(const char *block, size_t size, uint8_t *res) { - assert(1 <= size && size <= full_encoded_block_size); +bool decode_block_legacy(const char *block, size_t size, uint8_t *res) { + invariant(size <= full_encoded_block_size, ""); - int res_size = decoded_block_sizes::instance(size); - if (res_size <= 0) + int ires_size = decoded_block_sizes[size]; + if (ires_size <= 0) return false; // Invalid block size + auto res_size = static_cast(ires_size); uint64_t res_num = 0; uint64_t order = 1; for (size_t i = size; i-- > 0;) { - int digit = reverse_alphabet::instance(block[i]); + int digit = reverse_alphabet(block[i]); if (digit < 0) return false; // Invalid symbol uint64_t product_hi; - uint64_t tmp = res_num + mul128(order, digit, &product_hi); + uint64_t tmp = res_num + mul128(order, static_cast(digit), &product_hi); if (tmp < res_num || 0 != product_hi) return false; // Overflow @@ -111,12 +125,57 @@ bool decode_block(const char *block, size_t size, uint8_t *res) { order *= alphabet_size; // Never overflows, 58^10 < 2^64 } - if (static_cast(res_size) < full_block_size && (uint64_t(1) << (8 * res_size)) <= res_num) + if (res_size < full_block_size && (uint64_t(1) << (8 * res_size)) <= res_num) return false; // Overflow uint_be_to_bytes(res, res_size, res_num); return true; } + +bool decode_block_good(const char *block, size_t size, uint8_t *res) { + invariant(size <= full_encoded_block_size, ""); + + int ires_size = decoded_block_sizes[size]; + if (ires_size <= 0) + return false; // Invalid block size + auto res_size = static_cast(ires_size); + + uint64_t java_hi_part = 0; + uint64_t java_lo_part = 0; + size_t java_pos = 0; + for (size_t i = size; i-- > 0; java_pos += 1) { + int digit = reverse_alphabet(block[i]); + if (digit < 0) + return false; // Invalid symbol + java_hi_part += java_hi_parts[java_pos] * static_cast(digit); + java_lo_part += java_lo_parts[java_pos] * static_cast(digit); + } + java_hi_part += java_lo_part / 0x100000000; + java_lo_part %= 0x100000000; // Not strictly necessary + if (java_hi_part > 0x100000000) + return false; + if (res_size > 4) { + uint_be_to_bytes(res, res_size - 4, java_hi_part); + uint_be_to_bytes(res + res_size - 4, 4, java_lo_part); + } else + uint_be_to_bytes(res, res_size, java_lo_part); + return true; +} + +bool decode_block(const char *block, size_t size, uint8_t *res) { + invariant(size <= full_encoded_block_size, ""); + int ires_size = decoded_block_sizes[size]; + if (ires_size <= 0) + return false; // Invalid block size + auto res_size = static_cast(ires_size); + + uint8_t result_legacy[full_block_size]{}; + bool res1 = decode_block_good(block, size, res); + bool res2 = decode_block_legacy(block, size, result_legacy); + invariant(res1 == res2 && memcmp(res, result_legacy, res_size) == 0, ""); + return res1; +} + } // namespace std::string encode(const BinaryArray &data) { @@ -124,7 +183,7 @@ std::string encode(const BinaryArray &data) { size_t last_block_size = data.size() % full_block_size; size_t res_size = full_block_count * full_encoded_block_size + encoded_block_sizes[last_block_size]; - std::string res(res_size, '*'); + std::string res(res_size, '*'); // All asterisks must be replaced after encoding for (size_t i = 0; i < full_block_count; ++i) { encode_block(data.data() + i * full_block_size, full_block_size, &res[i * full_encoded_block_size]); } @@ -138,7 +197,7 @@ std::string encode(const BinaryArray &data) { bool decode(const std::string &enc, BinaryArray *data) { size_t full_block_count = enc.size() / full_encoded_block_size; size_t last_block_size = enc.size() % full_encoded_block_size; - int last_block_decoded_size = decoded_block_sizes::instance(last_block_size); + int last_block_decoded_size = decoded_block_sizes[last_block_size]; if (last_block_decoded_size < 0) return false; // Invalid enc length size_t data_size = full_block_count * full_block_size + last_block_decoded_size; @@ -155,19 +214,18 @@ bool decode(const std::string &enc, BinaryArray *data) { &(*data)[full_block_count * full_block_size])) return false; } - return true; } -std::string encode_addr(uint64_t tag, const BinaryArray &data) { - BinaryArray buf = get_varint_data(tag); +std::string encode_addr(const BinaryArray &tag, const BinaryArray &data) { + BinaryArray buf = tag; append(buf, data.begin(), data.end()); crypto::Hash hash = crypto::cn_fast_hash(buf.data(), buf.size()); append(buf, hash.data, hash.data + addr_checksum_size); return encode(buf); } -bool decode_addr(std::string addr, uint64_t *tag, BinaryArray *data) { +bool decode_addr(std::string addr, size_t body_size, BinaryArray *tag, BinaryArray *data) { BinaryArray addr_data; bool r = decode(addr, &addr_data); if (!r) @@ -178,191 +236,44 @@ bool decode_addr(std::string addr, uint64_t *tag, BinaryArray *data) { std::vector checksum(addr_data.end() - addr_checksum_size, addr_data.end()); addr_data.resize(addr_data.size() - addr_checksum_size); + + // std::cout << common::to_hex(addr_data) << std::endl; crypto::Hash hash = crypto::cn_fast_hash(addr_data.data(), addr_data.size()); + // std::cout << common::to_hex(hash.data, sizeof(hash.data)) << std::endl; std::vector expected_checksum(hash.data, hash.data + addr_checksum_size); if (expected_checksum != checksum) return false; - int read = common::read_varint(addr_data.begin(), addr_data.end(), *tag); - if (read <= 0) + if (addr_data.size() < body_size) return false; - - data->assign(addr_data.begin() + read, addr_data.end()); // addr_data.substr(read); + tag->assign(addr_data.begin(), addr_data.end() - body_size); + data->assign(addr_data.end() - body_size, addr_data.end()); return true; } -} - -static const uint32_t table[] = {0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, - 0x9E6495A3, 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, - 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, 0x136C9856, - 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, 0x3B6E20C8, 0x4C69105E, - 0xD56041E4, 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, - 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, - 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, - 0x58684C11, 0xC1611DAB, 0xB6662D3D, 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, - 0x9FBFE4A5, 0xE8B8D433, 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, - 0xE6635C01, 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, - 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, 0x4DB26158, - 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, 0x4369E96A, 0x346ED9FC, - 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, 0x5005713C, 0x270241AA, 0xBE0B1010, - 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, - 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, - 0x9DD277AF, 0x04DB2615, 0x73DC1683, 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, - 0x0A00AE27, 0x7D079EB1, 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, - 0x6E6B06E7, 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, - 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, 0xD80D2BDA, - 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, 0xCB61B38C, 0xBC66831A, - 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, - 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, - 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, - 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, - 0x6FB077E1, 0x18B74777, 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, - 0x166CCF45, 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, - 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, 0xBDBDF21C, - 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, 0xB3667A2E, 0xC4614AB8, - 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D}; - -uint32_t crc32(const uint8_t *data, size_t size, uint32_t crc) { - crc = crc ^ 0xFFFFFFFFU; - for (size_t i = 0; i != size; ++i) { - size_t y = (crc ^ data[i]) & 0xFFU; - crc = (crc >> 8) ^ table[y]; - } - return crc ^ 0xFFFFFFFFU; -} -// Optimized for checking by simple javascript -void encode_crc_block(uint32_t num, size_t encoded_size, char *res) { - for (size_t i = encoded_size; i-- > 0;) { - uint64_t remainder = num % base58::alphabet_size; - num /= base58::alphabet_size; - res[i] = base58::alphabet[remainder]; - } -} -const size_t future_addr_checksum_size = 4; -static_assert(future_addr_checksum_size <= 4, "we user crc32 so cannot be > 4"); -std::string encode_addr_future(std::string prefix, const BinaryArray &addr_data) { - std::string encoded_data = prefix + base58::encode(addr_data); - uint32_t crc = crc32(reinterpret_cast(encoded_data.data()), encoded_data.size()); - size_t res_size = base58::encoded_block_sizes[future_addr_checksum_size]; - std::string crc_str(res_size, '*'); - encode_crc_block(crc, res_size, &crc_str[0]); - return encoded_data + crc_str; -} - -bool decode_addr_future(std::string addr, std::string prefix, BinaryArray *addr_data) { - size_t res_size = base58::encoded_block_sizes[future_addr_checksum_size]; - if (addr.size() < res_size + prefix.size()) - return false; - std::string encoded_data = addr.substr(0, addr.size() - res_size); - uint32_t crc = crc32(reinterpret_cast(encoded_data.data()), encoded_data.size()); - std::string crc_str(res_size, '*'); - encode_crc_block(crc, res_size, &crc_str[0]); - - std::string actual_crc_str = addr.substr(addr.size() - res_size); - - if (crc_str != actual_crc_str) - return false; - if (encoded_data.substr(0, prefix.size()) != prefix) - return false; - return base58::decode(encoded_data.substr(prefix.size()), addr_data); -} - -void test_addr_future() { - // auto a1 = crypto::random_keypair(); - // auto a2 = crypto::random_keypair(); - // AccountPublicAddress ap{a1.public_key, a2.public_key}; - // std::cout << currency.account_address_as_string(ap) << std::endl; - // - // BinaryArray addr_data = seria::to_binary(ap); - // std::string addr = common::encode_addr_future("bcn", addr_data); - // std::cout << addr << std::endl; - // BinaryArray addr_data2; - // bool addr_good = common::decode_addr_future(addr, "bcn", &addr_data2); - // std::cout << addr_good << std::endl; -} +BinaryArray find_tag(const std::string &prefix) { + invariant(prefix.size() <= full_encoded_block_size, ""); + std::string str1 = prefix + std::string(full_encoded_block_size - prefix.size(), alphabet[0]); + std::string str2 = prefix + std::string(full_encoded_block_size - prefix.size(), alphabet[alphabet_size - 1]); + uint8_t result1[full_block_size]{}; + uint8_t result2[full_block_size]{}; + invariant(decode_block(str1.data(), str1.size(), result1), ""); + invariant(decode_block(str2.data(), str2.size(), result2), ""); + std::cout << "decode of " << str1 << " = " << common::to_hex(result1, full_block_size) << std::endl; + std::cout << "decode of " << str2 << " = " << common::to_hex(result2, full_block_size) << std::endl; + size_t pos = 0; + for (; pos != full_block_size; ++pos) + if (result1[pos] != result2[pos]) { + result1[pos] += 1; + pos += 1; + break; + } + BinaryArray tag(result1, result1 + pos); + std::cout << "tag= " << common::to_hex(tag) << std::endl; + std::cout << "address min= " << encode_addr(tag, BinaryArray(64, 0)) << std::endl; + std::cout << "address max= " << encode_addr(tag, BinaryArray(64, 0xff)) << std::endl; + return tag; } -// Example random addresses - -// bcn7rcNCaFR3gSFDgAcgRMn12Fm7BcHot3DBPEXmX3t6x8PdbxwwprTJtrbPN2ismWzYzNpKwmAXT6BqfEbMX5VtW8W5TTRQz -// bcncpS2YQUzZZN52XwyspjWsP6Mcz4Rb36RHEfEduvb1wTdFH9foAsr3xDJJHGzuvX94Qb2oaxsZDAVB5HdMpwtTuS45jxGNu -// bcnVsnFc5eSpMUPCVwtk8vGcf591HcLxjxkTVkPNWSDBvwSGg5p6CCXV6DCrK4Z6mYjrd3DtLFsirNdJEUBCDYPqDAm2U1aqx - -// crc 509845007 -// crc_str "1n46AA" -// bcnTvZ8DRwgnRZLC2Db3FHC375LBE9NfrYpREbXDByVwbpB5htuNXSJ1roUDjtbJ5nxVUPWbKCpiaq3eXPcN5x9iMxJ1n46AA - -// Javascript impl of future address checking - -// function crc32 ( str ) { -// // http://kevin.vanzonneveld.net -// // + original by: Webtoolkit.info (http://www.webtoolkit.info/) -// // + improved by: T0bsn -// // - depends on: utf8_encode -// // * example 1: crc32('Kevin van Zonneveld'); -// // * returns 1: 1249991249 -// -// var table = "00000000 77073096 EE0E612C 990951BA 076DC419 706AF48F E963A535 9E6495A3 0EDB8832 79DCB8A4 E0D5E91E -// 97D2D988 09B64C2B 7EB17CBD E7B82D07 90BF1D91 1DB71064 6AB020F2 F3B97148 84BE41DE 1ADAD47D 6DDDE4EB F4D4B551 -// 83D385C7 136C9856 646BA8C0 FD62F97A 8A65C9EC 14015C4F 63066CD9 FA0F3D63 8D080DF5 3B6E20C8 4C69105E D56041E4 -// A2677172 3C03E4D1 4B04D447 D20D85FD A50AB56B 35B5A8FA 42B2986C DBBBC9D6 ACBCF940 32D86CE3 45DF5C75 DCD60DCF -// ABD13D59 26D930AC 51DE003A C8D75180 BFD06116 21B4F4B5 56B3C423 CFBA9599 B8BDA50F 2802B89E 5F058808 C60CD9B2 -// B10BE924 2F6F7C87 58684C11 C1611DAB B6662D3D 76DC4190 01DB7106 98D220BC EFD5102A 71B18589 06B6B51F 9FBFE4A5 -// E8B8D433 7807C9A2 0F00F934 9609A88E E10E9818 7F6A0DBB 086D3D2D 91646C97 E6635C01 6B6B51F4 1C6C6162 856530D8 -// F262004E 6C0695ED 1B01A57B 8208F4C1 F50FC457 65B0D9C6 12B7E950 8BBEB8EA FCB9887C 62DD1DDF 15DA2D49 8CD37CF3 -// FBD44C65 4DB26158 3AB551CE A3BC0074 D4BB30E2 4ADFA541 3DD895D7 A4D1C46D D3D6F4FB 4369E96A 346ED9FC AD678846 -// DA60B8D0 44042D73 33031DE5 AA0A4C5F DD0D7CC9 5005713C 270241AA BE0B1010 C90C2086 5768B525 206F85B3 B966D409 -// CE61E49F 5EDEF90E 29D9C998 B0D09822 C7D7A8B4 59B33D17 2EB40D81 B7BD5C3B C0BA6CAD EDB88320 9ABFB3B6 03B6E20C -// 74B1D29A EAD54739 9DD277AF 04DB2615 73DC1683 E3630B12 94643B84 0D6D6A3E 7A6A5AA8 E40ECF0B 9309FF9D 0A00AE27 -// 7D079EB1 F00F9344 8708A3D2 1E01F268 6906C2FE F762575D 806567CB 196C3671 6E6B06E7 FED41B76 89D32BE0 10DA7A5A -// 67DD4ACC F9B9DF6F 8EBEEFF9 17B7BE43 60B08ED5 D6D6A3E8 A1D1937E 38D8C2C4 4FDFF252 D1BB67F1 A6BC5767 3FB506DD -// 48B2364B D80D2BDA AF0A1B4C 36034AF6 41047A60 DF60EFC3 A867DF55 316E8EEF 4669BE79 CB61B38C BC66831A 256FD2A0 -// 5268E236 CC0C7795 BB0B4703 220216B9 5505262F C5BA3BBE B2BD0B28 2BB45A92 5CB36A04 C2D7FFA7 B5D0CF31 2CD99E8B -// 5BDEAE1D 9B64C2B0 EC63F226 756AA39C 026D930A 9C0906A9 EB0E363F 72076785 05005713 95BF4A82 E2B87A14 7BB12BAE -// 0CB61B38 92D28E9B E5D5BE0D 7CDCEFB7 0BDBDF21 86D3D2D4 F1D4E242 68DDB3F8 1FDA836E 81BE16CD F6B9265B 6FB077E1 -// 18B74777 88085AE6 FF0F6A70 66063BCA 11010B5C 8F659EFF F862AE69 616BFFD3 166CCF45 A00AE278 D70DD2EE 4E048354 -// 3903B3C2 A7672661 D06016F7 4969474D 3E6E77DB AED16A4A D9D65ADC 40DF0B66 37D83BF0 A9BCAE53 DEBB9EC5 47B2CF7F -// 30B5FFE9 BDBDF21C CABAC28A 53B39330 24B4A3A6 BAD03605 CDD70693 54DE5729 23D967BF B3667A2E C4614AB8 5D681B02 -// 2A6F2B94 B40BBE37 C30C8EA1 5A05DF1B 2D02EF8D"; -// -// var crc = 0; -// var x = 0; -// var y = 0; -// -// crc = crc ^ (-1); -// for( var i = 0, iTop = str.length; i < iTop; i++ ) { -// y = ( crc ^ str.charCodeAt( i ) ) & 0xFF; -// x = "0x" + table.substr( y * 9, 8 ); -// crc = ( crc >>> 8 ) ^ x; -// } -// -// return (crc ^ (-1)) >>> 0; -//} -// var alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; -// -// function encode_crc_block(num, encoded_size) { -// var result = "" -// for(var i = encoded_size; i-- > 0 ;) { -// remainder = num % alphabet.length; -// num = Math.floor(num/alphabet.length); -// result = alphabet[remainder] + result; -// } -// return result -//} -// -// function check_address(addr, prefix) { -// var res_size = 6; -// if( addr.length < res_size + prefix.length ) -// return false; -// var actual_crc_str = addr.substr(addr.length - res_size); -// var encoded_addr = addr.substr(0, addr.length - res_size); -// var crc = crc32(encoded_addr) -// crc_str = encode_crc_block(crc, res_size); -// if(crc_str != actual_crc_str) -// return false; -// if(encoded_addr.substr(0, prefix.length) != prefix) -// return false; -// return true; -//} +}} // namespace common::base58 diff --git a/src/common/Base58.hpp b/src/common/Base58.hpp index 2a8b8859..dc107c33 100644 --- a/src/common/Base58.hpp +++ b/src/common/Base58.hpp @@ -7,18 +7,13 @@ #include #include "BinaryArray.hpp" -namespace common { -namespace base58 { +namespace common { namespace base58 { std::string encode(const BinaryArray &data); bool decode(const std::string &enc, BinaryArray *data); -std::string encode_addr(uint64_t tag, const BinaryArray &data); -bool decode_addr(std::string addr, uint64_t *tag, BinaryArray *data); -} +std::string encode_addr(const BinaryArray &tag, const BinaryArray &data); +bool decode_addr(std::string addr, size_t body_size, BinaryArray *tag, BinaryArray *data); -uint32_t crc32(const uint8_t *data, size_t size, uint32_t crc = 0); -std::string encode_addr_future(std::string prefix, const BinaryArray &addr_data); -bool decode_addr_future(std::string addr, std::string prefix, BinaryArray *addr_data); -void test_addr_future(); -} +BinaryArray find_tag(const std::string &prefix); +}} // namespace common::base58 diff --git a/src/common/Base64.cpp b/src/common/Base64.cpp index 3c34fd44..94f5df96 100644 --- a/src/common/Base64.cpp +++ b/src/common/Base64.cpp @@ -3,11 +3,9 @@ #include "Base64.hpp" -#include #include -namespace common { -namespace base64 { +namespace common { namespace base64 { static const uint8_t from_base64[128] = { // 8 rows of 16 = 128 @@ -43,9 +41,9 @@ std::string encode(const BinaryArray &data) { for (size_t i = 0; i < ret_size / 4; ++i) { // Read a group of three bytes (avoid buffer overrun by replacing with 0) const size_t index = i * 3; - const uint8_t b3_0 = (index + 0 < buf_len) ? buf[index + 0] : 0; - const uint8_t b3_1 = (index + 1 < buf_len) ? buf[index + 1] : 0; - const uint8_t b3_2 = (index + 2 < buf_len) ? buf[index + 2] : 0; + const uint8_t b3_0 = (index + 0 < buf_len) ? buf[index + 0] : uint8_t(0); + const uint8_t b3_1 = (index + 1 < buf_len) ? buf[index + 1] : uint8_t(0); + const uint8_t b3_2 = (index + 2 < buf_len) ? buf[index + 2] : uint8_t(0); // Transform into four base 64 characters const uint8_t b4_0 = ((b3_0 & 0xfc) >> 2); @@ -61,7 +59,7 @@ std::string encode(const BinaryArray &data) { } // Replace data that is invalid (always as many as there are missing bytes) - for (size_t i = 0; i != missing; ++i) + for (size_t i = 0; i != missing; ++i) ret[ret_size - i - 1] = '='; return ret; } @@ -80,13 +78,16 @@ bool decode(const std::string &in, BinaryArray *ret) { for (size_t i = 0; i < encoded_size; i += 4) { // Get values for each group of four base 64 characters const uint8_t b4_0 = - (static_cast(in[i + 0]) <= 'z') ? from_base64[static_cast(in[i + 0])] : 0xff; - const uint8_t b4_1 = - (i + 1 < N && static_cast(in[i + 1]) <= 'z') ? from_base64[static_cast(in[i + 1])] : 0xff; - const uint8_t b4_2 = - (i + 2 < N && static_cast(in[i + 2]) <= 'z') ? from_base64[static_cast(in[i + 2])] : 0xff; - const uint8_t b4_3 = - (i + 3 < N && static_cast(in[i + 3]) <= 'z') ? from_base64[static_cast(in[i + 3])] : 0xff; + (static_cast(in[i + 0]) <= 'z') ? from_base64[static_cast(in[i + 0])] : uint8_t(0xff); + const uint8_t b4_1 = (i + 1 < N && static_cast(in[i + 1]) <= 'z') + ? from_base64[static_cast(in[i + 1])] + : uint8_t(0xff); + const uint8_t b4_2 = (i + 2 < N && static_cast(in[i + 2]) <= 'z') + ? from_base64[static_cast(in[i + 2])] + : uint8_t(0xff); + const uint8_t b4_3 = (i + 3 < N && static_cast(in[i + 3]) <= 'z') + ? from_base64[static_cast(in[i + 3])] + : uint8_t(0xff); // Transform into a group of three bytes const uint8_t b3_0 = ((b4_0 & 0x3f) << 2) + ((b4_1 & 0x30) >> 4); @@ -103,5 +104,4 @@ bool decode(const std::string &in, BinaryArray *ret) { } return true; // TODO - find decoder which returns false on invalid data } -} -} +}} // namespace common::base64 diff --git a/src/common/Base64.hpp b/src/common/Base64.hpp index c2efc5b8..f5985319 100644 --- a/src/common/Base64.hpp +++ b/src/common/Base64.hpp @@ -7,10 +7,8 @@ #include #include "BinaryArray.hpp" -namespace common { -namespace base64 { +namespace common { namespace base64 { std::string encode(const BinaryArray &data); bool decode(const std::string &enc, BinaryArray *data); -} -} +}} // namespace common::base64 diff --git a/src/common/BinaryArray.cpp b/src/common/BinaryArray.cpp index 2636804d..161fafb4 100644 --- a/src/common/BinaryArray.cpp +++ b/src/common/BinaryArray.cpp @@ -2,109 +2,110 @@ // Licensed under the GNU Lesser General Public License. See LICENSE for details. #include "BinaryArray.hpp" +#include #include #include "Invariant.hpp" namespace common { -// TODO - check self-inserts and other corner cases +// Beware - some self-inserts and other corner cases do not work -BinaryArrayImpl::iterator BinaryArrayImpl::insert(iterator pos, const value_type *be, const value_type *en) { - size_t left = pos - m_data; - invariant(left <= m_size, "insert after the end"); - size_t right = m_size - left; - size_t add = en - be; - if (m_size + add <= m_reserved) { - good_memmove(m_data + left + add, m_data + left, right); - good_memmove(m_data + left, be, add); - m_size += add; - return m_data + left; - } - BinaryArrayImpl other((m_size + add + 32) * 3 / 2); - good_memmove(other.m_data, m_data, left); - good_memmove(other.m_data + left + add, m_data + left, right); - good_memmove(other.m_data + left, be, add); - other.m_size = m_size + add; - swap(other); - return m_data + left; +/*BinaryArrayImpl::iterator BinaryArrayImpl::insert(iterator pos, const value_type *be, const value_type *en) { + size_t left = pos - m_data; + invariant(left <= m_size, "insert after the end"); + size_t right = m_size - left; + size_t add = en - be; + if (m_size + add <= m_reserved) { + good_memmove(m_data + left + add, m_data + left, right); + good_memmove(m_data + left, be, add); + m_size += add; + return m_data + left; + } + BinaryArrayImpl other((m_size + add + 32) * 3 / 2); + good_memmove(other.m_data, m_data, left); + good_memmove(other.m_data + left + add, m_data + left, right); + good_memmove(other.m_data + left, be, add); + other.m_size = m_size + add; + swap(other); + return m_data + left; } BinaryArrayImpl::iterator BinaryArrayImpl::insert(iterator pos, size_t add, value_type va) { - size_t left = pos - m_data; - invariant(left <= m_size, "insert after the end"); - size_t right = m_size - left; - if (m_size + add <= m_reserved) { - good_memmove(m_data + left + add, m_data + left, right); - memset(m_data + left, va, add); - m_size += add; - return m_data + left; - } - BinaryArrayImpl other((m_size + add + 32) * 3 / 2); - good_memmove(other.m_data, m_data, left); - good_memmove(other.m_data + left + add, m_data + left, right); - memset(other.m_data + left, va, add); - other.m_size = m_size + add; - swap(other); - return m_data + left; + size_t left = pos - m_data; + invariant(left <= m_size, "insert after the end"); + size_t right = m_size - left; + if (m_size + add <= m_reserved) { + good_memmove(m_data + left + add, m_data + left, right); + memset(m_data + left, va, add); + m_size += add; + return m_data + left; + } + BinaryArrayImpl other((m_size + add + 32) * 3 / 2); + good_memmove(other.m_data, m_data, left); + good_memmove(other.m_data + left + add, m_data + left, right); + memset(other.m_data + left, va, add); + other.m_size = m_size + add; + swap(other); + return m_data + left; } void BinaryArrayImpl::reserve_grow(size_t re, bool more) { - if (re <= m_reserved) - return; - BinaryArrayImpl other(more ? (re + 32) * 3 / 2 : re); - good_memmove(other.m_data, m_data, m_size); - other.m_size = m_size; - swap(other); + if (re <= m_reserved) + return; + BinaryArrayImpl other(more ? (re + 32) * 3 / 2 : re); + good_memmove(other.m_data, m_data, m_size); + other.m_size = m_size; + swap(other); } void BinaryArrayImpl::reserve(size_t re) { reserve_grow(re, false); } void BinaryArrayImpl::push_back(value_type va) { - reserve_grow(m_size + 1, true); - m_data[m_size] = va; - m_size += 1; + reserve_grow(m_size + 1, true); + m_data[m_size] = va; + m_size += 1; } void BinaryArrayImpl::assign(const value_type *be, const value_type *en) { - size_t si = en - be; - if (si <= m_reserved) { - good_memmove(m_data, be, si); - m_size = si; - return; - } - BinaryArrayImpl other(be, en); - swap(other); + size_t si = en - be; + if (si <= m_reserved) { + good_memmove(m_data, be, si); + m_size = si; + return; + } + BinaryArrayImpl other(be, en); + swap(other); } -void BinaryArrayImpl::resize(size_t si) { - if (si <= m_reserved) { - m_size = si; - return; - } - BinaryArrayImpl other(si); - good_memmove(other.m_data, m_data, m_size); - swap(other); -} +// void BinaryArrayImpl::resize(size_t si) { +// if (si <= m_reserved) { +// m_size = si; +// return; +// } +// BinaryArrayImpl other(si); +// good_memmove(other.m_data, m_data, m_size); +// swap(other); +//} void BinaryArrayImpl::resize(size_t si, value_type va) { - if (si <= m_size) { - m_size = si; - return; - } - if (si <= m_reserved) { - memset(m_data + m_size, va, si - m_size); - m_size = si; - return; - } - BinaryArrayImpl other(si); - good_memmove(other.m_data, m_data, m_size); - memset(other.m_data + m_size, va, si - m_size); - swap(other); -} + if (si <= m_size) { + m_size = si; + return; + } + if (si <= m_reserved) { + memset(m_data + m_size, va, si - m_size); + m_size = si; + return; + } + BinaryArrayImpl other(si); + good_memmove(other.m_data, m_data, m_size); + memset(other.m_data + m_size, va, si - m_size); + swap(other); +}*/ const unsigned char *slow_memmem(const unsigned char *buf, size_t buflen, const unsigned char *pat, size_t patlen) { if (patlen == 0) return nullptr; while (buflen) { - auto char_ptr = (const unsigned char *)memchr(buf, pat[0], buflen); + auto char_ptr = reinterpret_cast(memchr(buf, pat[0], buflen)); if (char_ptr - buf + patlen > buflen) return nullptr; if (memcmp(char_ptr, pat, patlen) == 0) @@ -114,4 +115,4 @@ const unsigned char *slow_memmem(const unsigned char *buf, size_t buflen, const } return nullptr; } -} +} // namespace common diff --git a/src/common/BinaryArray.hpp b/src/common/BinaryArray.hpp index 620e87be..87866a84 100644 --- a/src/common/BinaryArray.hpp +++ b/src/common/BinaryArray.hpp @@ -3,119 +3,123 @@ #pragma once -#include #include -#include -#include -#include +#include +#include +//#include +//#include +//#include +//#include + namespace common { -class BinaryArrayImpl { +/*class BinaryArrayImpl { public: - typedef unsigned char value_type; - typedef value_type *iterator; - typedef const value_type *const_iterator; - BinaryArrayImpl() {} - ~BinaryArrayImpl() { delete[] m_data; } - explicit BinaryArrayImpl(size_t si) { alloc(si); } - explicit BinaryArrayImpl(size_t si, value_type va) { - alloc(si); - memset(m_data, va, si); - } - explicit BinaryArrayImpl(std::initializer_list li) { - alloc(li.size()); - memmove(m_data, li.begin(), li.size()); - } - explicit BinaryArrayImpl(const value_type *be, const value_type *en) { - alloc(en - be); - good_memmove(m_data, be, en - be); - } - explicit BinaryArrayImpl(const char *be, const char *en) { - alloc(en - be); - good_memmove(m_data, be, en - be); - } - BinaryArrayImpl(const BinaryArrayImpl &other) { - alloc(other.size()); - good_memmove(m_data, other.m_data, m_size); - } - void swap(BinaryArrayImpl &other) { - std::swap(m_data, other.m_data); - std::swap(m_size, other.m_size); - std::swap(m_reserved, other.m_reserved); - } - BinaryArrayImpl(BinaryArrayImpl &&other) { swap(other); } - BinaryArrayImpl &operator=(const BinaryArrayImpl &other) { - BinaryArrayImpl copy(other); - swap(copy); - return *this; - } - BinaryArrayImpl &operator=(BinaryArrayImpl &&other) { - swap(other); - return *this; - } - size_t size() const { return m_size; } - size_t empty() const { return size() == 0; } - const value_type *data() const { return m_data; } - value_type *data() { return m_data; } - const_iterator begin() const { return m_data; } - iterator begin() { return m_data; } - const_iterator end() const { return m_data + m_size; } - iterator end() { return m_data + m_size; } - value_type &operator[](size_t i) { return m_data[i]; } - const value_type &operator[](size_t i) const { return m_data[i]; } - value_type &at(size_t i) { - if (i >= m_size) - throw std::out_of_range("BinaryArray subscript out of range"); - return m_data[i]; - } - const value_type &at(size_t i) const { - if (i >= m_size) - throw std::out_of_range("BinaryArray subscript out of range"); - return m_data[i]; - } - void clear() { resize(0); } - void resize(size_t si); - void resize(size_t si, value_type va); - void reserve(size_t re); - void assign(const value_type *be, const value_type *en); - void assign(const char *be, const char *en) { - return assign(reinterpret_cast(be), reinterpret_cast(en)); - } - void push_back(value_type va); - iterator insert(iterator pos, const value_type *be, const value_type *en); - iterator insert(iterator pos, size_t add, value_type va); + typedef unsigned char value_type; + typedef value_type *iterator; + typedef const value_type *const_iterator; + BinaryArrayImpl() = default; + ~BinaryArrayImpl() { delete[] m_data; } + // explicit BinaryArrayImpl(size_t si) { alloc(si); } // We removed optimized-for-speed version in fear of + // undeterministic behaviour + explicit BinaryArrayImpl(size_t si, value_type va = 0) { + alloc(si); + memset(m_data, va, si); + } + explicit BinaryArrayImpl(std::initializer_list li) { + alloc(li.size()); + memmove(m_data, li.begin(), li.size()); + } + explicit BinaryArrayImpl(const value_type *be, const value_type *en) { + alloc(en - be); + good_memmove(m_data, be, en - be); + } + explicit BinaryArrayImpl(const char *be, const char *en) { + alloc(en - be); + good_memmove(m_data, be, en - be); + } + BinaryArrayImpl(const BinaryArrayImpl &other) { + alloc(other.size()); + good_memmove(m_data, other.m_data, m_size); + } + void swap(BinaryArrayImpl &other) noexcept { + std::swap(m_data, other.m_data); + std::swap(m_size, other.m_size); + std::swap(m_reserved, other.m_reserved); + } + BinaryArrayImpl(BinaryArrayImpl &&other) noexcept { swap(other); } + BinaryArrayImpl &operator=(const BinaryArrayImpl &other) { + BinaryArrayImpl copy(other); + swap(copy); + return *this; + } + BinaryArrayImpl &operator=(BinaryArrayImpl &&other) noexcept { + swap(other); + return *this; + } + size_t size() const { return m_size; } + bool empty() const { return size() == 0; } + const value_type *data() const { return m_data; } + value_type *data() { return m_data; } + const_iterator begin() const { return m_data; } + iterator begin() { return m_data; } + const_iterator end() const { return m_data + m_size; } + iterator end() { return m_data + m_size; } + value_type &operator[](size_t i) { return m_data[i]; } + const value_type &operator[](size_t i) const { return m_data[i]; } + value_type &at(size_t i) { + if (i >= m_size) + throw std::out_of_range("BinaryArray subscript out of range"); + return m_data[i]; + } + const value_type &at(size_t i) const { + if (i >= m_size) + throw std::out_of_range("BinaryArray subscript out of range"); + return m_data[i]; + } + void clear() { resize(0); } + // void resize(size_t si); // We removed optimized-for-speed version in fear of undeterministic behaviour + void resize(size_t si, value_type va = 0); + void reserve(size_t re); + void assign(const value_type *be, const value_type *en); + void assign(const char *be, const char *en) { + return assign(reinterpret_cast(be), reinterpret_cast(en)); + } + void push_back(value_type va); + iterator insert(iterator pos, const value_type *be, const value_type *en); + iterator insert(iterator pos, size_t add, value_type va); - bool operator==(const BinaryArrayImpl &other) const { - return m_size == other.m_size && memcmp(m_data, other.m_data, m_size) == 0; - } - bool operator!=(const BinaryArrayImpl &other) const { return !(*this == other); } - ptrdiff_t compare(const BinaryArrayImpl &other) const { - int diff = memcmp(m_data, other.m_data, m_size < other.m_size ? m_size : other.m_size); // We avoid std::min - return diff != 0 ? static_cast(diff) - : static_cast(m_size) - static_cast(other.m_size); - } - bool operator<(const BinaryArrayImpl &other) const { return compare(other) < 0; } - bool operator<=(const BinaryArrayImpl &other) const { return compare(other) <= 0; } - bool operator>(const BinaryArrayImpl &other) const { return compare(other) > 0; } - bool operator>=(const BinaryArrayImpl &other) const { return compare(other) >= 0; } + bool operator==(const BinaryArrayImpl &other) const { + return m_size == other.m_size && memcmp(m_data, other.m_data, m_size) == 0; + } + bool operator!=(const BinaryArrayImpl &other) const { return !(*this == other); } + ptrdiff_t compare(const BinaryArrayImpl &other) const { + int diff = memcmp(m_data, other.m_data, m_size < other.m_size ? m_size : other.m_size); // We avoid std::min + return diff != 0 ? static_cast(diff) + : static_cast(m_size) - static_cast(other.m_size); + } + bool operator<(const BinaryArrayImpl &other) const { return compare(other) < 0; } + bool operator<=(const BinaryArrayImpl &other) const { return compare(other) <= 0; } + bool operator>(const BinaryArrayImpl &other) const { return compare(other) > 0; } + bool operator>=(const BinaryArrayImpl &other) const { return compare(other) >= 0; } private: - value_type *m_data = nullptr; - size_t m_size = 0; - size_t m_reserved = 0; - void reserve_grow(size_t re, bool more); - void alloc(size_t si) { - m_size = m_reserved = si; - m_data = new value_type[m_reserved]; - } - void good_memmove(void *dst, const void *src, size_t size) { - if (size != 0) // We have src == nullptr when size == 0, this combination is prohibited by C++ standard - memmove(dst, src, size); - } -}; + value_type *m_data = nullptr; + size_t m_size = 0; + size_t m_reserved = 0; + void reserve_grow(size_t re, bool more); + void alloc(size_t si) { + m_size = m_reserved = si; + m_data = new value_type[m_reserved]; + } + void good_memmove(void *dst, const void *src, size_t size) { + if (size != 0) // We have src == nullptr when size == 0, this combination is prohibited by C++ standard + memmove(dst, src, size); + } +};*/ -// typedef std::vector BinaryArray; -typedef BinaryArrayImpl BinaryArray; +typedef std::vector BinaryArray; +// typedef BinaryArrayImpl BinaryArray; - Safety over performance template inline BinaryArray::iterator append(BinaryArray &ba, It be, It en) { @@ -124,6 +128,9 @@ inline BinaryArray::iterator append(BinaryArray &ba, It be, It en) { inline BinaryArray::iterator append(BinaryArray &ba, size_t add, BinaryArray::value_type va) { return ba.insert(ba.end(), add, va); } +inline BinaryArray::iterator append(BinaryArray &ba, const BinaryArray &other) { + return ba.insert(ba.end(), other.begin(), other.end()); +} const unsigned char *slow_memmem(const unsigned char *buf, size_t buflen, const unsigned char *pat, size_t patlen); -} +} // namespace common diff --git a/src/common/CRC32.cpp b/src/common/CRC32.cpp new file mode 100644 index 00000000..bdb2ff9a --- /dev/null +++ b/src/common/CRC32.cpp @@ -0,0 +1,65 @@ +// This file is generated by CRC32_generate.py. +#include + +#include "CRC32.hpp" + +namespace common { + +const uint32_t crc32_table[] = {3523407757, 2768625435, 1007455905, 1259060791, 3580832660, 2724731650, 996231864, + 1281784366, 3705235391, 2883475241, 852952723, 1171273221, 3686048678, 2897449776, 901431946, 1119744540, + 3484811241, 3098726271, 565944005, 1455205971, 3369614320, 3219065702, 651582172, 1372678730, 3245242331, + 3060352845, 794826487, 1483155041, 3322131394, 2969862996, 671994606, 1594548856, 3916222277, 2657877971, 123907689, + 1885708031, 3993045852, 2567322570, 1010288, 1997036262, 3887548279, 2427484129, 163128923, 2126386893, 3772416878, + 2547889144, 248832578, 2043925204, 4108050209, 2212294583, 450215437, 1842515611, 4088798008, 2226203566, 498629140, + 1790921346, 4194326291, 2366072709, 336475711, 1661535913, 4251816714, 2322244508, 325317158, 1684325040, + 2766056989, 3554254475, 1255198513, 1037565863, 2746444292, 3568589458, 1304234792, 985283518, 2852464175, + 3707901625, 1141589763, 856455061, 2909332022, 3664761504, 1130791706, 878818188, 3110715001, 3463352047, + 1466425173, 543223747, 3187964512, 3372436214, 1342839628, 655174618, 3081909835, 3233089245, 1505515367, 784033777, + 2967466578, 3352871620, 1590793086, 701932520, 2679148245, 3904355907, 1908338681, 112844655, 2564639436, + 4024072794, 1993550816, 30677878, 2439710439, 3865851505, 2137352139, 140662621, 2517025534, 3775001192, 2013832146, + 252678980, 2181537457, 4110462503, 1812594589, 453955339, 2238339752, 4067256894, 1801730948, 476252946, 2363233923, + 4225443349, 1657960367, 366298937, 2343686810, 4239843852, 1707062198, 314082080, 1069182125, 1220369467, + 3518238081, 2796764439, 953657524, 1339070498, 3604597144, 2715744526, 828499103, 1181144073, 3748627891, + 2825434405, 906764422, 1091244048, 3624026538, 2936369468, 571309257, 1426738271, 3422756325, 3137613171, 627095760, + 1382516806, 3413039612, 3161057642, 752284923, 1540473965, 3268974039, 3051332929, 733688034, 1555824756, + 3316994510, 2998034776, 81022053, 1943239923, 3940166985, 2648514015, 62490748, 1958656234, 3988253008, 2595281350, + 168805463, 2097738945, 3825313147, 2466682349, 224526414, 2053451992, 3815530850, 2490061300, 425942017, 1852075159, + 4151131437, 2154433979, 504272920, 1762240654, 4026595636, 2265434530, 397988915, 1623188645, 4189500703, + 2393998729, 282398762, 1741824188, 4275794182, 2312913296, 1231433021, 1046551979, 2808630289, 3496967303, + 1309403428, 957143474, 2684717064, 3607279774, 1203610895, 817534361, 2847130659, 3736401077, 1087398166, 936857984, + 2933784634, 3654889644, 1422998873, 601230799, 3135200373, 3453512931, 1404893504, 616286678, 3182598252, + 3400902906, 1510651243, 755860989, 3020215367, 3271812305, 1567060338, 710951396, 3010007134, 3295551688, + 1913130485, 84884835, 2617666777, 3942734927, 1969605100, 40040826, 2607524032, 3966539862, 2094237127, 198489425, + 2464015595, 3856323709, 2076066270, 213479752, 2511347954, 3803648100, 1874795921, 414723335, 2175892669, + 4139142187, 1758648712, 534112542, 2262612132, 4057696306, 1633981859, 375629109, 2406151311, 4167943193, + 1711886778, 286155052, 2282172566, 4278190080}; + +const uint32_t crc32_reverse_table[] = {258633766, 3558569575, 1660517093, 3112729764, 3561776544, 255681505, + 3118428003, 1655597346, 1649892715, 3106298666, 264525736, 3568652777, 3111209197, 1644187308, 3571597870, + 261309551, 3574606524, 241249533, 3095601279, 1676813886, 238436154, 3578214779, 1671771641, 3100914616, 3089958897, + 1666974128, 3586002226, 248449907, 1661670007, 3095008310, 244848820, 3588824821, 1626325843, 3147744530, 226281872, + 3591229393, 3150567125, 1622724756, 3596278806, 220977751, 231927326, 3601066079, 1614931165, 3140543132, + 3606379416, 226885081, 3144151387, 1612117786, 3130366409, 1642373000, 3607532298, 209163595, 1639156815, + 3133311502, 203458188, 3612442829, 3618157700, 215593669, 3124477511, 1632286726, 210673922, 3623855939, 1629334465, + 3127684480, 3496023756, 185602189, 3186760719, 1719867982, 189203274, 3493201163, 1725172105, 3181711304, + 3176938369, 1714240960, 3503239490, 197015299, 1719283207, 3171625030, 199828676, 3499631237, 169561174, 3513403927, + 1736984213, 3170451668, 3510458832, 172777361, 3165541139, 1742689618, 1730568475, 3159844698, 179661784, + 3519311257, 3154146461, 1735488220, 3516104286, 182614047, 3219432889, 1687528440, 3531059066, 151431483, + 1690480703, 3216225918, 156351228, 3525360829, 3537471732, 162041525, 3209331255, 1681622134, 167746930, 3532561203, + 1684838321, 3206386160, 1704918819, 3203397986, 135132640, 3548181409, 3199789733, 1707732196, 3542868070, + 140174887, 144953966, 3553809455, 1697700013, 3191987948, 3548760040, 150258089, 3189165355, 1701301098, 1779284915, + 2977742322, 127840624, 3706697521, 2972429877, 1784326260, 3703088374, 130654903, 120624894, 3695284415, 1789107261, + 2983369340, 3692462968, 124224825, 2978318779, 1794412538, 2960646441, 1795614568, 3722767338, 110489003, + 1800533167, 2954949358, 113442412, 3719559213, 3712666724, 104581669, 2967062183, 1806221542, 107796962, 3709722531, + 1811927841, 2962150752, 92801222, 3740864135, 1746600453, 3010069572, 3735952704, 98507521, 3007125379, 1749815746, + 1756702091, 3015975882, 86388552, 3730254089, 3012767757, 1759655500, 3724557006, 91307151, 3757199964, 75715613, + 2992724127, 1762680542, 81020890, 3752149403, 1766280473, 2989902680, 2999942929, 1774090576, 3747378642, 70087571, + 1776904855, 2996333782, 75128916, 3742066197, 3037125977, 1853348632, 3633698714, 65327579, 1847642335, 3042037406, + 62112284, 3636642909, 3627806740, 55244373, 3047750359, 1859779734, 52290962, 3631014867, 1854861137, 3053447440, + 1870432195, 3020784002, 49253632, 3651046209, 3025834565, 1865126916, 3653867654, 45653703, 37857934, 3643845839, + 1876074573, 3030623756, 3647454984, 35043657, 3035936203, 1871033226, 3666046508, 32663661, 3071304943, 1818321582, + 27622314, 3671358955, 1815507305, 3074914088, 3082699617, 1825522976, 3660401058, 22826979, 1821923047, 3085521062, + 17521700, 3665451621, 16331958, 3683136247, 1835679349, 3055237172, 3688833328, 11413361, 3058445299, 1832725938, + 1841568251, 3065323450, 5706552, 3676706169, 3068267645, 1838352956, 3681617598, 255}; + +} // namespace common \ No newline at end of file diff --git a/src/common/CRC32.hpp b/src/common/CRC32.hpp new file mode 100644 index 00000000..9e5252b3 --- /dev/null +++ b/src/common/CRC32.hpp @@ -0,0 +1,28 @@ +// Copyright (c) 2012-2018, The CryptoNote developers, The Bytecoin developers. +// Licensed under the GNU Lesser General Public License. See LICENSE for details. + +#pragma once + +#include +#include + +namespace common { + +extern const uint32_t crc32_table[]; + +inline uint32_t crc32_step_zero(uint32_t state) { return (state >> 8) ^ crc32_table[state & 0xff]; } + +inline uint32_t crc32_step(uint32_t state, char data) { return crc32_step_zero(state ^ data); } + +inline uint32_t crc32(const char *data, size_t size, uint32_t state = 0) { + for (const char *cur = data; cur != data + size; cur++) { + state = crc32_step(state, *cur); + } + return state; +} + +extern const uint32_t crc32_reverse_table[]; + +inline uint32_t crc32_reverse_step_zero(uint32_t state) { return (state << 8) ^ crc32_reverse_table[state >> 24]; } + +} // namespace common \ No newline at end of file diff --git a/src/common/CRC32_generate.py b/src/common/CRC32_generate.py new file mode 100644 index 00000000..193ebcfa --- /dev/null +++ b/src/common/CRC32_generate.py @@ -0,0 +1,29 @@ +# Copyright (c) 2012-2018, The CryptoNote developers, The Bytecoin developers. +# Licensed under the GNU Lesser General Public License. See LICENSE for details. + +from sys import argv +from zlib import crc32 + +output = "CRC32.cpp" + +# When changing the script, run and commit artefacts + +crc32_reverse = [0] +for i in range(1, 256): + v = (crc32_reverse[i >> 1] << 1) ^ ((i & 1) << 32) + crc32_reverse.append(v if v & (1 << 32) == 0 else v ^ 0x1db710641) + +with open(output, 'w', encoding='ASCII', newline='') as fd: + fd.write( +f'''// This file is generated by CRC32_generate.py. +#include + +#include "CRC32.hpp" + +namespace common {{ + +const uint32_t crc32_table[] = {{{', '.join(f'{crc32(bytes((i,)))}' for i in range(256))}}}; + +const uint32_t crc32_reverse_table[] = {{{', '.join(f'{i ^ 0xff}' for i in reversed(crc32_reverse))}}}; + +}}''') \ No newline at end of file diff --git a/src/common/CommandLine.cpp b/src/common/CommandLine.cpp index 5d321289..30926252 100644 --- a/src/common/CommandLine.cpp +++ b/src/common/CommandLine.cpp @@ -2,8 +2,8 @@ // Licensed under the GNU Lesser General Public License. See LICENSE for details. #include "CommandLine.hpp" -#include #include +#include // no std::string - no std::cout, printf is synced with it anyway @@ -23,7 +23,7 @@ CommandLine::CommandLine(int argc, const char *const argv[]) { return; bool positional_only = false; std::vector