From bc60d37b821babac60fa7a51b295ca9d058f6334 Mon Sep 17 00:00:00 2001 From: Konstantin Munichev Date: Tue, 16 Apr 2019 20:17:52 +0300 Subject: [PATCH] Add option to load peers from a file on startup Signed-off-by: Konstantin Munichev --- irohad/ametsuchi/impl/storage_impl.cpp | 28 +++++++++ irohad/ametsuchi/impl/storage_impl.hpp | 5 ++ irohad/ametsuchi/storage.hpp | 12 ++++ irohad/main/CMakeLists.txt | 7 +++ irohad/main/application.cpp | 49 ++++++++++----- irohad/main/application.hpp | 36 ++++++----- irohad/main/impl/peers_file_reader_impl.cpp | 60 +++++++++++++++++++ irohad/main/impl/peers_file_reader_impl.hpp | 26 ++++++++ irohad/main/irohad.cpp | 33 ++++++++++ irohad/main/peers_file_reader.hpp | 43 +++++++++++++ .../integration_framework/iroha_instance.cpp | 2 + .../integration_framework/iroha_instance.hpp | 1 + .../integration_framework/test_irohad.hpp | 3 + test/module/irohad/ametsuchi/mock_storage.hpp | 2 + 14 files changed, 277 insertions(+), 30 deletions(-) create mode 100644 irohad/main/impl/peers_file_reader_impl.cpp create mode 100644 irohad/main/impl/peers_file_reader_impl.hpp create mode 100644 irohad/main/peers_file_reader.hpp diff --git a/irohad/ametsuchi/impl/storage_impl.cpp b/irohad/ametsuchi/impl/storage_impl.cpp index 5a386dc245..2514b9390d 100644 --- a/irohad/ametsuchi/impl/storage_impl.cpp +++ b/irohad/ametsuchi/impl/storage_impl.cpp @@ -16,12 +16,14 @@ #include "ametsuchi/impl/postgres_block_query.hpp" #include "ametsuchi/impl/postgres_command_executor.hpp" #include "ametsuchi/impl/postgres_query_executor.hpp" +#include "ametsuchi/impl/postgres_wsv_command.hpp" #include "ametsuchi/impl/postgres_wsv_query.hpp" #include "ametsuchi/impl/temporary_wsv_impl.hpp" #include "backend/protobuf/permissions.hpp" #include "common/bind.hpp" #include "common/byteutils.hpp" #include "converters/protobuf/json_proto_converter.hpp" +#include "cryptography/public_key.hpp" #include "logger/logger.hpp" #include "logger/logger_manager.hpp" @@ -264,6 +266,18 @@ namespace iroha { return inserted; } + bool StorageImpl::insertPeer(const shared_model::interface::Peer &peer) { + log_->info("insert peer {}", peer.pubkey().hex()); + soci::session sql(*connection_); + PostgresWsvCommand wsv_command(sql); + auto status = wsv_command.insertPeer(peer); + if (auto e = boost::get>(&status)) { + log_->error("Failed to insert peer: {}", e->error); + return false; + } + return true; + } + void StorageImpl::reset() { log_->info("drop wsv records from db tables"); try { @@ -280,6 +294,16 @@ namespace iroha { } } + void StorageImpl::resetPeers() { + log_->info("remove everything from peers table"); + try { + soci::session sql(*connection_); + sql << reset_peers_; + } catch (std::exception &e) { + log_->warn("peers reset failed, reason: {}", e.what()); + } + } + void StorageImpl::dropStorage() { log_->info("drop storage"); if (connection_ == nullptr) { @@ -657,6 +681,10 @@ TRUNCATE TABLE tx_status_by_hash RESTART IDENTITY CASCADE; TRUNCATE TABLE height_by_account_set RESTART IDENTITY CASCADE; TRUNCATE TABLE index_by_creator_height RESTART IDENTITY CASCADE; TRUNCATE TABLE position_by_account_asset RESTART IDENTITY CASCADE; +)"; + + const std::string &StorageImpl::reset_peers_ = R"( +TRUNCATE TABLE peer RESTART IDENTITY CASCADE; )"; const std::string &StorageImpl::init_ = diff --git a/irohad/ametsuchi/impl/storage_impl.hpp b/irohad/ametsuchi/impl/storage_impl.hpp index 2c9ef9cbb6..a7b598a416 100644 --- a/irohad/ametsuchi/impl/storage_impl.hpp +++ b/irohad/ametsuchi/impl/storage_impl.hpp @@ -95,8 +95,12 @@ namespace iroha { const std::vector> &blocks) override; + bool insertPeer(const shared_model::interface::Peer &peer) override; + void reset() override; + void resetPeers() override; + void dropStorage() override; void freeConnections() override; @@ -188,6 +192,7 @@ namespace iroha { protected: static const std::string &drop_; static const std::string &reset_; + static const std::string &reset_peers_; static const std::string &init_; }; } // namespace ametsuchi diff --git a/irohad/ametsuchi/storage.hpp b/irohad/ametsuchi/storage.hpp index 1089d2f631..5d71797b40 100644 --- a/irohad/ametsuchi/storage.hpp +++ b/irohad/ametsuchi/storage.hpp @@ -60,6 +60,13 @@ namespace iroha { const std::vector> &blocks) = 0; + /** + * Inserts peer into WSV + * @param peer - peer to insert + * @return true if inserted + */ + virtual bool insertPeer(const shared_model::interface::Peer &peer) = 0; + /** * method called when block is written to the storage * @return observable with the Block committed @@ -73,6 +80,11 @@ namespace iroha { */ virtual void reset() = 0; + /** + * Removes all saved peers + */ + virtual void resetPeers() = 0; + /** * Remove all information from ledger * Tables and the database will be removed too diff --git a/irohad/main/CMakeLists.txt b/irohad/main/CMakeLists.txt index 12ce6b1e53..209b750c6a 100644 --- a/irohad/main/CMakeLists.txt +++ b/irohad/main/CMakeLists.txt @@ -21,6 +21,12 @@ target_link_libraries(raw_block_loader logger ) +add_library(peers_file_reader impl/peers_file_reader_impl.cpp) +target_link_libraries(peers_file_reader + shared_model_interfaces + parser + ) + add_library(application application.cpp # TODO andrei 08.11.2018 IR-1851 Create separate targets for initialization @@ -61,6 +67,7 @@ add_executable(irohad irohad.cpp) target_link_libraries(irohad application raw_block_loader + peers_file_reader gflags rapidjson keys_manager diff --git a/irohad/main/application.cpp b/irohad/main/application.cpp index 9d06253e04..cea88e92d0 100644 --- a/irohad/main/application.cpp +++ b/irohad/main/application.cpp @@ -8,6 +8,7 @@ #include #include "ametsuchi/impl/flat_file_block_storage_factory.hpp" +#include "ametsuchi/impl/postgres_wsv_command.hpp" #include "ametsuchi/impl/storage_impl.hpp" #include "ametsuchi/impl/tx_presence_cache_impl.hpp" #include "ametsuchi/impl/wsv_restorer_impl.hpp" @@ -74,21 +75,23 @@ static constexpr iroha::consensus::yac::ConsistencyModel /** * Configuring iroha daemon */ -Irohad::Irohad(const std::string &block_store_dir, - const std::string &pg_conn, - const std::string &listen_ip, - size_t torii_port, - size_t internal_port, - size_t max_proposal_size, - std::chrono::milliseconds proposal_delay, - std::chrono::milliseconds vote_delay, - std::chrono::minutes mst_expiration_time, - const shared_model::crypto::Keypair &keypair, - std::chrono::milliseconds max_rounds_delay, - size_t stale_stream_max_rounds, - logger::LoggerManagerTreePtr logger_manager, - const boost::optional - &opt_mst_gossip_params) +Irohad::Irohad( + const std::string &block_store_dir, + const std::string &pg_conn, + const std::string &listen_ip, + size_t torii_port, + size_t internal_port, + size_t max_proposal_size, + std::chrono::milliseconds proposal_delay, + std::chrono::milliseconds vote_delay, + std::chrono::minutes mst_expiration_time, + const shared_model::crypto::Keypair &keypair, + std::chrono::milliseconds max_rounds_delay, + size_t stale_stream_max_rounds, + std::vector> initial_peers, + logger::LoggerManagerTreePtr logger_manager, + const boost::optional + &opt_mst_gossip_params) : block_store_dir_(block_store_dir), pg_conn_(pg_conn), listen_ip_(listen_ip), @@ -101,6 +104,7 @@ Irohad::Irohad(const std::string &block_store_dir, mst_expiration_time_(mst_expiration_time), max_rounds_delay_(max_rounds_delay), stale_stream_max_rounds_(stale_stream_max_rounds), + initial_peers_(std::move(initial_peers)), opt_mst_gossip_params_(opt_mst_gossip_params), keypair(keypair), ordering_init(logger_manager->getLogger()), @@ -129,6 +133,7 @@ void Irohad::init() { // Recover WSV from the existing ledger to be sure it is consistent initWsvRestorer(); restoreWsv(); + updatePeers(); initCryptoProvider(); initBatchParser(); @@ -203,6 +208,20 @@ bool Irohad::restoreWsv() { }); } +bool Irohad::updatePeers() { + // drop old peers and insert new ones if --peers flag is set + if (not initial_peers_.empty()) { + storage->resetPeers(); + for (const auto &peer : initial_peers_) { + if (not storage->insertPeer(*peer)) { + log_->error("Peer insertion failed"); + return false; + } + } + } + return true; +} + /** * Initializing crypto provider */ diff --git a/irohad/main/application.hpp b/irohad/main/application.hpp index ff04104a97..b6cf4123ed 100644 --- a/irohad/main/application.hpp +++ b/irohad/main/application.hpp @@ -93,26 +93,29 @@ class Irohad { * transactions * @param stale_stream_max_rounds - maximum number of rounds between * consecutive status emissions + * * @param logger_manager - the logger manager to use * @param opt_mst_gossip_params - parameters for Gossip MST propagation * (optional). If not provided, disables mst processing support * TODO mboldyrev 03.11.2018 IR-1844 Refactor the constructor. */ - Irohad(const std::string &block_store_dir, - const std::string &pg_conn, - const std::string &listen_ip, - size_t torii_port, - size_t internal_port, - size_t max_proposal_size, - std::chrono::milliseconds proposal_delay, - std::chrono::milliseconds vote_delay, - std::chrono::minutes mst_expiration_time, - const shared_model::crypto::Keypair &keypair, - std::chrono::milliseconds max_rounds_delay, - size_t stale_stream_max_rounds, - logger::LoggerManagerTreePtr logger_manager, - const boost::optional - &opt_mst_gossip_params = boost::none); + Irohad( + const std::string &block_store_dir, + const std::string &pg_conn, + const std::string &listen_ip, + size_t torii_port, + size_t internal_port, + size_t max_proposal_size, + std::chrono::milliseconds proposal_delay, + std::chrono::milliseconds vote_delay, + std::chrono::minutes mst_expiration_time, + const shared_model::crypto::Keypair &keypair, + std::chrono::milliseconds max_rounds_delay, + size_t stale_stream_max_rounds, + std::vector> initial_peers, + logger::LoggerManagerTreePtr logger_manager, + const boost::optional + &opt_mst_gossip_params = boost::none); /** * Initialization of whole objects in system @@ -125,6 +128,8 @@ class Irohad { */ bool restoreWsv(); + bool updatePeers(); + /** * Drop wsv and block store */ @@ -197,6 +202,7 @@ class Irohad { std::chrono::minutes mst_expiration_time_; std::chrono::milliseconds max_rounds_delay_; size_t stale_stream_max_rounds_; + std::vector> initial_peers_; boost::optional opt_mst_gossip_params_; diff --git a/irohad/main/impl/peers_file_reader_impl.cpp b/irohad/main/impl/peers_file_reader_impl.cpp new file mode 100644 index 0000000000..78cd9ca10e --- /dev/null +++ b/irohad/main/impl/peers_file_reader_impl.cpp @@ -0,0 +1,60 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "peers_file_reader_impl.hpp" + +#include + +#include "cryptography/public_key.hpp" +#include "interfaces/common_objects/types.hpp" +#include "parser/parser.hpp" + +using namespace iroha::main; + +boost::optional PeersFileReaderImpl::openFile( + const std::string &name) { + std::ifstream file(name); + if (not file) { + return boost::none; + } + + std::string str((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + return str; +} + +boost::optional>> +PeersFileReaderImpl::readPeers( + const std::string &peers_data, + std::shared_ptr + common_objects_factory) { + auto strings = parser::split(peers_data); + if (strings.size() % 2 != 0) { + return boost::none; + } + + std::vector> peers{}; + + for (uint32_t i = 0; i < strings.size(); i += 2) { + shared_model::interface::types::AddressType address = strings.at(i); + shared_model::interface::types::PubkeyType key( + shared_model::interface::types::PubkeyType::fromHexString( + strings.at(i + 1))); + auto peer = common_objects_factory->createPeer(address, key); + + if (auto e = boost::get>(&peer)) { + return boost::none; + } + + peers.emplace_back(std::move( + boost::get< + expected::Value>>( + &peer) + ->value)); + } + return boost::make_optional< + std::vector>>( + std::move(peers)); +} diff --git a/irohad/main/impl/peers_file_reader_impl.hpp b/irohad/main/impl/peers_file_reader_impl.hpp new file mode 100644 index 0000000000..264e9e8eff --- /dev/null +++ b/irohad/main/impl/peers_file_reader_impl.hpp @@ -0,0 +1,26 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_PEERS_FILE_READER_IMPL_HPP +#define IROHA_PEERS_FILE_READER_IMPL_HPP + +#include "main/peers_file_reader.hpp" + +namespace iroha { + namespace main { + class PeersFileReaderImpl : public PeersFileReader { + public: + boost::optional openFile(const std::string &name) override; + + boost::optional< + std::vector>> + readPeers(const std::string &peers_data, + std::shared_ptr + common_objects_factory) override; + }; + } // namespace main +} // namespace iroha + +#endif // IROHA_PEERS_FILE_READER_IMPL_HPP diff --git a/irohad/main/irohad.cpp b/irohad/main/irohad.cpp index fd71352c21..e2d5b7290c 100644 --- a/irohad/main/irohad.cpp +++ b/irohad/main/irohad.cpp @@ -10,15 +10,18 @@ #include #include #include "ametsuchi/storage.hpp" +#include "backend/protobuf/common_objects/proto_common_objects_factory.hpp" #include "common/irohad_version.hpp" #include "common/result.hpp" #include "crypto/keys_manager_impl.hpp" #include "logger/logger.hpp" #include "logger/logger_manager.hpp" #include "main/application.hpp" +#include "main/impl/peers_file_reader_impl.hpp" #include "main/iroha_conf_literals.hpp" #include "main/iroha_conf_loader.hpp" #include "main/raw_block_loader.hpp" +#include "validators/field_validator.hpp" static const std::string kListenIp = "0.0.0.0"; static const std::string kLogSettingsFromConfigFile = "config_file"; @@ -73,6 +76,11 @@ DEFINE_string(keypair_name, "", "Specify name of .pub and .priv files"); */ DEFINE_validator(keypair_name, &validate_keypair_name); +/** + * Create input argument for the initial peer list + */ +DEFINE_string(peers, "", "Specify initial peers"); + /** * Creating boolean flag for overwriting already existing block storage */ @@ -154,6 +162,30 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } + boost::optional>> + peers = boost::make_optional< + std::vector>>({}); + if (not FLAGS_peers.empty()) { + iroha::main::PeersFileReaderImpl file_reader; + auto peers_file_data = file_reader.openFile(FLAGS_peers); + if (not peers_file_data) { + log->error("Failed to load peers file"); + return EXIT_FAILURE; + } + + auto validators_config = + std::make_shared( + config.max_proposal_size); + peers = file_reader.readPeers( + *peers_file_data, + std::make_shared>(validators_config)); + if (not peers) { + log->error("Failed to parse peers file"); + return EXIT_FAILURE; + } + } + // Configuring iroha daemon Irohad irohad( config.block_store_path, @@ -171,6 +203,7 @@ int main(int argc, char *argv[]) { std::chrono::milliseconds( config.max_round_delay_ms.value_or(kMaxRoundsDelayDefault)), config.stale_stream_max_rounds.value_or(kStaleStreamMaxRoundsDefault), + std::move(*peers), log_manager->getChild("Irohad"), boost::make_optional(config.mst_support, iroha::GossipPropagationStrategyParams{})); diff --git a/irohad/main/peers_file_reader.hpp b/irohad/main/peers_file_reader.hpp new file mode 100644 index 0000000000..f6878e3e02 --- /dev/null +++ b/irohad/main/peers_file_reader.hpp @@ -0,0 +1,43 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_PEERS_FILE_READER_HPP +#define IROHA_PEERS_FILE_READER_HPP + +#include +#include "interfaces/common_objects/common_objects_factory.hpp" +#include "interfaces/common_objects/peer.hpp" + +namespace iroha { + namespace main { + /** + * Peers reader interface from a file + */ + class PeersFileReader { + public: + /** + * Opens file with peers and returns its contents + * @param name - file name to open + * @return optional for a file contents or boost::none + */ + virtual boost::optional openFile( + const std::string &name) = 0; + + /** + * Parses peers from a provided string + * @param peers_data - peers string to parse + * @param common_objects_factory - factory to create peers + * @return optional for collection of peers or boost::none + */ + virtual boost::optional< + std::vector>> + readPeers(const std::string &peers_data, + std::shared_ptr + common_objects_factory) = 0; + }; + } // namespace main +} // namespace iroha + +#endif // IROHA_PEERS_FILE_READER_HPP diff --git a/test/framework/integration_framework/iroha_instance.cpp b/test/framework/integration_framework/iroha_instance.cpp index 702f2d69b7..b6d7d33fd9 100644 --- a/test/framework/integration_framework/iroha_instance.cpp +++ b/test/framework/integration_framework/iroha_instance.cpp @@ -78,6 +78,7 @@ namespace integration_framework { void IrohaInstance::initPipeline( const shared_model::crypto::Keypair &key_pair, size_t max_proposal_size) { + auto peers = std::vector>{}; instance_ = std::make_shared(block_store_dir_, pg_conn_, listen_ip_, @@ -90,6 +91,7 @@ namespace integration_framework { key_pair, max_rounds_delay_, stale_stream_max_rounds_, + std::move(peers), irohad_log_manager_, log_, opt_mst_gossip_params_); diff --git a/test/framework/integration_framework/iroha_instance.hpp b/test/framework/integration_framework/iroha_instance.hpp index abb3db1021..7480e9afc4 100644 --- a/test/framework/integration_framework/iroha_instance.hpp +++ b/test/framework/integration_framework/iroha_instance.hpp @@ -21,6 +21,7 @@ namespace shared_model { namespace interface { class Block; + class Peer; } // namespace interface namespace crypto { class Keypair; diff --git a/test/framework/integration_framework/test_irohad.hpp b/test/framework/integration_framework/test_irohad.hpp index 5c16307793..4066521d87 100644 --- a/test/framework/integration_framework/test_irohad.hpp +++ b/test/framework/integration_framework/test_irohad.hpp @@ -28,6 +28,8 @@ namespace integration_framework { const shared_model::crypto::Keypair &keypair, std::chrono::milliseconds max_rounds_delay, size_t stale_stream_max_rounds, + std::vector> + initial_peers, logger::LoggerManagerTreePtr irohad_log_manager, logger::LoggerPtr log, const boost::optional @@ -44,6 +46,7 @@ namespace integration_framework { keypair, max_rounds_delay, stale_stream_max_rounds, + std::move(initial_peers), std::move(irohad_log_manager), opt_mst_gossip_params), log_(std::move(log)) {} diff --git a/test/module/irohad/ametsuchi/mock_storage.hpp b/test/module/irohad/ametsuchi/mock_storage.hpp index 140bb30e6c..4207e12fd9 100644 --- a/test/module/irohad/ametsuchi/mock_storage.hpp +++ b/test/module/irohad/ametsuchi/mock_storage.hpp @@ -45,7 +45,9 @@ namespace iroha { MOCK_METHOD1(insertBlocks, bool(const std::vector< std::shared_ptr> &)); + MOCK_METHOD1(insertPeer, bool(const shared_model::interface::Peer &)); MOCK_METHOD0(reset, void(void)); + MOCK_METHOD0(resetPeers, void(void)); MOCK_METHOD0(dropStorage, void(void)); MOCK_METHOD0(freeConnections, void(void)); MOCK_METHOD1(prepareBlock_, void(std::unique_ptr &));