Automatic database backend

nano/core_test/toml.cpp

@@ -578,7 +578,6 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	vacancy_threshold = 99
 
 	[node.rocksdb]
-	enable = true
 	io_threads = 99
 	read_cache = 99
 	write_cache = 99
@@ -748,8 +747,6 @@ TEST (toml, daemon_config_deserialize_no_defaults)
 	ASSERT_NE (conf.node.lmdb_config.max_databases, defaults.node.lmdb_config.max_databases);
 	ASSERT_NE (conf.node.lmdb_config.map_size, defaults.node.lmdb_config.map_size);
 
-	ASSERT_TRUE (conf.node.rocksdb_config.enable);
-	ASSERT_EQ (nano::rocksdb_config::using_rocksdb_in_tests (), defaults.node.rocksdb_config.enable);
 	ASSERT_NE (conf.node.rocksdb_config.io_threads, defaults.node.rocksdb_config.io_threads);
 	ASSERT_NE (conf.node.rocksdb_config.read_cache, defaults.node.rocksdb_config.read_cache);
 	ASSERT_NE (conf.node.rocksdb_config.write_cache, defaults.node.rocksdb_config.write_cache);

nano/lib/rocksdbconfig.cpp

@@ -4,7 +4,6 @@
 
 nano::error nano::rocksdb_config::serialize_toml (nano::tomlconfig & toml) const
 {
-	toml.put ("enable", enable, "Whether to use the RocksDB backend for the ledger database.\ntype:bool");
 	toml.put ("io_threads", io_threads, "Number of threads to use with the background compaction and flushing.\ntype:uint32");
 	toml.put ("read_cache", read_cache, "Amount of megabytes per table allocated to read cache. Valid range is 1 - 1024. Default is 32.\nCarefully monitor memory usage if non-default values are used\ntype:long");
 	toml.put ("write_cache", write_cache, "Total amount of megabytes allocated to write cache. Valid range is 1 - 256. Default is 64.\nCarefully monitor memory usage if non-default values are used\ntype:long");
@@ -14,7 +13,6 @@ nano::error nano::rocksdb_config::serialize_toml (nano::tomlconfig & toml) const
 
 nano::error nano::rocksdb_config::deserialize_toml (nano::tomlconfig & toml)
 {
-	toml.get_optional<bool> ("enable", enable);
 	toml.get_optional<unsigned> ("io_threads", io_threads);
 	toml.get_optional<long> ("read_cache", read_cache);
 	toml.get_optional<long> ("write_cache", write_cache);

nano/lib/rocksdbconfig.hpp

@@ -13,11 +13,6 @@ class tomlconfig;
 class rocksdb_config final
 {
 public:
-	rocksdb_config () :
-		enable{ using_rocksdb_in_tests () }
-	{
-	}
-
 	nano::error serialize_toml (nano::tomlconfig &) const;
 	nano::error deserialize_toml (nano::tomlconfig &);
 

nano/node/make_store.cpp

@@ -3,12 +3,40 @@
 #include <nano/store/lmdb/lmdb.hpp>
 #include <nano/store/rocksdb/rocksdb.hpp>
 
-std::unique_ptr<nano::store::component> nano::make_store (nano::logger & logger, std::filesystem::path const & path, nano::ledger_constants & constants, bool read_only, bool add_db_postfix, nano::rocksdb_config const & rocksdb_config, nano::txn_tracking_config const & txn_tracking_config_a, std::chrono::milliseconds block_processor_batch_max_time_a, nano::lmdb_config const & lmdb_config_a, bool backup_before_upgrade)
+std::unique_ptr<nano::store::component> nano::make_store (nano::logger & logger, std::filesystem::path const & path, nano::ledger_constants & constants, bool read_only, bool add_db_postfix, nano::node_config const & node_config)
 {
-	if (rocksdb_config.enable)
+	if (node_config.database_backend == nano::database_backend::rocksdb)
 	{
-		return std::make_unique<nano::store::rocksdb::component> (logger, add_db_postfix ? path / "rocksdb" : path, constants, rocksdb_config, read_only);
+		return std::make_unique<nano::store::rocksdb::component> (logger, add_db_postfix ? path / "rocksdb" : path, constants, node_config.rocksdb_config, read_only);
 	}
-
-	return std::make_unique<nano::store::lmdb::component> (logger, add_db_postfix ? path / "data.ldb" : path, constants, txn_tracking_config_a, block_processor_batch_max_time_a, lmdb_config_a, backup_before_upgrade);
+	else if (node_config.database_backend == nano::database_backend::lmdb)
+	{
+		return std::make_unique<nano::store::lmdb::component> (logger, add_db_postfix ? path / "data.ldb" : path, constants, node_config.diagnostics_config.txn_tracking, node_config.block_processor_batch_max_time, node_config.lmdb_config, node_config.backup_before_upgrade);
+	}
+	else if (node_config.database_backend == nano::database_backend::automatic)
+	{
+		bool lmdb_ledger_found = std::filesystem::exists (path / "data.ldb");
+		bool rocks_ledger_found = std::filesystem::exists (path / "rocksdb");
+		if (lmdb_ledger_found && rocks_ledger_found)
+		{
+			logger.warn (nano::log::type::ledger, "Multiple ledgers were found! Using RocksDb ledger");
+			return std::make_unique<nano::store::rocksdb::component> (logger, add_db_postfix ? path / "rocksdb" : path, constants, node_config.rocksdb_config, read_only);
+		}
+		else if (lmdb_ledger_found)
+		{
+			logger.info (nano::log::type::ledger, "Using existing LMDB ledger");
+			return std::make_unique<nano::store::lmdb::component> (logger, add_db_postfix ? path / "data.ldb" : path, constants, node_config.diagnostics_config.txn_tracking, node_config.block_processor_batch_max_time, node_config.lmdb_config, node_config.backup_before_upgrade);
+		}
+		else if (rocks_ledger_found)
+		{
+			logger.info (nano::log::type::ledger, "Using existing RocksDb ledger");
+			return std::make_unique<nano::store::rocksdb::component> (logger, add_db_postfix ? path / "rocksdb" : path, constants, node_config.rocksdb_config, read_only);
+		}
+		else if (!lmdb_ledger_found && !rocks_ledger_found)
+		{
+			logger.info (nano::log::type::ledger, "No ledger found. Creating new LMDB ledger");
+			return std::make_unique<nano::store::lmdb::component> (logger, add_db_postfix ? path / "data.ldb" : path, constants, node_config.diagnostics_config.txn_tracking, node_config.block_processor_batch_max_time, node_config.lmdb_config, node_config.backup_before_upgrade);
+		}
+	}
+	debug_assert (false);
 }

nano/node/make_store.hpp

@@ -4,6 +4,7 @@
 #include <nano/lib/lmdbconfig.hpp>
 #include <nano/lib/logging.hpp>
 #include <nano/lib/rocksdbconfig.hpp>
+#include <nano/node/nodeconfig.hpp>
 
 #include <chrono>
 
@@ -12,6 +13,7 @@ namespace nano
 class ledger_constants;
 class lmdb_config;
 class rocksdb_config;
+class node_config;
 class txn_tracking_config;
 }
 
@@ -22,5 +24,6 @@ class component;
 
 namespace nano
 {
-std::unique_ptr<nano::store::component> make_store (nano::logger &, std::filesystem::path const & path, nano::ledger_constants & constants, bool open_read_only = false, bool add_db_postfix = true, nano::rocksdb_config const & rocksdb_config = nano::rocksdb_config{}, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false);
+std::unique_ptr<nano::store::component> make_store (
+nano::logger &, std::filesystem::path const & path, nano::ledger_constants & constants, bool open_read_only = false, bool add_db_postfix = true, nano::node_config const & node_config = nano::node_config{});
 }

nano/node/node.cpp

@@ -83,7 +83,7 @@ nano::node::node (std::shared_ptr<boost::asio::io_context> io_ctx_a, std::filesy
 	flags (flags_a),
 	work (work_a),
 	distributed_work (*this),
-	store_impl (nano::make_store (logger, application_path_a, network_params.ledger, flags.read_only, true, config_a.rocksdb_config, config_a.diagnostics_config.txn_tracking, config_a.block_processor_batch_max_time, config_a.lmdb_config, config_a.backup_before_upgrade)),
+	store_impl (nano::make_store (logger, application_path_a, network_params.ledger, flags.read_only, true, config_a)),
 	store (*store_impl),
 	unchecked{ config.max_unchecked_blocks, stats, flags.disable_block_processor_unchecked_deletion },
 	wallets_store_impl (std::make_unique<nano::mdb_wallets_store> (application_path_a / "wallets.ldb", config_a.lmdb_config)),

nano/node/nodeconfig.cpp

@@ -100,6 +100,7 @@ nano::error nano::node_config::serialize_toml (nano::tomlconfig & toml) const
 		toml.put ("peering_port", *peering_port, "Node peering port.\ntype:uint16");
 	}
 
+	toml.put ("database_backend", serialize_database_backend (database_backend), "Database used for storing the ledger. Default is auto\ntype:string,{auto,rocksdb,lmdb}");
 	toml.put ("bootstrap_fraction_numerator", bootstrap_fraction_numerator, "Change bootstrap threshold (online stake / 256 * bootstrap_fraction_numerator).\ntype:uint32");
 	toml.put ("receive_minimum", receive_minimum.to_string_dec (), "Minimum receive amount. Only affects node wallets. A large amount is recommended to avoid automatic work generation for tiny transactions.\ntype:string,amount,raw");
 	toml.put ("online_weight_minimum", online_weight_minimum.to_string_dec (), "When calculating online weight, the node is forced to assume at least this much voting weight is online, thus setting a floor for voting weight to confirm transactions at online_weight_minimum * \"quorum delta\".\ntype:string,amount,raw");
@@ -526,6 +527,12 @@ nano::error nano::node_config::deserialize_toml (nano::tomlconfig & toml)
 			lmdb_config.deserialize_toml (lmdb_config_l);
 		}
 
+		if (toml.has_key ("database_backend"))
+		{
+			auto database_backend_l (toml.get<std::string> ("database_backend"));
+			database_backend = deserialize_database_backend (database_backend_l);
+		}
+
 		boost::asio::ip::address_v6 external_address_l;
 		toml.get<boost::asio::ip::address_v6> ("external_address", external_address_l);
 		external_address = external_address_l.to_string ();
@@ -646,6 +653,38 @@ void nano::node_config::deserialize_address (std::string const & entry_a, std::v
 	}
 }
 
+std::string nano::node_config::serialize_database_backend (nano::database_backend mode_a) const
+{
+	switch (mode_a)
+	{
+		case nano::database_backend::automatic:
+			return "auto";
+		case nano::database_backend::rocksdb:
+			return "rocksdb";
+		case nano::database_backend::lmdb:
+			return "lmdb";
+		default:
+			return "auto";
+	}
+}
+
+nano::database_backend nano::node_config::deserialize_database_backend (std::string const & string_a)
+{
+	if (string_a == "auto")
+	{
+		return nano::database_backend::automatic;
+	}
+	else if (string_a == "rocksdb")
+	{
+		return nano::database_backend::rocksdb;
+	}
+	else if (string_a == "lmdb")
+	{
+		return nano::database_backend::lmdb;
+	}
+	debug_assert (false);
+}
+
 nano::account nano::node_config::random_representative () const
 {
 	debug_assert (!preconfigured_representatives.empty ());

nano/node/nodeconfig.hpp

@@ -41,6 +41,13 @@ namespace nano
 {
 class tomlconfig;
 
+enum class database_backend : uint8_t
+{
+	automatic, // backend is determined based on existing ledger files found
+	rocksdb,
+	lmdb
+};
+
 /**
  * Node configuration
  */
@@ -133,6 +140,8 @@ class node_config
 	uint64_t max_pruning_depth{ 0 };
 	nano::rocksdb_config rocksdb_config;
 	nano::lmdb_config lmdb_config;
+	nano::database_backend database_backend{ std::string (std::getenv ("BACKEND") ? std::getenv ("BACKEND") : "") == "rocksdb" ? nano::database_backend::rocksdb : std::string (std::getenv ("BACKEND") ? std::getenv ("BACKEND") : "") == "lmdb" ? nano::database_backend::lmdb
+																																																												  : nano::database_backend::automatic };
 	bool enable_upnp{ true };
 	nano::vote_cache_config vote_cache;
 	nano::rep_crawler_config rep_crawler;
@@ -152,6 +161,8 @@ class node_config
 public:
 	/** Entry is ignored if it cannot be parsed as a valid address:port */
 	void deserialize_address (std::string const &, std::vector<std::pair<std::string, uint16_t>> &) const;
+	std::string serialize_database_backend (nano::database_backend) const;
+	nano::database_backend deserialize_database_backend (std::string const & string_a);
 
 private:
 	static std::optional<unsigned> env_io_threads ();

nano/secure/ledger.cpp

@@ -1282,8 +1282,9 @@ bool nano::ledger::migrate_lmdb_to_rocksdb (std::filesystem::path const & data_p
 
 	// Open rocksdb database
 	nano::rocksdb_config rocksdb_config;
-	rocksdb_config.enable = true;
-	auto rocksdb_store = nano::make_store (logger, data_path_a, nano::dev::constants, false, true, rocksdb_config);
+	nano::node_config node_config;
+	node_config.database_backend = database_backend::rocksdb;
+	auto rocksdb_store = nano::make_store (logger, data_path_a, nano::dev::constants, false, true, node_config);
 
 	if (!rocksdb_store->init_error ())
 	{