Merge pull request #59 from cajun-rat/import-root-keys

Import initial root.json from the filesystem

CMakeLists.txt

@@ -37,8 +37,6 @@ set(SOTA_PACKED_CREDENTIALS "" CACHE STRING "Credentials.zip for tests involving
 set(TESTSUITE_ONLY "" CACHE STRING "Only run tests matching this list of labels")
 set(TESTSUITE_EXCLUDE "" CACHE STRING "Exclude tests matching this list of labels")
 
-set(STORAGE_TYPE "sqlite" CACHE STRING "")
-
 if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}")
     message(FATAL_ERROR "Aktualizr does not support building in the source tree. Please remove CMakeCache.txt and the CMakeFiles/ directory, then create a subdirectory to build in: mkdir build; cd build; cmake ..")
 endif()

src/libaktualizr/storage/CMakeLists.txt

@@ -6,29 +6,32 @@ add_custom_command(OUTPUT sql_schemas.cc sql_schemas_target
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
 )
 
-if(STORAGE_TYPE STREQUAL "sqlite")
-  set(SOURCES sqlstorage.cc sqlstorage_base.cc)
-  set(HEADERS sqlstorage.h sql_utils.h sqlstorage_base.h storage_exception.h)
-else()
-  message(FATAL_ERROR "Unknown storage type: ${storage_type}")
-endif()
+set(HEADERS fsstorage_read.h
+            invstorage.h
+            sql_utils.h
+            sqlstorage.h
+            sqlstorage_base.h
+            storage_exception.h)
 
-set(HEADERS ${HEADERS} fsstorage_read.h invstorage.h)
-set(SOURCES ${SOURCES} fsstorage_read.cc invstorage.cc)
+set(SOURCES fsstorage_read.cc
+            invstorage.cc
+            sqlstorage.cc
+            sqlstorage_base.cc)
 
 target_sources(config PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/storage_config.cc)
 
-if(STORAGE_TYPE STREQUAL "sqlite")
-  add_aktualizr_test(NAME storage_atomic SOURCES storage_atomic_test.cc PROJECT_WORKING_DIRECTORY)
-  add_aktualizr_test(NAME sql_utils SOURCES sql_utils_test.cc PROJECT_WORKING_DIRECTORY)
-  add_aktualizr_test(NAME sqlstorage SOURCES sqlstorage_test.cc ARGS ${CMAKE_CURRENT_SOURCE_DIR}/test)
-  list(REMOVE_ITEM TEST_SOURCES sql_schemas.cc)
-  add_aktualizr_test(NAME storage_common SOURCES storage_common_test.cc PROJECT_WORKING_DIRECTORY)
-
-  add_test(NAME test_schema_migration
-      COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/schema_migration_test.sh ${PROJECT_SOURCE_DIR}/config/sql)
-  set_tests_properties(test_schema_migration PROPERTIES LABELS "noptest")
-endif(STORAGE_TYPE STREQUAL "sqlite")
+add_aktualizr_test(NAME storage_atomic SOURCES storage_atomic_test.cc PROJECT_WORKING_DIRECTORY)
+add_aktualizr_test(NAME sql_utils SOURCES sql_utils_test.cc PROJECT_WORKING_DIRECTORY)
+add_aktualizr_test(NAME sqlstorage SOURCES sqlstorage_test.cc ARGS ${CMAKE_CURRENT_SOURCE_DIR}/test)
+add_aktualizr_test(NAME storage_common
+                   SOURCES storage_common_test.cc
+                   LIBRARIES uptane_generator_lib
+                   PROJECT_WORKING_DIRECTORY)
+
+add_test(NAME test_schema_migration
+         COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/schema_migration_test.sh ${PROJECT_SOURCE_DIR}/config/sql)
+
+set_tests_properties(test_schema_migration PROPERTIES LABELS "noptest")
 
 add_library(storage OBJECT ${SOURCES} sql_schemas.cc)
 

src/libaktualizr/storage/invstorage.cc

@@ -7,20 +7,6 @@
 #include "sqlstorage.h"
 #include "utilities/utils.h"
 
-void INvStorage::importSimple(const boost::filesystem::path& base_path, store_data_t store_func, load_data_t load_func,
-                              const utils::BasedPath& imported_data_path, const std::string& data_name) {
-  if (!(this->*load_func)(nullptr) && !imported_data_path.empty()) {
-    boost::filesystem::path abs_path = imported_data_path.get(base_path);
-    if (!boost::filesystem::exists(abs_path)) {
-      LOG_ERROR << "Couldn't import " << data_name << ": " << abs_path << " doesn't exist.";
-      return;
-    }
-    std::string content = Utils::readFile(abs_path.string());
-    (this->*store_func)(content);
-    LOG_DEBUG << "Successfully imported " << data_name << " from " << abs_path;
-  }
-}
-
 void INvStorage::importUpdateSimple(const boost::filesystem::path& base_path, store_data_t store_func,
                                     load_data_t load_func, const utils::BasedPath& imported_data_path,
                                     const std::string& data_name) {
@@ -132,6 +118,33 @@ void INvStorage::importInstalledVersions(const boost::filesystem::path& base_pat
   }
 }
 
+void INvStorage::importInitialRoot(const boost::filesystem::path& base_path) {
+  importInitialRootFile(base_path / "repo/root.json", Uptane::RepositoryType::Image());
+  importInitialRootFile(base_path / "director/root.json", Uptane::RepositoryType::Director());
+}
+
+void INvStorage::importInitialRootFile(const boost::filesystem::path& root_path, Uptane::RepositoryType repo_type) {
+  std::string root_tmp;  // Only needed for loadLatestRoot
+  if (!loadLatestRoot(&root_tmp, repo_type)) {
+    if (boost::filesystem::is_regular_file(root_path)) {
+      try {
+        std::string root_str = Utils::readFile(root_path);
+        Uptane::Root orig_root(Uptane::Root::Policy::kAcceptAll);
+        Uptane::Root new_root(repo_type, Utils::parseJSON(root_str), orig_root);
+        // No exception. Save it
+        storeRoot(root_str, repo_type, Uptane::Version(new_root.version()));
+        LOG_INFO << "Imported initial " << repo_type << " root keys from " << root_path;
+      } catch (Uptane::Exception& e) {
+        LOG_WARNING << "Couldn't import initial " << repo_type << " root keys from " << root_path << " " << e.what();
+      }
+    } else {
+      LOG_DEBUG << "Not importing " << root_path << " because it doesn't exist";
+    }
+  } else {
+    LOG_TRACE << "Root for " << repo_type << " already present, not importing";
+  }
+}
+
 void INvStorage::importData(const ImportConfig& import_config) {
   importPrimaryKeys(import_config.base_path, import_config.uptane_public_key_path,
                     import_config.uptane_private_key_path);
@@ -141,6 +154,7 @@ void INvStorage::importData(const ImportConfig& import_config) {
   importUpdateSimple(import_config.base_path, &INvStorage::storeTlsPkey, &INvStorage::loadTlsPkey,
                      import_config.tls_pkey_path, "client TLS key");
   importInstalledVersions(import_config.base_path);
+  importInitialRoot(import_config.base_path);
 }
 
 std::shared_ptr<INvStorage> INvStorage::newStorage(const StorageConfig& config, const bool readonly) {

src/libaktualizr/storage/invstorage.h

@@ -147,8 +147,6 @@ class INvStorage {
   virtual std::vector<std::string> getAllTargetNames() const = 0;
   virtual void deleteTargetInfo(const std::string& targetname) const = 0;
 
-  virtual void cleanUp() = 0;
-
   // Special constructors and utilities
   static std::shared_ptr<INvStorage> newStorage(const StorageConfig& config, bool readonly = false);
   static void FSSToSQLS(FSStorageRead& fs_storage, SQLStorage& sql_storage);
@@ -170,14 +168,23 @@ class INvStorage {
   void importInstalledVersions(const boost::filesystem::path& base_path);
 
  private:
-  void importSimple(const boost::filesystem::path& base_path, store_data_t store_func, load_data_t load_func,
-                    const utils::BasedPath& imported_data_path, const std::string& data_name);
   void importUpdateSimple(const boost::filesystem::path& base_path, store_data_t store_func, load_data_t load_func,
                           const utils::BasedPath& imported_data_path, const std::string& data_name);
   void importUpdateCertificate(const boost::filesystem::path& base_path, const utils::BasedPath& imported_data_path);
   void importPrimaryKeys(const boost::filesystem::path& base_path, const utils::BasedPath& import_pubkey_path,
                          const utils::BasedPath& import_privkey_path);
 
+  /**
+   * Import initial image and director root.json from the filesystem.
+   * These would be loaded onto the device during provisioning at a well-known
+   * location such as /var/sota/import/repo/root.json (image repo) and
+   * /var/sota/import/director/root.json for the director repo.
+   *
+   * @param base_path e.g. '/var/sota/import'
+   */
+  void importInitialRoot(const boost::filesystem::path& base_path);
+  void importInitialRootFile(const boost::filesystem::path& root_path, Uptane::RepositoryType repo_type);
+
  protected:
   const StorageConfig config_;
 };

src/libaktualizr/storage/sqlstorage.cc

@@ -1730,5 +1730,3 @@ void SQLStorage::deleteTargetInfo(const std::string& targetname) const {
     throw SQLException(std::string("Failed to clear Target filenames: ") + db.errmsg());
   }
 }
-
-void SQLStorage::cleanUp() { boost::filesystem::remove_all(dbPath()); }

src/libaktualizr/storage/sqlstorage.h

@@ -110,7 +110,6 @@ class SQLStorage : public SQLStorageBase, public INvStorage {
   std::vector<std::string> getAllTargetNames() const override;
   void deleteTargetInfo(const std::string& targetname) const override;
 
-  void cleanUp() override;
   StorageType type() override { return StorageType::kSqlite; };
 
  private:

src/libaktualizr/storage/storage_common_test.cc

@@ -6,24 +6,20 @@
 #include <boost/filesystem.hpp>
 
 #include "libaktualizr/types.h"
+#include "repo.h"
 #include "storage/sqlstorage.h"
 #include "utilities/utils.h"
 
-StorageType current_storage_type{StorageType::kSqlite};
+namespace fs = boost::filesystem;
 
-std::unique_ptr<INvStorage> Storage(const boost::filesystem::path &dir) {
+std::unique_ptr<INvStorage> Storage(const fs::path &dir) {
   StorageConfig storage_config;
-  storage_config.type = current_storage_type;
+  storage_config.type = StorageType::kSqlite;
   storage_config.path = dir;
-
-  if (storage_config.type == StorageType::kSqlite) {
-    return std::unique_ptr<INvStorage>(new SQLStorage(storage_config, false));
-  } else {
-    throw std::runtime_error("Invalid config type");
-  }
+  return std::unique_ptr<INvStorage>(new SQLStorage(storage_config, false));
 }
 
-StorageConfig MakeConfig(StorageType type, const boost::filesystem::path &storage_dir) {
+StorageConfig MakeConfig(StorageType type, const fs::path &storage_dir) {
   StorageConfig config;
 
   config.type = type;
@@ -531,7 +527,7 @@ TEST(StorageCommon, LoadStoreSecondaryInfo) {
 TEST(StorageImport, ImportData) {
   TemporaryDirectory temp_dir;
   std::unique_ptr<INvStorage> storage = Storage(temp_dir.Path());
-  boost::filesystem::create_directories(temp_dir / "import");
+  fs::create_directories(temp_dir / "import");
 
   ImportConfig import_config;
   import_config.base_path = temp_dir.Path() / "import";
@@ -647,16 +643,48 @@ TEST(StorageImport, ImportData) {
   EXPECT_EQ(tls_pkey, tls_pkey_in3);
 }
 
+TEST(StorageImport, ImportInitialRoot) {
+  TemporaryDirectory temp_dir;
+  std::unique_ptr<INvStorage> storage = Storage(temp_dir.Path());
+  fs::create_directories(temp_dir / "import");
+
+  ImportConfig import_config;
+  import_config.base_path = temp_dir.Path() / "import";
+
+  // Generate a set of valid Uptane root keys
+  auto repo_path = temp_dir.Path() / "repo";
+  Repo image_repo{Uptane::RepositoryType::Image(), repo_path, "", ""};
+  image_repo.generateRepo();
+  Repo director_repo{Uptane::RepositoryType::Director(), repo_path, "", ""};
+  director_repo.generateRepo();
+  director_repo.rotate(Uptane::Role::Root());
+
+  EXPECT_FALSE(storage->loadLatestRoot(nullptr, Uptane::RepositoryType::Image()));
+  EXPECT_FALSE(storage->loadLatestRoot(nullptr, Uptane::RepositoryType::Director()));
+
+  fs::create_directories(import_config.base_path / "repo");
+  fs::create_directories(import_config.base_path / "director");
+
+  fs::copy(repo_path / "repo/repo/root.json", import_config.base_path / "repo/root.json");
+  Utils::writeFile(import_config.base_path / "director/root.json", std::string("invalid"));
+
+  storage->importData(import_config);
+  EXPECT_TRUE(storage->loadLatestRoot(nullptr, Uptane::RepositoryType::Image()));
+  EXPECT_FALSE(storage->loadLatestRoot(nullptr, Uptane::RepositoryType::Director()))
+      << "Director root.json was invalid. It shouldn't have been imported";
+
+  // Copy the real director root.json over
+  fs::copy_file(repo_path / "repo/director/root.json", import_config.base_path / "director/root.json",
+                fs::copy_option::overwrite_if_exists);
+  storage->importData(import_config);
+  EXPECT_TRUE(storage->loadLatestRoot(nullptr, Uptane::RepositoryType::Director()));
+}
+
 #ifndef __NO_MAIN__
 int main(int argc, char **argv) {
   ::testing::InitGoogleTest(&argc, argv);
   logger_init();
   logger_set_threshold(boost::log::trivial::trace);
-
-  std::cout << "Running tests for SQLStorage" << std::endl;
-  current_storage_type = StorageType::kSqlite;
-  int res_sql = RUN_ALL_TESTS();
-
-  return res_sql;  // 0 indicates success
+  return RUN_ALL_TESTS();
 }
 #endif

src/uptane_generator/repo.cc

@@ -14,11 +14,7 @@ Repo::Repo(Uptane::RepositoryType repo_type, boost::filesystem::path path, const
            std::string correlation_id)
     : repo_type_(repo_type), path_(std::move(path)), correlation_id_(std::move(correlation_id)) {
   expiration_time_ = getExpirationTime(expires);
-  if (boost::filesystem::exists(path_)) {
-    if (boost::filesystem::directory_iterator(path_) != boost::filesystem::directory_iterator()) {
-      readKeys();
-    }
-  }
+  readKeys();
 
   if (repo_type_ == Uptane::RepositoryType::Director()) {
     repo_dir_ = path_ / DirectorRepo::dir;
@@ -284,6 +280,9 @@ Json::Value Repo::getTarget(const std::string &target_name) {
 
 void Repo::readKeys() {
   auto keys_path = path_ / "keys" / repo_type_.ToString();
+  if (!boost::filesystem::exists(keys_path)) {
+    return;
+  }
   for (auto &p : boost::filesystem::directory_iterator(keys_path)) {
     std::string public_key_string = Utils::readFile(p / "public.key");
     std::istringstream key_type_str(Utils::readFile(p / "key_type"));