From bf22569019749e421e8ffe0a73cff988a9a9c846 Mon Sep 17 00:00:00 2001 From: Jude Melton-Houghton Date: Fri, 7 Jan 2022 13:28:49 -0500 Subject: [PATCH] Use a database for mod storage (#11763) --- doc/minetest.6 | 4 + src/client/client.cpp | 33 ++- src/client/client.h | 3 +- src/content/mods.cpp | 85 ++------ src/content/mods.h | 10 +- src/content/subgames.cpp | 1 + src/database/database-dummy.cpp | 38 ++++ src/database/database-dummy.h | 9 +- src/database/database-files.cpp | 136 +++++++++++- src/database/database-files.h | 26 +++ src/database/database-sqlite3.cpp | 105 +++++++++ src/database/database-sqlite3.h | 25 +++ src/database/database.h | 13 ++ src/gamedef.h | 3 +- src/main.cpp | 5 + src/script/lua_api/l_storage.cpp | 18 +- src/server.cpp | 140 ++++++++++-- src/server.h | 11 +- src/unittest/CMakeLists.txt | 1 + src/unittest/test.cpp | 6 +- src/unittest/test_modmetadatadatabase.cpp | 253 ++++++++++++++++++++++ 21 files changed, 798 insertions(+), 127 deletions(-) create mode 100644 src/unittest/test_modmetadatadatabase.cpp diff --git a/doc/minetest.6 b/doc/minetest.6 index 42ed1a45f..6a3601f80 100644 --- a/doc/minetest.6 +++ b/doc/minetest.6 @@ -112,6 +112,10 @@ leveldb, and files. Migrate from current players backend to another. Possible values are sqlite3, leveldb, postgresql, dummy, and files. .TP +.B \-\-migrate-mod-storage +Migrate from current mod storage backend to another. Possible values are +sqlite3, dummy, and files. +.TP .B \-\-terminal Display an interactive terminal over ncurses during execution. diff --git a/src/client/client.cpp b/src/client/client.cpp index 6e4a90a79..2caa953e4 100644 --- a/src/client/client.cpp +++ b/src/client/client.cpp @@ -128,6 +128,11 @@ Client::Client( // Add local player m_env.setLocalPlayer(new LocalPlayer(this, playername)); + // Make the mod storage database and begin the save for later + m_mod_storage_database = + new ModMetadataDatabaseSQLite3(porting::path_user + DIR_DELIM + "client"); + m_mod_storage_database->beginSave(); + if (g_settings->getBool("enable_minimap")) { m_minimap = new Minimap(this); } @@ -305,6 +310,11 @@ Client::~Client() m_minimap = nullptr; delete m_media_downloader; + + // Write the changes and delete + if (m_mod_storage_database) + m_mod_storage_database->endSave(); + delete m_mod_storage_database; } void Client::connect(Address address, bool is_local_server) @@ -641,19 +651,12 @@ void Client::step(float dtime) } } + // Write changes to the mod storage m_mod_storage_save_timer -= dtime; if (m_mod_storage_save_timer <= 0.0f) { m_mod_storage_save_timer = g_settings->getFloat("server_map_save_interval"); - int n = 0; - for (std::unordered_map::const_iterator - it = m_mod_storages.begin(); it != m_mod_storages.end(); ++it) { - if (it->second->isModified()) { - it->second->save(getModStoragePath()); - n++; - } - } - if (n > 0) - infostream << "Saved " << n << " modified mod storages." << std::endl; + m_mod_storage_database->endSave(); + m_mod_storage_database->beginSave(); } // Write server map @@ -1960,16 +1963,8 @@ void Client::unregisterModStorage(const std::string &name) { std::unordered_map::const_iterator it = m_mod_storages.find(name); - if (it != m_mod_storages.end()) { - // Save unconditionaly on unregistration - it->second->save(getModStoragePath()); + if (it != m_mod_storages.end()) m_mod_storages.erase(name); - } -} - -std::string Client::getModStoragePath() const -{ - return porting::path_user + DIR_DELIM + "client" + DIR_DELIM + "mod_storage"; } /* diff --git a/src/client/client.h b/src/client/client.h index bae40f389..694cd7d1b 100644 --- a/src/client/client.h +++ b/src/client/client.h @@ -380,8 +380,8 @@ public: { return checkPrivilege(priv); } virtual scene::IAnimatedMesh* getMesh(const std::string &filename, bool cache = false); const std::string* getModFile(std::string filename); + ModMetadataDatabase *getModStorageDatabase() override { return m_mod_storage_database; } - std::string getModStoragePath() const override; bool registerModStorage(ModMetadata *meta) override; void unregisterModStorage(const std::string &name) override; @@ -590,6 +590,7 @@ private: // Client modding ClientScripting *m_script = nullptr; std::unordered_map m_mod_storages; + ModMetadataDatabase *m_mod_storage_database = nullptr; float m_mod_storage_save_timer = 10.0f; std::vector m_mods; StringMap m_mod_vfs; diff --git a/src/content/mods.cpp b/src/content/mods.cpp index 6f088a5b3..455506967 100644 --- a/src/content/mods.cpp +++ b/src/content/mods.cpp @@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include #include #include "content/mods.h" +#include "database/database.h" #include "filesys.h" #include "log.h" #include "content/subgames.h" @@ -422,83 +423,29 @@ ClientModConfiguration::ClientModConfiguration(const std::string &path) : } #endif -ModMetadata::ModMetadata(const std::string &mod_name) : m_mod_name(mod_name) +ModMetadata::ModMetadata(const std::string &mod_name, ModMetadataDatabase *database): + m_mod_name(mod_name), m_database(database) { + m_database->getModEntries(m_mod_name, &m_stringvars); } void ModMetadata::clear() { + for (const auto &pair : m_stringvars) { + m_database->removeModEntry(m_mod_name, pair.first); + } Metadata::clear(); - m_modified = true; -} - -bool ModMetadata::save(const std::string &root_path) -{ - Json::Value json; - for (StringMap::const_iterator it = m_stringvars.begin(); - it != m_stringvars.end(); ++it) { - json[it->first] = it->second; - } - - if (!fs::PathExists(root_path)) { - if (!fs::CreateAllDirs(root_path)) { - errorstream << "ModMetadata[" << m_mod_name - << "]: Unable to save. '" << root_path - << "' tree cannot be created." << std::endl; - return false; - } - } else if (!fs::IsDir(root_path)) { - errorstream << "ModMetadata[" << m_mod_name << "]: Unable to save. '" - << root_path << "' is not a directory." << std::endl; - return false; - } - - bool w_ok = fs::safeWriteToFile( - root_path + DIR_DELIM + m_mod_name, fastWriteJson(json)); - - if (w_ok) { - m_modified = false; - } else { - errorstream << "ModMetadata[" << m_mod_name << "]: failed write file." - << std::endl; - } - return w_ok; -} - -bool ModMetadata::load(const std::string &root_path) -{ - m_stringvars.clear(); - - std::ifstream is((root_path + DIR_DELIM + m_mod_name).c_str(), - std::ios_base::binary); - if (!is.good()) { - return false; - } - - Json::Value root; - Json::CharReaderBuilder builder; - builder.settings_["collectComments"] = false; - std::string errs; - - if (!Json::parseFromStream(builder, is, &root, &errs)) { - errorstream << "ModMetadata[" << m_mod_name - << "]: failed read data " - "(Json decoding failure). Message: " - << errs << std::endl; - return false; - } - - const Json::Value::Members attr_list = root.getMemberNames(); - for (const auto &it : attr_list) { - Json::Value attr_value = root[it]; - m_stringvars[it] = attr_value.asString(); - } - - return true; } bool ModMetadata::setString(const std::string &name, const std::string &var) { - m_modified = Metadata::setString(name, var); - return m_modified; + if (Metadata::setString(name, var)) { + if (var.empty()) { + m_database->removeModEntry(m_mod_name, name); + } else { + m_database->setModEntry(m_mod_name, name, var); + } + return true; + } + return false; } diff --git a/src/content/mods.h b/src/content/mods.h index b56a97edb..dd3b6e0e6 100644 --- a/src/content/mods.h +++ b/src/content/mods.h @@ -31,6 +31,8 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "config.h" #include "metadata.h" +class ModMetadataDatabase; + #define MODNAME_ALLOWED_CHARS "abcdefghijklmnopqrstuvwxyz0123456789_" struct ModSpec @@ -149,20 +151,16 @@ class ModMetadata : public Metadata { public: ModMetadata() = delete; - ModMetadata(const std::string &mod_name); + ModMetadata(const std::string &mod_name, ModMetadataDatabase *database); ~ModMetadata() = default; virtual void clear(); - bool save(const std::string &root_path); - bool load(const std::string &root_path); - - bool isModified() const { return m_modified; } const std::string &getModName() const { return m_mod_name; } virtual bool setString(const std::string &name, const std::string &var); private: std::string m_mod_name; - bool m_modified = false; + ModMetadataDatabase *m_database; }; diff --git a/src/content/subgames.cpp b/src/content/subgames.cpp index 30447c838..e834f40cd 100644 --- a/src/content/subgames.cpp +++ b/src/content/subgames.cpp @@ -358,6 +358,7 @@ void loadGameConfAndInitWorld(const std::string &path, const std::string &name, conf.set("backend", "sqlite3"); conf.set("player_backend", "sqlite3"); conf.set("auth_backend", "sqlite3"); + conf.set("mod_storage_backend", "sqlite3"); conf.setBool("creative_mode", g_settings->getBool("creative_mode")); conf.setBool("enable_damage", g_settings->getBool("enable_damage")); diff --git a/src/database/database-dummy.cpp b/src/database/database-dummy.cpp index b56f341c5..629b2fb04 100644 --- a/src/database/database-dummy.cpp +++ b/src/database/database-dummy.cpp @@ -80,3 +80,41 @@ void Database_Dummy::listPlayers(std::vector &res) res.emplace_back(player); } } + +bool Database_Dummy::getModEntries(const std::string &modname, StringMap *storage) +{ + const auto mod_pair = m_mod_meta_database.find(modname); + if (mod_pair != m_mod_meta_database.cend()) { + for (const auto &pair : mod_pair->second) { + (*storage)[pair.first] = pair.second; + } + } + return true; +} + +bool Database_Dummy::setModEntry(const std::string &modname, + const std::string &key, const std::string &value) +{ + auto mod_pair = m_mod_meta_database.find(modname); + if (mod_pair == m_mod_meta_database.end()) { + m_mod_meta_database[modname] = StringMap({{key, value}}); + } else { + mod_pair->second[key] = value; + } + return true; +} + +bool Database_Dummy::removeModEntry(const std::string &modname, const std::string &key) +{ + auto mod_pair = m_mod_meta_database.find(modname); + if (mod_pair != m_mod_meta_database.end()) + return mod_pair->second.erase(key) > 0; + return false; +} + +void Database_Dummy::listMods(std::vector *res) +{ + for (const auto &pair : m_mod_meta_database) { + res->push_back(pair.first); + } +} diff --git a/src/database/database-dummy.h b/src/database/database-dummy.h index b69919f84..44b9e8d68 100644 --- a/src/database/database-dummy.h +++ b/src/database/database-dummy.h @@ -24,7 +24,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "database.h" #include "irrlichttypes.h" -class Database_Dummy : public MapDatabase, public PlayerDatabase +class Database_Dummy : public MapDatabase, public PlayerDatabase, public ModMetadataDatabase { public: bool saveBlock(const v3s16 &pos, const std::string &data); @@ -37,10 +37,17 @@ public: bool removePlayer(const std::string &name); void listPlayers(std::vector &res); + bool getModEntries(const std::string &modname, StringMap *storage); + bool setModEntry(const std::string &modname, + const std::string &key, const std::string &value); + bool removeModEntry(const std::string &modname, const std::string &key); + void listMods(std::vector *res); + void beginSave() {} void endSave() {} private: std::map m_database; std::set m_player_database; + std::unordered_map m_mod_meta_database; }; diff --git a/src/database/database-files.cpp b/src/database/database-files.cpp index d9d113b4e..9021ae61b 100644 --- a/src/database/database-files.cpp +++ b/src/database/database-files.cpp @@ -18,7 +18,6 @@ with this program; if not, write to the Free Software Foundation, Inc., */ #include -#include #include "convert_json.h" #include "database-files.h" #include "remoteplayer.h" @@ -376,3 +375,138 @@ bool AuthDatabaseFiles::writeAuthFile() } return true; } + +ModMetadataDatabaseFiles::ModMetadataDatabaseFiles(const std::string &savedir): + m_storage_dir(savedir + DIR_DELIM + "mod_storage") +{ +} + +bool ModMetadataDatabaseFiles::getModEntries(const std::string &modname, StringMap *storage) +{ + Json::Value *meta = getOrCreateJson(modname); + if (!meta) + return false; + + const Json::Value::Members attr_list = meta->getMemberNames(); + for (const auto &it : attr_list) { + Json::Value attr_value = (*meta)[it]; + (*storage)[it] = attr_value.asString(); + } + + return true; +} + +bool ModMetadataDatabaseFiles::setModEntry(const std::string &modname, + const std::string &key, const std::string &value) +{ + Json::Value *meta = getOrCreateJson(modname); + if (!meta) + return false; + + (*meta)[key] = Json::Value(value); + m_modified.insert(modname); + + return true; +} + +bool ModMetadataDatabaseFiles::removeModEntry(const std::string &modname, + const std::string &key) +{ + Json::Value *meta = getOrCreateJson(modname); + if (!meta) + return false; + + Json::Value removed; + if (meta->removeMember(key, &removed)) { + m_modified.insert(modname); + return true; + } + return false; +} + +void ModMetadataDatabaseFiles::beginSave() +{ +} + +void ModMetadataDatabaseFiles::endSave() +{ + if (!fs::CreateAllDirs(m_storage_dir)) { + errorstream << "ModMetadataDatabaseFiles: Unable to save. '" << m_storage_dir + << "' tree cannot be created." << std::endl; + return; + } + + for (auto it = m_modified.begin(); it != m_modified.end();) { + const std::string &modname = *it; + + if (!fs::PathExists(m_storage_dir)) { + if (!fs::CreateAllDirs(m_storage_dir)) { + errorstream << "ModMetadataDatabaseFiles[" << modname + << "]: Unable to save. '" << m_storage_dir + << "' tree cannot be created." << std::endl; + ++it; + continue; + } + } else if (!fs::IsDir(m_storage_dir)) { + errorstream << "ModMetadataDatabaseFiles[" << modname << "]: Unable to save. '" + << m_storage_dir << "' is not a directory." << std::endl; + ++it; + continue; + } + + const Json::Value &json = m_mod_meta[modname]; + + if (!fs::safeWriteToFile(m_storage_dir + DIR_DELIM + modname, fastWriteJson(json))) { + errorstream << "ModMetadataDatabaseFiles[" << modname + << "]: failed write file." << std::endl; + ++it; + continue; + } + + it = m_modified.erase(it); + } +} + +void ModMetadataDatabaseFiles::listMods(std::vector *res) +{ + // List in-memory metadata first. + for (const auto &pair : m_mod_meta) { + res->push_back(pair.first); + } + + // List other metadata present in the filesystem. + for (const auto &entry : fs::GetDirListing(m_storage_dir)) { + if (!entry.dir && m_mod_meta.count(entry.name) == 0) + res->push_back(entry.name); + } +} + +Json::Value *ModMetadataDatabaseFiles::getOrCreateJson(const std::string &modname) +{ + auto found = m_mod_meta.find(modname); + if (found == m_mod_meta.end()) { + fs::CreateAllDirs(m_storage_dir); + + Json::Value meta(Json::objectValue); + + std::string path = m_storage_dir + DIR_DELIM + modname; + if (fs::PathExists(path)) { + std::ifstream is(path.c_str(), std::ios_base::binary); + + Json::CharReaderBuilder builder; + builder.settings_["collectComments"] = false; + std::string errs; + + if (!Json::parseFromStream(builder, is, &meta, &errs)) { + errorstream << "ModMetadataDatabaseFiles[" << modname + << "]: failed read data (Json decoding failure). Message: " + << errs << std::endl; + return nullptr; + } + } + + return &(m_mod_meta[modname] = meta); + } else { + return &found->second; + } +} diff --git a/src/database/database-files.h b/src/database/database-files.h index e647a2e24..962e4d7bb 100644 --- a/src/database/database-files.h +++ b/src/database/database-files.h @@ -25,6 +25,8 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "database.h" #include +#include +#include class PlayerDatabaseFiles : public PlayerDatabase { @@ -69,3 +71,27 @@ private: bool readAuthFile(); bool writeAuthFile(); }; + +class ModMetadataDatabaseFiles : public ModMetadataDatabase +{ +public: + ModMetadataDatabaseFiles(const std::string &savedir); + virtual ~ModMetadataDatabaseFiles() = default; + + virtual bool getModEntries(const std::string &modname, StringMap *storage); + virtual bool setModEntry(const std::string &modname, + const std::string &key, const std::string &value); + virtual bool removeModEntry(const std::string &modname, const std::string &key); + virtual void listMods(std::vector *res); + + virtual void beginSave(); + virtual void endSave(); + +private: + Json::Value *getOrCreateJson(const std::string &modname); + bool writeJson(const std::string &modname, const Json::Value &json); + + std::string m_storage_dir; + std::unordered_map m_mod_meta; + std::unordered_set m_modified; +}; diff --git a/src/database/database-sqlite3.cpp b/src/database/database-sqlite3.cpp index 898acc265..e9442118e 100644 --- a/src/database/database-sqlite3.cpp +++ b/src/database/database-sqlite3.cpp @@ -779,3 +779,108 @@ void AuthDatabaseSQLite3::writePrivileges(const AuthEntry &authEntry) sqlite3_reset(m_stmt_write_privs); } } + +ModMetadataDatabaseSQLite3::ModMetadataDatabaseSQLite3(const std::string &savedir): + Database_SQLite3(savedir, "mod_storage"), ModMetadataDatabase() +{ +} + +ModMetadataDatabaseSQLite3::~ModMetadataDatabaseSQLite3() +{ + FINALIZE_STATEMENT(m_stmt_remove) + FINALIZE_STATEMENT(m_stmt_set) + FINALIZE_STATEMENT(m_stmt_get) +} + +void ModMetadataDatabaseSQLite3::createDatabase() +{ + assert(m_database); // Pre-condition + + SQLOK(sqlite3_exec(m_database, + "CREATE TABLE IF NOT EXISTS `entries` (\n" + " `modname` TEXT NOT NULL,\n" + " `key` BLOB NOT NULL,\n" + " `value` BLOB NOT NULL,\n" + " PRIMARY KEY (`modname`, `key`)\n" + ");\n", + NULL, NULL, NULL), + "Failed to create database table"); +} + +void ModMetadataDatabaseSQLite3::initStatements() +{ + PREPARE_STATEMENT(get, "SELECT `key`, `value` FROM `entries` WHERE `modname` = ?"); + PREPARE_STATEMENT(set, + "REPLACE INTO `entries` (`modname`, `key`, `value`) VALUES (?, ?, ?)"); + PREPARE_STATEMENT(remove, "DELETE FROM `entries` WHERE `modname` = ? AND `key` = ?"); +} + +bool ModMetadataDatabaseSQLite3::getModEntries(const std::string &modname, StringMap *storage) +{ + verifyDatabase(); + + str_to_sqlite(m_stmt_get, 1, modname); + while (sqlite3_step(m_stmt_get) == SQLITE_ROW) { + const char *key_data = (const char *) sqlite3_column_blob(m_stmt_get, 0); + size_t key_len = sqlite3_column_bytes(m_stmt_get, 0); + const char *value_data = (const char *) sqlite3_column_blob(m_stmt_get, 1); + size_t value_len = sqlite3_column_bytes(m_stmt_get, 1); + (*storage)[std::string(key_data, key_len)] = std::string(value_data, value_len); + } + sqlite3_vrfy(sqlite3_errcode(m_database), SQLITE_DONE); + + sqlite3_reset(m_stmt_get); + + return true; +} + +bool ModMetadataDatabaseSQLite3::setModEntry(const std::string &modname, + const std::string &key, const std::string &value) +{ + verifyDatabase(); + + str_to_sqlite(m_stmt_set, 1, modname); + SQLOK(sqlite3_bind_blob(m_stmt_set, 2, key.data(), key.size(), NULL), + "Internal error: failed to bind query at " __FILE__ ":" TOSTRING(__LINE__)); + SQLOK(sqlite3_bind_blob(m_stmt_set, 3, value.data(), value.size(), NULL), + "Internal error: failed to bind query at " __FILE__ ":" TOSTRING(__LINE__)); + SQLRES(sqlite3_step(m_stmt_set), SQLITE_DONE, "Failed to set mod entry") + + sqlite3_reset(m_stmt_set); + + return true; +} + +bool ModMetadataDatabaseSQLite3::removeModEntry(const std::string &modname, + const std::string &key) +{ + verifyDatabase(); + + str_to_sqlite(m_stmt_remove, 1, modname); + SQLOK(sqlite3_bind_blob(m_stmt_remove, 2, key.data(), key.size(), NULL), + "Internal error: failed to bind query at " __FILE__ ":" TOSTRING(__LINE__)); + sqlite3_vrfy(sqlite3_step(m_stmt_remove), SQLITE_DONE); + int changes = sqlite3_changes(m_database); + + sqlite3_reset(m_stmt_remove); + + return changes > 0; +} + +void ModMetadataDatabaseSQLite3::listMods(std::vector *res) +{ + verifyDatabase(); + + char *errmsg; + int status = sqlite3_exec(m_database, + "SELECT `modname` FROM `entries` GROUP BY `modname`;", + [](void *res_vp, int n_col, char **cols, char **col_names) -> int { + ((decltype(res)) res_vp)->emplace_back(cols[0]); + return 0; + }, (void *) res, &errmsg); + if (status != SQLITE_OK) { + DatabaseException e(std::string("Error trying to list mods with metadata: ") + errmsg); + sqlite3_free(errmsg); + throw e; + } +} diff --git a/src/database/database-sqlite3.h b/src/database/database-sqlite3.h index d7202a918..5e3d7c96c 100644 --- a/src/database/database-sqlite3.h +++ b/src/database/database-sqlite3.h @@ -232,3 +232,28 @@ private: sqlite3_stmt *m_stmt_delete_privs = nullptr; sqlite3_stmt *m_stmt_last_insert_rowid = nullptr; }; + +class ModMetadataDatabaseSQLite3 : private Database_SQLite3, public ModMetadataDatabase +{ +public: + ModMetadataDatabaseSQLite3(const std::string &savedir); + virtual ~ModMetadataDatabaseSQLite3(); + + virtual bool getModEntries(const std::string &modname, StringMap *storage); + virtual bool setModEntry(const std::string &modname, + const std::string &key, const std::string &value); + virtual bool removeModEntry(const std::string &modname, const std::string &key); + virtual void listMods(std::vector *res); + + virtual void beginSave() { Database_SQLite3::beginSave(); } + virtual void endSave() { Database_SQLite3::endSave(); } + +protected: + virtual void createDatabase(); + virtual void initStatements(); + +private: + sqlite3_stmt *m_stmt_get = nullptr; + sqlite3_stmt *m_stmt_set = nullptr; + sqlite3_stmt *m_stmt_remove = nullptr; +}; diff --git a/src/database/database.h b/src/database/database.h index b7d551935..fbb5befea 100644 --- a/src/database/database.h +++ b/src/database/database.h @@ -25,6 +25,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "irr_v3d.h" #include "irrlichttypes.h" #include "util/basic_macros.h" +#include "util/string.h" class Database { @@ -84,3 +85,15 @@ public: virtual void listNames(std::vector &res) = 0; virtual void reload() = 0; }; + +class ModMetadataDatabase : public Database +{ +public: + virtual ~ModMetadataDatabase() = default; + + virtual bool getModEntries(const std::string &modname, StringMap *storage) = 0; + virtual bool setModEntry(const std::string &modname, + const std::string &key, const std::string &value) = 0; + virtual bool removeModEntry(const std::string &modname, const std::string &key) = 0; + virtual void listMods(std::vector *res) = 0; +}; diff --git a/src/gamedef.h b/src/gamedef.h index bc0ee14c3..8a9246da2 100644 --- a/src/gamedef.h +++ b/src/gamedef.h @@ -33,6 +33,7 @@ class EmergeManager; class Camera; class ModChannel; class ModMetadata; +class ModMetadataDatabase; namespace irr { namespace scene { class IAnimatedMesh; @@ -70,9 +71,9 @@ public: virtual const std::vector &getMods() const = 0; virtual const ModSpec* getModSpec(const std::string &modname) const = 0; virtual std::string getWorldPath() const { return ""; } - virtual std::string getModStoragePath() const = 0; virtual bool registerModStorage(ModMetadata *storage) = 0; virtual void unregisterModStorage(const std::string &name) = 0; + virtual ModMetadataDatabase *getModStorageDatabase() = 0; virtual bool joinModChannel(const std::string &channel) = 0; virtual bool leaveModChannel(const std::string &channel) = 0; diff --git a/src/main.cpp b/src/main.cpp index 1044b327a..ca95ef874 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -299,6 +299,8 @@ static void set_allowed_options(OptionList *allowed_options) _("Migrate from current players backend to another (Only works when using minetestserver or with --server)")))); allowed_options->insert(std::make_pair("migrate-auth", ValueSpec(VALUETYPE_STRING, _("Migrate from current auth backend to another (Only works when using minetestserver or with --server)")))); + allowed_options->insert(std::make_pair("migrate-mod-storage", ValueSpec(VALUETYPE_STRING, + _("Migrate from current mod storage backend to another (Only works when using minetestserver or with --server)")))); allowed_options->insert(std::make_pair("terminal", ValueSpec(VALUETYPE_FLAG, _("Feature an interactive terminal (Only works when using minetestserver or with --server)")))); allowed_options->insert(std::make_pair("recompress", ValueSpec(VALUETYPE_FLAG, @@ -886,6 +888,9 @@ static bool run_dedicated_server(const GameParams &game_params, const Settings & if (cmd_args.exists("migrate-auth")) return ServerEnvironment::migrateAuthDatabase(game_params, cmd_args); + if (cmd_args.exists("migrate-mod-storage")) + return Server::migrateModStorageDatabase(game_params, cmd_args); + if (cmd_args.getFlag("recompress")) return recompress_map_database(game_params, cmd_args, bind_addr); diff --git a/src/script/lua_api/l_storage.cpp b/src/script/lua_api/l_storage.cpp index 978b315d5..b8f4347a8 100644 --- a/src/script/lua_api/l_storage.cpp +++ b/src/script/lua_api/l_storage.cpp @@ -32,19 +32,23 @@ int ModApiStorage::l_get_mod_storage(lua_State *L) std::string mod_name = readParam(L, -1); - ModMetadata *store = new ModMetadata(mod_name); + ModMetadata *store = nullptr; + if (IGameDef *gamedef = getGameDef(L)) { - store->load(gamedef->getModStoragePath()); - gamedef->registerModStorage(store); + store = new ModMetadata(mod_name, gamedef->getModStorageDatabase()); + if (gamedef->registerModStorage(store)) { + StorageRef::create(L, store); + int object = lua_gettop(L); + lua_pushvalue(L, object); + return 1; + } } else { - delete store; assert(false); // this should not happen } - StorageRef::create(L, store); - int object = lua_gettop(L); + delete store; - lua_pushvalue(L, object); + lua_pushnil(L); return 1; } diff --git a/src/server.cpp b/src/server.cpp index a910185b9..6cf790de1 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -66,6 +66,10 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "server/player_sao.h" #include "server/serverinventorymgr.h" #include "translation.h" +#include "database/database-sqlite3.h" +#include "database/database-files.h" +#include "database/database-dummy.h" +#include "gameparams.h" class ClientNotFoundException : public BaseException { @@ -344,10 +348,15 @@ Server::~Server() delete m_thread; } + // Write any changes before deletion. + if (m_mod_storage_database) + m_mod_storage_database->endSave(); + // Delete things in the reverse order of creation delete m_emerge; delete m_env; delete m_rollback; + delete m_mod_storage_database; delete m_banmanager; delete m_itemdef; delete m_nodedef; @@ -393,6 +402,10 @@ void Server::init() std::string ban_path = m_path_world + DIR_DELIM "ipban.txt"; m_banmanager = new BanManager(ban_path); + // Create mod storage database and begin a save for later + m_mod_storage_database = openModStorageDatabase(m_path_world); + m_mod_storage_database->beginSave(); + m_modmgr = std::unique_ptr(new ServerModManager(m_path_world)); std::vector unsatisfied_mods = m_modmgr->getUnsatisfiedMods(); // complain about mods with unsatisfied dependencies @@ -733,20 +746,12 @@ void Server::AsyncRunStep(bool initial_step) } m_clients.unlock(); - // Save mod storages if modified + // Write changes to the mod storage m_mod_storage_save_timer -= dtime; if (m_mod_storage_save_timer <= 0.0f) { m_mod_storage_save_timer = g_settings->getFloat("server_map_save_interval"); - int n = 0; - for (std::unordered_map::const_iterator - it = m_mod_storages.begin(); it != m_mod_storages.end(); ++it) { - if (it->second->isModified()) { - it->second->save(getModStoragePath()); - n++; - } - } - if (n > 0) - infostream << "Saved " << n << " modified mod storages." << std::endl; + m_mod_storage_database->endSave(); + m_mod_storage_database->beginSave(); } } @@ -3689,11 +3694,6 @@ std::string Server::getBuiltinLuaPath() return porting::path_share + DIR_DELIM + "builtin"; } -std::string Server::getModStoragePath() const -{ - return m_path_world + DIR_DELIM + "mod_storage"; -} - v3f Server::findSpawnPos() { ServerMap &map = m_env->getServerMap(); @@ -3857,11 +3857,8 @@ bool Server::registerModStorage(ModMetadata *storage) void Server::unregisterModStorage(const std::string &name) { std::unordered_map::const_iterator it = m_mod_storages.find(name); - if (it != m_mod_storages.end()) { - // Save unconditionaly on unregistration - it->second->save(getModStoragePath()); + if (it != m_mod_storages.end()) m_mod_storages.erase(name); - } } void dedicated_server_loop(Server &server, bool &kill) @@ -3999,3 +3996,106 @@ Translations *Server::getTranslationLanguage(const std::string &lang_code) return translations; } + +ModMetadataDatabase *Server::openModStorageDatabase(const std::string &world_path) +{ + std::string world_mt_path = world_path + DIR_DELIM + "world.mt"; + Settings world_mt; + if (!world_mt.readConfigFile(world_mt_path.c_str())) + throw BaseException("Cannot read world.mt!"); + + std::string backend = world_mt.exists("mod_storage_backend") ? + world_mt.get("mod_storage_backend") : "files"; + if (backend == "files") + warningstream << "/!\\ You are using the old mod storage files backend. " + << "This backend is deprecated and may be removed in a future release /!\\" + << std::endl << "Switching to SQLite3 is advised, " + << "please read http://wiki.minetest.net/Database_backends." << std::endl; + + return openModStorageDatabase(backend, world_path, world_mt); +} + +ModMetadataDatabase *Server::openModStorageDatabase(const std::string &backend, + const std::string &world_path, const Settings &world_mt) +{ + if (backend == "sqlite3") + return new ModMetadataDatabaseSQLite3(world_path); + + if (backend == "files") + return new ModMetadataDatabaseFiles(world_path); + + if (backend == "dummy") + return new Database_Dummy(); + + throw BaseException("Mod storage database backend " + backend + " not supported"); +} + +bool Server::migrateModStorageDatabase(const GameParams &game_params, const Settings &cmd_args) +{ + std::string migrate_to = cmd_args.get("migrate-mod-storage"); + Settings world_mt; + std::string world_mt_path = game_params.world_path + DIR_DELIM + "world.mt"; + if (!world_mt.readConfigFile(world_mt_path.c_str())) { + errorstream << "Cannot read world.mt!" << std::endl; + return false; + } + + std::string backend = world_mt.exists("mod_storage_backend") ? + world_mt.get("mod_storage_backend") : "files"; + if (backend == migrate_to) { + errorstream << "Cannot migrate: new backend is same" + << " as the old one" << std::endl; + return false; + } + + ModMetadataDatabase *srcdb = nullptr; + ModMetadataDatabase *dstdb = nullptr; + + bool succeeded = false; + + try { + srcdb = Server::openModStorageDatabase(backend, game_params.world_path, world_mt); + dstdb = Server::openModStorageDatabase(migrate_to, game_params.world_path, world_mt); + + dstdb->beginSave(); + + std::vector mod_list; + srcdb->listMods(&mod_list); + for (const std::string &modname : mod_list) { + StringMap meta; + srcdb->getModEntries(modname, &meta); + for (const auto &pair : meta) { + dstdb->setModEntry(modname, pair.first, pair.second); + } + } + + dstdb->endSave(); + + succeeded = true; + + actionstream << "Successfully migrated the metadata of " + << mod_list.size() << " mods" << std::endl; + world_mt.set("mod_storage_backend", migrate_to); + if (!world_mt.updateConfigFile(world_mt_path.c_str())) + errorstream << "Failed to update world.mt!" << std::endl; + else + actionstream << "world.mt updated" << std::endl; + + } catch (BaseException &e) { + errorstream << "An error occurred during migration: " << e.what() << std::endl; + } + + delete srcdb; + delete dstdb; + + if (succeeded && backend == "files") { + // Back up files + const std::string storage_path = game_params.world_path + DIR_DELIM + "mod_storage"; + const std::string backup_path = game_params.world_path + DIR_DELIM + "mod_storage.bak"; + if (!fs::Rename(storage_path, backup_path)) + warningstream << "After migration, " << storage_path + << " could not be renamed to " << backup_path << std::endl; + } + + return succeeded; +} diff --git a/src/server.h b/src/server.h index c5db0fdfb..12158feb7 100644 --- a/src/server.h +++ b/src/server.h @@ -283,6 +283,7 @@ public: virtual u16 allocateUnknownNodeId(const std::string &name); IRollbackManager *getRollbackManager() { return m_rollback; } virtual EmergeManager *getEmergeManager() { return m_emerge; } + virtual ModMetadataDatabase *getModStorageDatabase() { return m_mod_storage_database; } IWritableItemDefManager* getWritableItemDefManager(); NodeDefManager* getWritableNodeDefManager(); @@ -293,7 +294,6 @@ public: void getModNames(std::vector &modlist); std::string getBuiltinLuaPath(); virtual std::string getWorldPath() const { return m_path_world; } - virtual std::string getModStoragePath() const; inline bool isSingleplayer() { return m_simple_singleplayer_mode; } @@ -377,6 +377,14 @@ public: // Get or load translations for a language Translations *getTranslationLanguage(const std::string &lang_code); + static ModMetadataDatabase *openModStorageDatabase(const std::string &world_path); + + static ModMetadataDatabase *openModStorageDatabase(const std::string &backend, + const std::string &world_path, const Settings &world_mt); + + static bool migrateModStorageDatabase(const GameParams &game_params, + const Settings &cmd_args); + // Bind address Address m_bind_addr; @@ -678,6 +686,7 @@ private: s32 nextSoundId(); std::unordered_map m_mod_storages; + ModMetadataDatabase *m_mod_storage_database = nullptr; float m_mod_storage_save_timer = 10.0f; // CSM restrictions byteflag diff --git a/src/unittest/CMakeLists.txt b/src/unittest/CMakeLists.txt index 4d295e4ed..ce7921b55 100644 --- a/src/unittest/CMakeLists.txt +++ b/src/unittest/CMakeLists.txt @@ -14,6 +14,7 @@ set (UNITTEST_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/test_map_settings_manager.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_mapnode.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_modchannels.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/test_modmetadatadatabase.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_nodedef.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_noderesolver.cpp ${CMAKE_CURRENT_SOURCE_DIR}/test_noise.cpp diff --git a/src/unittest/test.cpp b/src/unittest/test.cpp index af324e1b1..f223d567e 100644 --- a/src/unittest/test.cpp +++ b/src/unittest/test.cpp @@ -25,6 +25,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "gamedef.h" #include "modchannels.h" #include "content/mods.h" +#include "database/database-dummy.h" #include "util/numeric.h" #include "porting.h" @@ -55,6 +56,7 @@ public: scene::ISceneManager *getSceneManager() { return m_scenemgr; } IRollbackManager *getRollbackManager() { return m_rollbackmgr; } EmergeManager *getEmergeManager() { return m_emergemgr; } + ModMetadataDatabase *getModStorageDatabase() { return m_mod_storage_database; } scene::IAnimatedMesh *getMesh(const std::string &filename) { return NULL; } bool checkLocalPrivilege(const std::string &priv) { return false; } @@ -68,7 +70,6 @@ public: return testmodspec; } virtual const ModSpec* getModSpec(const std::string &modname) const { return NULL; } - virtual std::string getModStoragePath() const { return "."; } virtual bool registerModStorage(ModMetadata *meta) { return true; } virtual void unregisterModStorage(const std::string &name) {} bool joinModChannel(const std::string &channel); @@ -89,11 +90,13 @@ private: scene::ISceneManager *m_scenemgr = nullptr; IRollbackManager *m_rollbackmgr = nullptr; EmergeManager *m_emergemgr = nullptr; + ModMetadataDatabase *m_mod_storage_database = nullptr; std::unique_ptr m_modchannel_mgr; }; TestGameDef::TestGameDef() : + m_mod_storage_database(new Database_Dummy()), m_modchannel_mgr(new ModChannelMgr()) { m_itemdef = createItemDefManager(); @@ -107,6 +110,7 @@ TestGameDef::~TestGameDef() { delete m_itemdef; delete m_nodedef; + delete m_mod_storage_database; } diff --git a/src/unittest/test_modmetadatadatabase.cpp b/src/unittest/test_modmetadatadatabase.cpp new file mode 100644 index 000000000..be97fae5e --- /dev/null +++ b/src/unittest/test_modmetadatadatabase.cpp @@ -0,0 +1,253 @@ +/* +Minetest +Copyright (C) 2018 bendeutsch, Ben Deutsch +Copyright (C) 2021 TurkeyMcMac, Jude Melton-Houghton + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as published by +the Free Software Foundation; either version 2.1 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along +with this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +*/ + +// This file is an edited copy of test_authdatabase.cpp + +#include "test.h" + +#include +#include "database/database-files.h" +#include "database/database-sqlite3.h" +#include "filesys.h" + +namespace +{ +// Anonymous namespace to create classes that are only +// visible to this file +// +// These are helpers that return a *ModMetadataDatabase and +// allow us to run the same tests on different databases and +// database acquisition strategies. + +class ModMetadataDatabaseProvider +{ +public: + virtual ~ModMetadataDatabaseProvider() = default; + virtual ModMetadataDatabase *getModMetadataDatabase() = 0; +}; + +class FixedProvider : public ModMetadataDatabaseProvider +{ +public: + FixedProvider(ModMetadataDatabase *mod_meta_db) : mod_meta_db(mod_meta_db){}; + virtual ~FixedProvider(){}; + virtual ModMetadataDatabase *getModMetadataDatabase() { return mod_meta_db; }; + +private: + ModMetadataDatabase *mod_meta_db; +}; + +class FilesProvider : public ModMetadataDatabaseProvider +{ +public: + FilesProvider(const std::string &dir) : dir(dir){}; + virtual ~FilesProvider() + { + if (mod_meta_db) + mod_meta_db->endSave(); + delete mod_meta_db; + } + virtual ModMetadataDatabase *getModMetadataDatabase() + { + if (mod_meta_db) + mod_meta_db->endSave(); + delete mod_meta_db; + mod_meta_db = new ModMetadataDatabaseFiles(dir); + mod_meta_db->beginSave(); + return mod_meta_db; + }; + +private: + std::string dir; + ModMetadataDatabase *mod_meta_db = nullptr; +}; + +class SQLite3Provider : public ModMetadataDatabaseProvider +{ +public: + SQLite3Provider(const std::string &dir) : dir(dir){}; + virtual ~SQLite3Provider() + { + if (mod_meta_db) + mod_meta_db->endSave(); + delete mod_meta_db; + } + virtual ModMetadataDatabase *getModMetadataDatabase() + { + if (mod_meta_db) + mod_meta_db->endSave(); + delete mod_meta_db; + mod_meta_db = new ModMetadataDatabaseSQLite3(dir); + mod_meta_db->beginSave(); + return mod_meta_db; + }; + +private: + std::string dir; + ModMetadataDatabase *mod_meta_db = nullptr; +}; +} + +class TestModMetadataDatabase : public TestBase +{ +public: + TestModMetadataDatabase() { TestManager::registerTestModule(this); } + const char *getName() { return "TestModMetadataDatabase"; } + + void runTests(IGameDef *gamedef); + void runTestsForCurrentDB(); + + void testRecallFail(); + void testCreate(); + void testRecall(); + void testChange(); + void testRecallChanged(); + void testListMods(); + void testRemove(); + +private: + ModMetadataDatabaseProvider *mod_meta_provider; +}; + +static TestModMetadataDatabase g_test_instance; + +void TestModMetadataDatabase::runTests(IGameDef *gamedef) +{ + // fixed directory, for persistence + thread_local const std::string test_dir = getTestTempDirectory(); + + // Each set of tests is run twice for each database type: + // one where we reuse the same ModMetadataDatabase object (to test local caching), + // and one where we create a new ModMetadataDatabase object for each call + // (to test actual persistence). + + rawstream << "-------- Files database (same object)" << std::endl; + + ModMetadataDatabase *mod_meta_db = new ModMetadataDatabaseFiles(test_dir); + mod_meta_provider = new FixedProvider(mod_meta_db); + + runTestsForCurrentDB(); + + delete mod_meta_db; + delete mod_meta_provider; + + // reset database + fs::RecursiveDelete(test_dir + DIR_DELIM + "mod_storage"); + + rawstream << "-------- Files database (new objects)" << std::endl; + + mod_meta_provider = new FilesProvider(test_dir); + + runTestsForCurrentDB(); + + delete mod_meta_provider; + + rawstream << "-------- SQLite3 database (same object)" << std::endl; + + mod_meta_db = new ModMetadataDatabaseSQLite3(test_dir); + mod_meta_provider = new FixedProvider(mod_meta_db); + + runTestsForCurrentDB(); + + delete mod_meta_db; + delete mod_meta_provider; + + // reset database + fs::DeleteSingleFileOrEmptyDirectory(test_dir + DIR_DELIM + "mod_storage.sqlite"); + + rawstream << "-------- SQLite3 database (new objects)" << std::endl; + + mod_meta_provider = new SQLite3Provider(test_dir); + + runTestsForCurrentDB(); + + delete mod_meta_provider; +} + +//////////////////////////////////////////////////////////////////////////////// + +void TestModMetadataDatabase::runTestsForCurrentDB() +{ + TEST(testRecallFail); + TEST(testCreate); + TEST(testRecall); + TEST(testChange); + TEST(testRecallChanged); + TEST(testListMods); + TEST(testRemove); + TEST(testRecallFail); +} + +void TestModMetadataDatabase::testRecallFail() +{ + ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase(); + StringMap recalled; + mod_meta_db->getModEntries("mod1", &recalled); + UASSERT(recalled.empty()); +} + +void TestModMetadataDatabase::testCreate() +{ + ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase(); + StringMap recalled; + UASSERT(mod_meta_db->setModEntry("mod1", "key1", "value1")); +} + +void TestModMetadataDatabase::testRecall() +{ + ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase(); + StringMap recalled; + mod_meta_db->getModEntries("mod1", &recalled); + UASSERT(recalled.size() == 1); + UASSERT(recalled["key1"] == "value1"); +} + +void TestModMetadataDatabase::testChange() +{ + ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase(); + StringMap recalled; + UASSERT(mod_meta_db->setModEntry("mod1", "key1", "value2")); +} + +void TestModMetadataDatabase::testRecallChanged() +{ + ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase(); + StringMap recalled; + mod_meta_db->getModEntries("mod1", &recalled); + UASSERT(recalled.size() == 1); + UASSERT(recalled["key1"] == "value2"); +} + +void TestModMetadataDatabase::testListMods() +{ + ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase(); + UASSERT(mod_meta_db->setModEntry("mod2", "key1", "value1")); + std::vector mod_list; + mod_meta_db->listMods(&mod_list); + UASSERT(mod_list.size() == 2); + UASSERT(std::find(mod_list.cbegin(), mod_list.cend(), "mod1") != mod_list.cend()); + UASSERT(std::find(mod_list.cbegin(), mod_list.cend(), "mod2") != mod_list.cend()); +} + +void TestModMetadataDatabase::testRemove() +{ + ModMetadataDatabase *mod_meta_db = mod_meta_provider->getModMetadataDatabase(); + UASSERT(mod_meta_db->removeModEntry("mod1", "key1")); +}