backport: bitcoin#19137, #19253, #20365, #20687, #23349 - descriptor wallets part III

ci/test/00_setup_env_native_cxx20.sh

@@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8
 export CONTAINER_NAME=ci_native_cxx20
 export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev"
 export DEP_OPTS="NO_UPNP=1 DEBUG=1"
-export CPPFLAGS="-DDEBUG_LOCKORDER -DENABLE_DASH_DEBUG -DARENA_DEBUG"
+export CPPFLAGS="-DDEBUG_LOCKORDER -DARENA_DEBUG"
 export PYZMQ=true
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="install"

ci/test/00_setup_env_native_fuzz.sh

@@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8
 export CONTAINER_NAME=ci_native_fuzz
 export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev"
 export DEP_OPTS="NO_UPNP=1 DEBUG=1"
-export CPPFLAGS="-DDEBUG_LOCKORDER -DENABLE_DASH_DEBUG -DARENA_DEBUG"
+export CPPFLAGS="-DDEBUG_LOCKORDER -DARENA_DEBUG"
 export CXXFLAGS="-Werror -Wno-unused-command-line-argument -Wno-unused-value -Wno-deprecated-builtins"
 export PYZMQ=true
 export RUN_UNIT_TESTS=false

ci/test/00_setup_env_native_tsan.sh

@@ -13,5 +13,5 @@ export TEST_RUNNER_EXTRA="--extended --exclude feature_pruning,feature_dbcrash,w
 export TEST_RUNNER_EXTRA="${TEST_RUNNER_EXTRA} --timeout-factor=4"  # Increase timeout because sanitizers slow down
 export GOAL="install"
 export BITCOIN_CONFIG="--enable-zmq --with-gui=no --with-sanitizers=thread CC=clang-16 CXX=clang++-16 CXXFLAGS='-g' --with-boost-process"
-export CPPFLAGS="-DDEBUG_LOCKORDER -DENABLE_DASH_DEBUG -DARENA_DEBUG"
+export CPPFLAGS="-DDEBUG_LOCKORDER -DARENA_DEBUG"
 export PYZMQ=true

src/Makefile.am

@@ -371,6 +371,7 @@ BITCOIN_CORE_H = \
   wallet/context.h \
   wallet/crypter.h \
   wallet/db.h \
+  wallet/dump.h \
   wallet/fees.h \
   wallet/hdchain.h \
   wallet/ismine.h \
@@ -549,6 +550,7 @@ libbitcoin_wallet_a_SOURCES = \
   wallet/context.cpp \
   wallet/crypter.cpp \
   wallet/db.cpp \
+  wallet/dump.cpp \
   wallet/fees.cpp \
   wallet/hdchain.cpp \
   wallet/interfaces.cpp \

src/bitcoin-wallet.cpp

@@ -25,9 +25,13 @@ static void SetupWalletToolArgs(ArgsManager& argsman)
     SetupChainParamsBaseOptions(argsman);
 
     argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-usehd", strprintf("Create HD (hierarchical deterministic) wallet (default: %d)", true), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-datadir=<dir>", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-wallet=<wallet-name>", "Specify wallet name", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-dumpfile=<file name>", "When used with 'dump', writes out the records to this file. When used with 'createfromdump', loads the records into a new wallet.", ArgsManager::ALLOW_STRING, OptionsCategory::OPTIONS);
     argsman.AddArg("-debug=<category>", "Output debugging information (default: 0).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
+    argsman.AddArg("-descriptors", "Create descriptors wallet. Only for 'create'", ArgsManager::ALLOW_BOOL, OptionsCategory::OPTIONS);
+    argsman.AddArg("-format=<format>", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST);
 
     argsman.AddArg("info", "Get wallet info", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS);
@@ -36,6 +40,8 @@ static void SetupWalletToolArgs(ArgsManager& argsman)
     // Hidden
     argsman.AddArg("salvage", "Attempt to recover private keys from a corrupt wallet. Warning: 'salvage' is experimental.", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS);
     argsman.AddArg("wipetxes", "Wipe all transactions from a wallet", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS);
+    argsman.AddArg("dump", "Print out all of the wallet key-value records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS);
+    argsman.AddArg("createfromdump", "Create new wallet file from dumped records", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS);
 }
 
 static bool WalletAppInit(int argc, char* argv[])
@@ -114,8 +120,9 @@ MAIN_FUNCTION
     std::string name = gArgs.GetArg("-wallet", "");
 
     ECC_Start();
-    if (!WalletTool::ExecuteWalletToolFunc(method, name))
+    if (!WalletTool::ExecuteWalletToolFunc(gArgs, method, name)) {
         return EXIT_FAILURE;
+    }
     ECC_Stop();
     return EXIT_SUCCESS;
 }

src/util/system.h

@@ -34,17 +34,6 @@
 #include <utility>
 #include <vector>
 
-// Debugging macros
-
-// Uncomment the following line to enable debugging messages
-// or enable on a per file basis prior to inclusion of util.h
-//#define ENABLE_DASH_DEBUG
-#ifdef ENABLE_DASH_DEBUG
-#define DBG( x ) x
-#else
-#define DBG( x )
-#endif
-
 //Dash only features
 
 extern bool fMasternodeMode;

src/wallet/dump.cpp

@@ -0,0 +1,282 @@
+// Copyright (c) 2020 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <wallet/dump.h>
+
+#include <util/translation.h>
+#include <wallet/wallet.h>
+
+static const std::string DUMP_MAGIC = "BITCOIN_CORE_WALLET_DUMP";
+uint32_t DUMP_VERSION = 1;
+
+bool DumpWallet(CWallet& wallet, bilingual_str& error)
+{
+    // Get the dumpfile
+    std::string dump_filename = gArgs.GetArg("-dumpfile", "");
+    if (dump_filename.empty()) {
+        error = _("No dump file provided. To use dump, -dumpfile=<filename> must be provided.");
+        return false;
+    }
+
+    fs::path path = dump_filename;
+    path = fs::absolute(path);
+    if (fs::exists(path)) {
+        error = strprintf(_("File %s already exists. If you are sure this is what you want, move it out of the way first."), path.string());
+        return false;
+    }
+    fsbridge::ofstream dump_file;
+    dump_file.open(path);
+    if (dump_file.fail()) {
+        error = strprintf(_("Unable to open %s for writing"), path.string());
+        return false;
+    }
+
+    CHashWriter hasher(0, 0);
+
+    WalletDatabase& db = wallet.GetDatabase();
+    std::unique_ptr<DatabaseBatch> batch = db.MakeBatch();
+
+    bool ret = true;
+    if (!batch->StartCursor()) {
+        error = _("Error: Couldn't create cursor into database");
+        ret = false;
+    }
+
+    // Write out a magic string with version
+    std::string line = strprintf("%s,%u\n", DUMP_MAGIC, DUMP_VERSION);
+    dump_file.write(line.data(), line.size());
+    hasher << Span{line};
+
+    // Write out the file format
+    line = strprintf("%s,%s\n", "format", db.Format());
+    dump_file.write(line.data(), line.size());
+    hasher << Span{line};
+
+    if (ret) {
+
+        // Read the records
+        while (true) {
+            CDataStream ss_key(SER_DISK, CLIENT_VERSION);
+            CDataStream ss_value(SER_DISK, CLIENT_VERSION);
+            bool complete;
+            ret = batch->ReadAtCursor(ss_key, ss_value, complete);
+            if (complete) {
+                ret = true;
+                break;
+            } else if (!ret) {
+                error = _("Error reading next record from wallet database");
+                break;
+            }
+            std::string key_str = HexStr(ss_key);
+            std::string value_str = HexStr(ss_value);
+            line = strprintf("%s,%s\n", key_str, value_str);
+            dump_file.write(line.data(), line.size());
+            hasher << Span{line};
+        }
+    }
+
+    batch->CloseCursor();
+    batch.reset();
+
+    // Close the wallet after we're done with it. The caller won't be doing this
+    wallet.Close();
+
+    if (ret) {
+        // Write the hash
+        tfm::format(dump_file, "checksum,%s\n", HexStr(hasher.GetHash()));
+        dump_file.close();
+    } else {
+        // Remove the dumpfile on failure
+        dump_file.close();
+        fs::remove(path);
+    }
+
+    return ret;
+}
+
+// The standard wallet deleter function blocks on the validation interface
+// queue, which doesn't exist for the bitcoin-wallet. Define our own
+// deleter here.
+static void WalletToolReleaseWallet(CWallet* wallet)
+{
+    wallet->WalletLogPrintf("Releasing wallet\n");
+    wallet->Close();
+    delete wallet;
+}
+
+bool CreateFromDump(const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings)
+{
+    // Get the dumpfile
+    std::string dump_filename = gArgs.GetArg("-dumpfile", "");
+    if (dump_filename.empty()) {
+        error = _("No dump file provided. To use createfromdump, -dumpfile=<filename> must be provided.");
+        return false;
+    }
+
+    fs::path dump_path = dump_filename;
+    dump_path = fs::absolute(dump_path);
+    if (!fs::exists(dump_path)) {
+        error = strprintf(_("Dump file %s does not exist."), dump_path.string());
+        return false;
+    }
+    fsbridge::ifstream dump_file(dump_path);
+
+    // Compute the checksum
+    CHashWriter hasher(0, 0);
+    uint256 checksum;
+
+    // Check the magic and version
+    std::string magic_key;
+    std::getline(dump_file, magic_key, ',');
+    std::string version_value;
+    std::getline(dump_file, version_value, '\n');
+    if (magic_key != DUMP_MAGIC) {
+        error = strprintf(_("Error: Dumpfile identifier record is incorrect. Got \"%s\", expected \"%s\"."), magic_key, DUMP_MAGIC);
+        dump_file.close();
+        return false;
+    }
+    // Check the version number (value of first record)
+    uint32_t ver;
+    if (!ParseUInt32(version_value, &ver)) {
+        error =strprintf(_("Error: Unable to parse version %u as a uint32_t"), version_value);
+        dump_file.close();
+        return false;
+    }
+    if (ver != DUMP_VERSION) {
+        error = strprintf(_("Error: Dumpfile version is not supported. This version of bitcoin-wallet only supports version 1 dumpfiles. Got dumpfile with version %s"), version_value);
+        dump_file.close();
+        return false;
+    }
+    std::string magic_hasher_line = strprintf("%s,%s\n", magic_key, version_value);
+    hasher << Span{magic_hasher_line};
+
+    // Get the stored file format
+    std::string format_key;
+    std::getline(dump_file, format_key, ',');
+    std::string format_value;
+    std::getline(dump_file, format_value, '\n');
+    if (format_key != "format") {
+        error = strprintf(_("Error: Dumpfile format record is incorrect. Got \"%s\", expected \"format\"."), format_key);
+        dump_file.close();
+        return false;
+    }
+    // Get the data file format with format_value as the default
+    std::string file_format = gArgs.GetArg("-format", format_value);
+    if (file_format.empty()) {
+        error = _("No wallet file format provided. To use createfromdump, -format=<format> must be provided.");
+        return false;
+    }
+    DatabaseFormat data_format;
+    if (file_format == "bdb") {
+        data_format = DatabaseFormat::BERKELEY;
+    } else if (file_format == "sqlite") {
+        data_format = DatabaseFormat::SQLITE;
+    } else {
+        error = strprintf(_("Unknown wallet file format \"%s\" provided. Please provide one of \"bdb\" or \"sqlite\"."), file_format);
+        return false;
+    }
+    if (file_format != format_value) {
+        warnings.push_back(strprintf(_("Warning: Dumpfile wallet format \"%s\" does not match command line specified format \"%s\"."), format_value, file_format));
+    }
+    std::string format_hasher_line = strprintf("%s,%s\n", format_key, format_value);
+    hasher << Span{format_hasher_line};
+
+    DatabaseOptions options;
+    DatabaseStatus status;
+    options.require_create = true;
+    options.require_format = data_format;
+    std::unique_ptr<WalletDatabase> database = MakeDatabase(wallet_path, options, status, error);
+    if (!database) return false;
+
+    // dummy chain interface
+    bool ret = true;
+    std::shared_ptr<CWallet> wallet(new CWallet(nullptr /* chain */, /*coinjoin_loader=*/ nullptr, name, std::move(database)), WalletToolReleaseWallet);
+    {
+        LOCK(wallet->cs_wallet);
+        bool first_run = true;
+        DBErrors load_wallet_ret = wallet->LoadWallet(first_run);
+        if (load_wallet_ret != DBErrors::LOAD_OK) {
+            error = strprintf(_("Error creating %s"), name);
+            return false;
+        }
+
+        // Get the database handle
+        WalletDatabase& db = wallet->GetDatabase();
+        std::unique_ptr<DatabaseBatch> batch = db.MakeBatch();
+        batch->TxnBegin();
+
+        // Read the records from the dump file and write them to the database
+        while (dump_file.good()) {
+            std::string key;
+            std::getline(dump_file, key, ',');
+            std::string value;
+            std::getline(dump_file, value, '\n');
+
+            if (key == "checksum") {
+                std::vector<unsigned char> parsed_checksum = ParseHex(value);
+                std::copy(parsed_checksum.begin(), parsed_checksum.end(), checksum.begin());
+                break;
+            }
+
+            std::string line = strprintf("%s,%s\n", key, value);
+            hasher << Span{line};
+
+            if (key.empty() || value.empty()) {
+                continue;
+            }
+
+            if (!IsHex(key)) {
+                error = strprintf(_("Error: Got key that was not hex: %s"), key);
+                ret = false;
+                break;
+            }
+            if (!IsHex(value)) {
+                error = strprintf(_("Error: Got value that was not hex: %s"), value);
+                ret = false;
+                break;
+            }
+
+            std::vector<unsigned char> k = ParseHex(key);
+            std::vector<unsigned char> v = ParseHex(value);
+
+            CDataStream ss_key(k, SER_DISK, CLIENT_VERSION);
+            CDataStream ss_value(v, SER_DISK, CLIENT_VERSION);
+
+            if (!batch->Write(ss_key, ss_value)) {
+                error = strprintf(_("Error: Unable to write record to new wallet"));
+                ret = false;
+                break;
+            }
+        }
+
+        if (ret) {
+            uint256 comp_checksum = hasher.GetHash();
+            if (checksum.IsNull()) {
+                error = _("Error: Missing checksum");
+                ret = false;
+            } else if (checksum != comp_checksum) {
+                error = strprintf(_("Error: Dumpfile checksum does not match. Computed %s, expected %s"), HexStr(comp_checksum), HexStr(checksum));
+                ret = false;
+            }
+        }
+
+        if (ret) {
+            batch->TxnCommit();
+        } else {
+            batch->TxnAbort();
+        }
+
+        batch.reset();
+
+        dump_file.close();
+    }
+    wallet.reset(); // The pointer deleter will close the wallet for us.
+
+    // Remove the wallet dir if we have a failure
+    if (!ret) {
+        fs::remove_all(wallet_path);
+    }
+
+    return ret;
+}

src/wallet/dump.h

@@ -0,0 +1,17 @@
+// Copyright (c) 2020 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_WALLET_DUMP_H
+#define BITCOIN_WALLET_DUMP_H
+
+#include <fs.h>
+
+class CWallet;
+
+struct bilingual_str;
+
+bool DumpWallet(CWallet& wallet, bilingual_str& error);
+bool CreateFromDump(const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings);
+
+#endif // BITCOIN_WALLET_DUMP_H