From e2f0ec7c69e0af001c2ddb4a6aa18be2efbc1f30 Mon Sep 17 00:00:00 2001 From: SChernykh <15806605+SChernykh@users.noreply.github.com> Date: Sat, 18 Oct 2025 12:21:16 +0200 Subject: [PATCH] De-duplicate tx hashes and pub keys to save memory (off by default) (#382) P2Pool-main: 8.2 MB saved P2Pool-mini: 66 MB saved P2Pool-nano: 25.2 MB saved The feature is available only when building from source and is intended for use on low-memory systems (for example, a VPS server with < 1 GB RAM). It only makes sense to use with `--no-cache --no-randomx` in the command line because cache and RandomX hasher take much more memory. --- .github/workflows/c-cpp.yml | 4 +- .github/workflows/clang-tidy.yml | 2 +- .github/workflows/coverage.yml | 2 +- .github/workflows/cppcheck.yml | 2 +- .github/workflows/test-sync.yml | 14 +- CMakeLists.txt | 6 + src/block_template.cpp | 21 ++- src/block_template.h | 4 + src/common.h | 93 ++++++++- src/console_commands.cpp | 4 + src/crypto.cpp | 17 +- src/indexed_hash.cpp | 291 +++++++++++++++++++++++++++++ src/json_parsers.h | 17 ++ src/log.h | 9 +- src/main.cpp | 4 + src/merge_mining_client_tari.cpp | 15 +- src/p2p_server.cpp | 2 +- src/pool_block.cpp | 45 ++++- src/pool_block.h | 12 +- src/pool_block_parser.inl | 58 ++++-- src/side_chain.cpp | 44 +++-- tests/CMakeLists.txt | 8 +- tests/src/block_template_tests.cpp | 35 +++- tests/src/crypto_tests.cpp | 8 +- tests/src/pool_block_tests.cpp | 26 ++- 25 files changed, 640 insertions(+), 103 deletions(-) create mode 100644 src/indexed_hash.cpp diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml index 69e49ce..599979e 100644 --- a/.github/workflows/c-cpp.yml +++ b/.github/workflows/c-cpp.yml @@ -429,8 +429,8 @@ jobs: strategy: matrix: config: - - {c: "gcc", cxx: "g++", flags: "-ffunction-sections -Wno-error=maybe-uninitialized -Wno-error=attributes -Wno-unknown-attributes"} - - {c: "clang", cxx: "clang++", flags: "-fuse-ld=lld -Wno-unused-command-line-argument -Wno-nan-infinity-disabled -Wno-unknown-attributes"} + - {c: "gcc", cxx: "g++", flags: "-ffunction-sections -Wno-error=maybe-uninitialized -Wno-error=attributes -Wno-attributes"} + - {c: "clang", cxx: "clang++", flags: "-fuse-ld=lld -Wno-unused-command-line-argument -Wno-nan-infinity-disabled -Wno-attributes"} defaults: run: diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 3ed4078..5b844fe 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -32,7 +32,7 @@ jobs: run: | mkdir build cd build - cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER=clang-21 -DCMAKE_CXX_COMPILER=clang++-21 -DDEV_CLANG_TIDY=ON -DSTATIC_LIBS=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" + cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER=clang-21 -DCMAKE_CXX_COMPILER=clang++-21 -DDEV_CLANG_TIDY=ON -DSTATIC_LIBS=ON -DWITH_INDEXED_HASHES=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" - name: Run clang-tidy run: | diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 3d4274a..5e23e2e 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -38,7 +38,7 @@ jobs: cd tests mkdir build cd build - cmake .. -DCMAKE_BUILD_TYPE=Release -DDEV_DEBUG=ON -DWITH_COVERAGE=ON -DCMAKE_C_COMPILER=clang-21 -DCMAKE_CXX_COMPILER=clang++-21 -DCMAKE_POLICY_VERSION_MINIMUM="3.5" + cmake .. -DCMAKE_BUILD_TYPE=Release -DDEV_DEBUG=ON -DWITH_INDEXED_HASHES=ON -DWITH_COVERAGE=ON -DCMAKE_C_COMPILER=clang-21 -DCMAKE_CXX_COMPILER=clang++-21 -DCMAKE_POLICY_VERSION_MINIMUM="3.5" make -j$(nproc) p2pool_tests - name: Run tests diff --git a/.github/workflows/cppcheck.yml b/.github/workflows/cppcheck.yml index d9d2a26..2d41309 100644 --- a/.github/workflows/cppcheck.yml +++ b/.github/workflows/cppcheck.yml @@ -38,7 +38,7 @@ jobs: run: | mkdir build cd build - cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DSTATIC_LIBS=ON -DWITH_GRPC=OFF -DCMAKE_POLICY_VERSION_MINIMUM="3.5" + cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DSTATIC_LIBS=ON -DWITH_GRPC=OFF -DWITH_INDEXED_HASHES=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" python ../cppcheck/remove_external.py compile_commands.json - name: Run cppcheck diff --git a/.github/workflows/test-sync.yml b/.github/workflows/test-sync.yml index 10f39c3..6d0f396 100644 --- a/.github/workflows/test-sync.yml +++ b/.github/workflows/test-sync.yml @@ -57,7 +57,7 @@ jobs: run: | mkdir build cd build - cmake .. -DDEV_TEST_SYNC=ON -DDEV_WITH_TSAN=ON -DCMAKE_C_COMPILER=gcc-12 -DCMAKE_CXX_COMPILER=g++-12 -DCMAKE_C_FLAGS='-fsanitize=thread -Og -fno-omit-frame-pointer -g' -DCMAKE_CXX_FLAGS='-fsanitize=thread -Og -fno-omit-frame-pointer -g' -DWITH_LTO=OFF -DSTATIC_LIBS=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" + cmake .. -DDEV_TEST_SYNC=ON -DDEV_WITH_TSAN=ON -DCMAKE_C_COMPILER=gcc-12 -DCMAKE_CXX_COMPILER=g++-12 -DCMAKE_C_FLAGS='-fsanitize=thread -Og -fno-omit-frame-pointer -g' -DCMAKE_CXX_FLAGS='-fsanitize=thread -Og -fno-omit-frame-pointer -g' -DWITH_LTO=OFF -DSTATIC_LIBS=ON -DWITH_INDEXED_HASHES=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" make -j$(nproc) p2pool - name: Run p2pool @@ -148,7 +148,7 @@ jobs: run: | mkdir build cd build - cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang-21 -DCMAKE_CXX_COMPILER=clang++-21 -DCMAKE_C_FLAGS='-fsanitize=memory -fsanitize-recover -fsanitize-memory-track-origins -fno-omit-frame-pointer' -DCMAKE_CXX_FLAGS='-nostdinc++ -nostdlib++ -fsanitize=memory -fsanitize-recover -fsanitize-memory-track-origins -isystem /tmp/libcxx_msan/include/c++/v1 -L/tmp/libcxx_msan/lib -Wl,-rpath /tmp/libcxx_msan/lib -lc++ -lc++abi -Wno-unused-command-line-argument -fuse-ld=lld-21 -fno-omit-frame-pointer' -DDEV_TEST_SYNC=ON -DDEV_WITH_MSAN=ON -DWITH_LTO=OFF -DSTATIC_LIBS=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" + cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang-21 -DCMAKE_CXX_COMPILER=clang++-21 -DCMAKE_C_FLAGS='-fsanitize=memory -fsanitize-recover -fsanitize-memory-track-origins -fno-omit-frame-pointer' -DCMAKE_CXX_FLAGS='-nostdinc++ -nostdlib++ -fsanitize=memory -fsanitize-recover -fsanitize-memory-track-origins -isystem /tmp/libcxx_msan/include/c++/v1 -L/tmp/libcxx_msan/lib -Wl,-rpath /tmp/libcxx_msan/lib -lc++ -lc++abi -Wno-unused-command-line-argument -fuse-ld=lld-21 -fno-omit-frame-pointer' -DDEV_TEST_SYNC=ON -DDEV_WITH_MSAN=ON -DWITH_LTO=OFF -DSTATIC_LIBS=ON -DWITH_INDEXED_HASHES=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" make -j$(nproc) p2pool - name: Run p2pool @@ -201,7 +201,7 @@ jobs: run: | mkdir build cd build - cmake .. -DDEV_TEST_SYNC=ON -DDEV_WITH_UBSAN=ON -DCMAKE_C_COMPILER=gcc-12 -DCMAKE_CXX_COMPILER=g++-12 -DWITH_LTO=OFF -DCMAKE_POLICY_VERSION_MINIMUM="3.5" + cmake .. -DDEV_TEST_SYNC=ON -DDEV_WITH_UBSAN=ON -DCMAKE_C_COMPILER=gcc-12 -DCMAKE_CXX_COMPILER=g++-12 -DWITH_LTO=OFF -DWITH_INDEXED_HASHES=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" make -j$(nproc) p2pool - name: Run p2pool @@ -254,7 +254,7 @@ jobs: run: | mkdir build cd build - cmake .. -DDEV_TEST_SYNC=ON -DDEV_DEBUG=ON -DDEV_WITH_ASAN=ON -DCMAKE_C_COMPILER=gcc-12 -DCMAKE_CXX_COMPILER=g++-12 -DCMAKE_C_FLAGS="-fno-omit-frame-pointer -fsanitize=address -Og -g" -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer -fsanitize=address -Og -g" -DWITH_LTO=OFF -DCMAKE_POLICY_VERSION_MINIMUM="3.5" + cmake .. -DDEV_TEST_SYNC=ON -DDEV_DEBUG=ON -DDEV_WITH_ASAN=ON -DCMAKE_C_COMPILER=gcc-12 -DCMAKE_CXX_COMPILER=g++-12 -DCMAKE_C_FLAGS="-fno-omit-frame-pointer -fsanitize=address -Og -g" -DCMAKE_CXX_FLAGS="-fno-omit-frame-pointer -fsanitize=address -Og -g" -DWITH_LTO=OFF -DWITH_INDEXED_HASHES=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" make -j$(nproc) p2pool - name: Run p2pool @@ -329,7 +329,7 @@ jobs: run: | mkdir build cd build - cmake .. -DCMAKE_C_COMPILER="$(brew --prefix llvm@18)/bin/clang" -DCMAKE_CXX_COMPILER="$(brew --prefix llvm@18)/bin/clang++" -DCMAKE_AR="$(brew --prefix llvm@18)/bin/llvm-ar" -DCMAKE_RANLIB="$(brew --prefix llvm@18)/bin/llvm-ranlib" -DCMAKE_C_FLAGS='${{ matrix.config.flags }}' -DCMAKE_CXX_FLAGS='${{ matrix.config.flags }}' -DWITH_LTO=OFF -DSTATIC_LIBS=ON -DDEV_TEST_SYNC=ON -DDEV_DEBUG=ON + cmake .. -DCMAKE_C_COMPILER="$(brew --prefix llvm@18)/bin/clang" -DCMAKE_CXX_COMPILER="$(brew --prefix llvm@18)/bin/clang++" -DCMAKE_AR="$(brew --prefix llvm@18)/bin/llvm-ar" -DCMAKE_RANLIB="$(brew --prefix llvm@18)/bin/llvm-ranlib" -DCMAKE_C_FLAGS='${{ matrix.config.flags }}' -DCMAKE_CXX_FLAGS='${{ matrix.config.flags }}' -DWITH_LTO=OFF -DSTATIC_LIBS=ON -DDEV_TEST_SYNC=ON -DDEV_DEBUG=ON -DWITH_INDEXED_HASHES=ON make -j4 p2pool - name: Run p2pool @@ -378,7 +378,7 @@ jobs: run: | mkdir build cd build - cmake .. -G "Visual Studio 17 2022" -DCMAKE_SYSTEM_VERSION="10.0" -DDEV_TEST_SYNC=ON -DDEV_WITH_ASAN=ON -DDEV_DEBUG=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" + cmake .. -G "Visual Studio 17 2022" -DCMAKE_SYSTEM_VERSION="10.0" -DDEV_TEST_SYNC=ON -DDEV_WITH_ASAN=ON -DDEV_DEBUG=ON -DWITH_INDEXED_HASHES=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" & "C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\Msbuild\\Current\\Bin\\amd64\\msbuild" -v:m /m /p:Configuration=Debug p2pool.vcxproj - name: Run p2pool @@ -424,7 +424,7 @@ jobs: run: | mkdir build cd build - cmake .. -G "Visual Studio 17 2022" -DCMAKE_SYSTEM_VERSION="10.0" -DDEV_TEST_SYNC=ON -DDEV_DEBUG=ON -DDEV_TRACK_MEMORY=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" + cmake .. -G "Visual Studio 17 2022" -DCMAKE_SYSTEM_VERSION="10.0" -DDEV_TEST_SYNC=ON -DDEV_DEBUG=ON -DDEV_TRACK_MEMORY=ON -DWITH_INDEXED_HASHES=ON -DCMAKE_POLICY_VERSION_MINIMUM="3.5" & "C:\\Program Files\\Microsoft Visual Studio\\2022\\Enterprise\\Msbuild\\Current\\Bin\\amd64\\msbuild" -v:m /m /p:Configuration=RelWithDebInfo p2pool.vcxproj - name: Run p2pool diff --git a/CMakeLists.txt b/CMakeLists.txt index bb6bada..f651dbc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,7 @@ option(WITH_LTO "Use link-time compiler optimization (if linking fails for you, option(WITH_UPNP "Include UPnP support. If this is turned off, p2pool will not be able to configure port forwarding on UPnP-enabled routers." ON) option(WITH_GRPC "Include gRPC support. If this is turned off, p2pool will not be able to merge mine with Tari." ON) option(WITH_TLS "Include TLS support. If this is turned off, p2pool will not support Stratum TLS connections, and lack many other security features. It's recommended to keep it ON!" ON) +option(WITH_INDEXED_HASHES "Save memory used for storing transaction hashes and public keys (a bit slower block verification - use it only if you compile for a very low memory system)" OFF) option(WITH_MERGE_MINING_DONATION "Merge mine donations to the author. This doesn't affect your hashrate or payouts in any way - only unused merge mining capacity will be utilised. If you merge mine yourself, your settings will take priority." ON) @@ -190,6 +191,11 @@ if (WITH_TLS) set(SOURCES ${SOURCES} src/tls.cpp) endif() +if (WITH_INDEXED_HASHES) + add_compile_definitions(WITH_INDEXED_HASHES) + set(SOURCES ${SOURCES} src/indexed_hash.cpp) +endif() + source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}" PREFIX "Header Files" FILES ${HEADERS}) source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}" PREFIX "Source Files" FILES ${SOURCES}) diff --git a/src/block_template.cpp b/src/block_template.cpp index b40c1a8..a5d2542 100644 --- a/src/block_template.cpp +++ b/src/block_template.cpp @@ -395,7 +395,8 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const LOGERR(1, "Added transaction " << tx.id << " twice. Fix the code!"); continue; } - m_transactionHashes.insert(m_transactionHashes.end(), tx.id.h, tx.id.h + HASH_SIZE); + const hash h = tx.id; + m_transactionHashes.insert(m_transactionHashes.end(), h.h, h.h + HASH_SIZE); final_fees += tx.fee; final_weight += tx.weight; } @@ -476,7 +477,8 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const LOGERR(1, "Added transaction " << tx.id << " twice. Fix the code!"); continue; } - m_transactionHashes.insert(m_transactionHashes.end(), tx.id.h, tx.id.h + HASH_SIZE); + const hash h = tx.id; + m_transactionHashes.insert(m_transactionHashes.end(), h.h, h.h + HASH_SIZE); final_fees += tx.fee; final_weight += tx.weight; } @@ -728,7 +730,7 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const LOGINFO(3, "final reward = " << log::Gray() << log::XMRAmount(final_reward) << log::NoColor() << ", weight = " << log::Gray() << final_weight << log::NoColor() << - ", outputs = " << log::Gray() << m_poolBlockTemplate->m_outputs.size() << log::NoColor() << + ", outputs = " << log::Gray() << m_poolBlockTemplate->m_outputAmounts.size() << log::NoColor() << ", " << log::Gray() << m_numTransactionHashes << log::NoColor() << " of " << log::Gray() << m_mempoolTxs.size() << log::NoColor() << " transactions included"); @@ -847,7 +849,8 @@ void BlockTemplate::select_mempool_transactions(const Mempool& mempool) PoolBlock* b = m_poolBlockTemplate; b->m_transactions.clear(); b->m_transactions.resize(1); - b->m_outputs.clear(); + b->m_ephPublicKeys.clear(); + b->m_outputAmounts.clear(); // Block template size without coinbase outputs and transactions (minus 2 bytes for output and tx count dummy varints) size_t k = b->serialize_mainchain_data().size() + b->serialize_sidechain_data().size() - 2; @@ -909,8 +912,11 @@ int BlockTemplate::create_miner_tx(const MinerData& data, const std::vectorm_outputs.clear(); - m_poolBlockTemplate->m_outputs.reserve(num_outputs); + m_poolBlockTemplate->m_ephPublicKeys.clear(); + m_poolBlockTemplate->m_outputAmounts.clear(); + + m_poolBlockTemplate->m_ephPublicKeys.reserve(num_outputs); + m_poolBlockTemplate->m_outputAmounts.reserve(num_outputs); uint64_t reward_amounts_weight = 0; for (size_t i = 0; i < num_outputs; ++i) { @@ -932,7 +938,8 @@ int BlockTemplate::create_miner_tx(const MinerData& data, const std::vectorm_outputs.emplace_back(m_rewards[i], eph_public_key, view_tag); + m_poolBlockTemplate->m_ephPublicKeys.emplace_back(eph_public_key); + m_poolBlockTemplate->m_outputAmounts.emplace_back(m_rewards[i], view_tag); } m_minerTx.emplace_back(view_tag); diff --git a/src/block_template.h b/src/block_template.h index a7ac0c7..b6a256f 100644 --- a/src/block_template.h +++ b/src/block_template.h @@ -107,7 +107,11 @@ private: PoolBlock* m_poolBlockTemplate; +#ifdef P2POOL_UNIT_TESTS + BlockTemplate* m_oldTemplates[1] = {}; +#else BlockTemplate* m_oldTemplates[4] = {}; +#endif std::atomic m_finalReward; diff --git a/src/common.h b/src/common.h index 4e14c40..f331c70 100644 --- a/src/common.h +++ b/src/common.h @@ -377,6 +377,85 @@ FORCEINLINE difficulty_type operator/(const difficulty_type& a, const T& b) return result; } +#ifdef WITH_INDEXED_HASHES +struct indexed_hash +{ + enum { + BUCKET_BITS = 12, + BUCKET_SHIFT = 32 - BUCKET_BITS, + }; + + static_assert((BUCKET_BITS > 0) && (BUCKET_BITS < 32), "Invalid bucket bit size"); + + FORCEINLINE indexed_hash() : m_index(std::numeric_limits::max()) {} + + explicit indexed_hash(const hash& h); + ~indexed_hash(); + + indexed_hash(const indexed_hash& h); + FORCEINLINE indexed_hash(indexed_hash&& h) : m_index(h.m_index) { h.m_index = std::numeric_limits::max(); } + + indexed_hash& operator=(const indexed_hash& h); + indexed_hash& operator=(indexed_hash&& h); + + FORCEINLINE indexed_hash& operator=(const hash& h) + { + if (*this == h) { + return *this; + } + return operator=(indexed_hash(h)); + } + + FORCEINLINE bool operator==(const indexed_hash& h) const { return m_index == h.m_index; } + + FORCEINLINE bool operator==(const hash& h) const + { + const uint32_t i = m_index; + + if (i == std::numeric_limits::max()) { + return h.empty(); + } + + const uint32_t bucket = i >> BUCKET_SHIFT; + + if (bucket != get_bucket(h)) { + return false; + } + + return is_same(i, bucket, h); + } + + FORCEINLINE operator hash() const + { + const uint32_t i = m_index; + + if (i == std::numeric_limits::max()) { + return {}; + } + + return get(i); + } + + FORCEINLINE uint32_t get_bucket() const { return m_index >> BUCKET_SHIFT; } + + static void cleanup_storage(); + + static void print_status(); + +private: + uint32_t m_index; + + static FORCEINLINE uint32_t get_bucket(const hash& h) { return h.u64()[3] >> (64 - BUCKET_BITS); } + + static hash get(uint32_t index); + static bool is_same(uint32_t index, uint32_t bucket, const hash& h); +}; + +static_assert(sizeof(indexed_hash) == 4, "indexed_hash has wrong size"); +#else +typedef hash indexed_hash; +#endif + struct TxMempoolData { FORCEINLINE TxMempoolData() : id(), blob_size(0), weight(0), fee(0), time_received(0) {} @@ -395,10 +474,22 @@ struct TxMempoolData if (weight > tx.weight) return false; // If two transactions have exactly the same fee and weight, just order them by id +#ifdef WITH_INDEXED_HASHES + const uint32_t id_bucket = id.get_bucket(); + const uint32_t tx_id_bucket = tx.id.get_bucket(); + + if (id_bucket < tx_id_bucket) return true; + if (id_bucket > tx_id_bucket) return false; + + const hash h1 = id; + const hash h2 = tx.id; + return h1 < h2; +#else return id < tx.id; +#endif } - hash id; + indexed_hash id; uint64_t blob_size; uint64_t weight; uint64_t fee; diff --git a/src/console_commands.cpp b/src/console_commands.cpp index 5f394d5..7445a09 100644 --- a/src/console_commands.cpp +++ b/src/console_commands.cpp @@ -224,6 +224,10 @@ static void do_status(p2pool *m_pool, const char * /* args */) m_pool->print_merge_mining_status(); +#ifdef WITH_INDEXED_HASHES + indexed_hash::print_status(); +#endif + bkg_jobs_tracker->print_status(); if (p2p) { diff --git a/src/crypto.cpp b/src/crypto.cpp index 4998353..9a6be36 100644 --- a/src/crypto.cpp +++ b/src/crypto.cpp @@ -218,7 +218,7 @@ public: { WriteLock lock(derivations_lock); - auto entry = derivations->emplace(index, DerivationEntry{ derivation, { 0xFFFFFFFFUL, 0xFFFFFFFFUL }, {}, t }).first; + auto entry = derivations->emplace(index, DerivationEntry{ derivation, { 0xFFFFFFFFUL, 0xFFFFFFFFUL }, {}, static_cast(t) }).first; entry->second.add_view_tag(static_cast(output_index << 8) | view_tag); } @@ -262,7 +262,7 @@ public: const uint64_t t = seconds_since_epoch(); { WriteLock lock(public_keys_lock); - public_keys->emplace(index, PublicKeyEntry{ derived_key, t }); + public_keys->emplace(index, PublicKeyEntry{ static_cast(derived_key), static_cast(t) }); } return true; @@ -297,7 +297,7 @@ public: const uint64_t t = seconds_since_epoch(); { WriteLock lock(tx_keys_lock); - tx_keys->emplace(index, TxKeyEntry{ pub, sec, t }); + tx_keys->emplace(index, TxKeyEntry{ pub, sec, static_cast(t) }); } } @@ -306,7 +306,8 @@ public: if (timestamp) { auto clean_old = [timestamp](auto* table) { for (auto it = table->begin(); it != table->end();) { - if (it->second.m_timestamp < timestamp) { + // Wraparound-safe way of checking "it->second.m_timestamp < timestamp" + if (((it->second.m_timestamp - static_cast(timestamp)) & 0x80000000UL) != 0) { it = table->erase(it); } else { @@ -357,7 +358,7 @@ private: uint32_t m_viewTags1[2] = { 0xFFFFFFFFUL, 0xFFFFFFFFUL }; std::vector m_viewTags2; // cppcheck-suppress unusedStructMember - uint64_t m_timestamp; + uint32_t m_timestamp; FORCEINLINE bool find_view_tag(size_t output_index, uint8_t& view_tag) const { @@ -407,9 +408,9 @@ private: struct PublicKeyEntry { - hash m_key; + indexed_hash m_key; // cppcheck-suppress unusedStructMember - uint64_t m_timestamp; + uint32_t m_timestamp; }; struct TxKeyEntry @@ -417,7 +418,7 @@ private: hash m_pub; hash m_sec; // cppcheck-suppress unusedStructMember - uint64_t m_timestamp; + uint32_t m_timestamp; }; typedef unordered_map, DerivationEntry> DerivationsMap; diff --git a/src/indexed_hash.cpp b/src/indexed_hash.cpp new file mode 100644 index 0000000..b716656 --- /dev/null +++ b/src/indexed_hash.cpp @@ -0,0 +1,291 @@ +/* + * This file is part of the Monero P2Pool + * Copyright (c) 2021-2025 SChernykh + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "common.h" +#include "util.h" +#include "uv_util.h" + +LOG_CATEGORY(indexed_hash) + +namespace p2pool { + +static constexpr uint32_t BUCKET_COUNT = 1U << indexed_hash::BUCKET_BITS; +static constexpr uint32_t INDEX_MASK = (1U << indexed_hash::BUCKET_SHIFT) - 1U; + +// Stores hashes only +static std::vector storage1[BUCKET_COUNT]; + +// Stores first 4 bytes of each hash (for better cache locality during search) and reference counters +static std::vector> storage2[BUCKET_COUNT]; + +static constexpr uint32_t LOCK_COUNT = 64; + +static ReadWriteLock locks[LOCK_COUNT]; + +static FORCEINLINE void decref(uint32_t index) +{ + if (index == std::numeric_limits::max()) { + return; + } + + const uint32_t bucket = index >> indexed_hash::BUCKET_SHIFT; + index &= INDEX_MASK; + + auto& d2 = storage2[bucket]; + + WriteLock lock(locks[bucket % LOCK_COUNT]); + + auto& p2 = d2[index]; + +#ifdef P2POOL_DEBUGGING + if (p2.second == 0) { + LOGERR(0, "fatal error: reference counter is 0 when it shouldn't be 0"); + PANIC_STOP(); + } +#endif + + --p2.second; +} + +static FORCEINLINE void incref(uint32_t index) +{ + if (index == std::numeric_limits::max()) { + return; + } + + const uint32_t bucket = index >> indexed_hash::BUCKET_SHIFT; + index &= INDEX_MASK; + + auto& d2 = storage2[bucket]; + + WriteLock lock(locks[bucket % LOCK_COUNT]); + ++d2[index].second; +} + +indexed_hash::indexed_hash(const hash& h) +{ + if (h.empty()) { + m_index = std::numeric_limits::max(); + return; + } + + const uint32_t bucket = get_bucket(h); + + auto& d1 = storage1[bucket]; + auto& d2 = storage2[bucket]; + + const uint32_t h32 = static_cast(h.u64()[0]); + uint32_t first_free_slot = std::numeric_limits::max(); + + WriteLock lock(locks[bucket % LOCK_COUNT]); + + const uint32_t n = static_cast(d1.size()); + + for (uint32_t i = 0; i < n; ++i) { + auto& p2 = d2[i]; + + if ((p2.first == h32) && (d1[i] == h)) { + m_index = (bucket << BUCKET_SHIFT) | i; + ++p2.second; + return; + } + + if ((p2.second == 0) && (first_free_slot == std::numeric_limits::max())) { + first_free_slot = i; + } + } + + if (first_free_slot != std::numeric_limits::max()) { + m_index = (bucket << BUCKET_SHIFT) | first_free_slot; + + d1[first_free_slot] = h; + + auto& p2 = d2[first_free_slot]; + p2.first = h32; + p2.second = 1; + } + else { + if (n > INDEX_MASK) { + LOGERR(0, "fatal error: storage overflow"); + PANIC_STOP(); + } + + m_index = (bucket << BUCKET_SHIFT) | n; + + if (n == d1.capacity()) { + const uint32_t new_capacity = static_cast(std::min(n + ((n + 3) / 4), INDEX_MASK + 1)); + d1.reserve(new_capacity); + d2.reserve(new_capacity); + } + + d1.emplace_back(h); + d2.emplace_back(h32, 1); + } +} + +indexed_hash::~indexed_hash() +{ + decref(m_index); +} + +indexed_hash::indexed_hash(const indexed_hash& h) +{ + const uint32_t i = h.m_index; + + m_index = i; + incref(i); +} + +indexed_hash& indexed_hash::operator=(const indexed_hash& h) +{ + if (this == &h) { + return *this; + } + + const uint32_t i = m_index; + const uint32_t j = h.m_index; + + if (i == j) { + return *this; + } + + decref(i); + + m_index = j; + incref(j); + + return *this; +} + +indexed_hash& indexed_hash::operator=(indexed_hash&& h) +{ + if (this == &h) { + return *this; + } + + const uint32_t i = m_index; + const uint32_t j = h.m_index; + + // Empty h because we're moving it + h.m_index = std::numeric_limits::max(); + + if (i == j) { + // If h had the same index, decrease h's reference counter because now we have 1 instead of 2 objects using this index + decref(j); + return *this; + } + + // If h had a different index, decrease our reference counter because our index will be overwritten by the index from h + decref(i); + + // No need to change the reference counter anymore because now we are referencing the hash that was referenced by h + m_index = j; + + return *this; +} + +hash indexed_hash::get(uint32_t index) +{ + const uint32_t bucket = index >> BUCKET_SHIFT; + + const auto& d1 = storage1[bucket]; + index &= INDEX_MASK; + + ReadLock lock(locks[bucket % LOCK_COUNT]); + +#ifdef P2POOL_DEBUGGING + if (storage2[bucket][index].second == 0) { + LOGERR(0, "fatal error: reference counter is 0 when it shouldn't be 0"); + PANIC_STOP(); + } +#endif + + return d1[index]; +} + +bool indexed_hash::is_same(uint32_t index, uint32_t bucket, const hash& h) +{ + const auto& d1 = storage1[bucket]; + index &= INDEX_MASK; + + ReadLock lock(locks[bucket % LOCK_COUNT]); + +#ifdef P2POOL_DEBUGGING + if (storage2[bucket][index].second == 0) { + LOGERR(0, "fatal error: reference counter is 0 when it shouldn't be 0"); + PANIC_STOP(); + } +#endif + + return d1[index] == h; +} + +void indexed_hash::cleanup_storage() +{ + for (auto& d1 : storage1) { + d1.clear(); + d1.shrink_to_fit(); + } + + for (auto& d2 : storage2) { +#ifdef P2POOL_DEBUGGING + for (const auto& p2 : d2) { + if (p2.second) { + LOGERR(0, "storage is not empty at exit. Fix the code!"); + PANIC_STOP(); + } + } +#endif + d2.clear(); + d2.shrink_to_fit(); + } +} + +void indexed_hash::print_status() +{ + size_t total_hashes_stored = 0; + size_t total_refcount = 0; + size_t memory_used = 0; + + const auto& s1 = storage1; + const auto& s2 = storage2; + + for (uint32_t i = 0; i < LOCK_COUNT; ++i) { + ReadLock lock(locks[i]); + + for (uint32_t j = i; j < BUCKET_COUNT; j += LOCK_COUNT) { + memory_used += s1[j].capacity() * HASH_SIZE + s2[j].capacity() * sizeof(uint32_t) * 2; + + for (const auto& p2 : storage2[j]) { + if (p2.second) { + ++total_hashes_stored; + total_refcount += p2.second; + } + } + } + } + + memory_used += total_refcount * sizeof(indexed_hash); + + LOGINFO(4, "status" << + "\nTotal hashes stored = " << total_hashes_stored << + "\nTotal refcount = " << total_refcount << + "\nMemory used = " << static_cast(memory_used) / 1048576.0 << " MB (reduced from " << static_cast(total_refcount * HASH_SIZE) / 1048576.0 << " MB)" + ); +} + +} // namespace p2pool diff --git a/src/json_parsers.h b/src/json_parsers.h index 84df272..5030f77 100644 --- a/src/json_parsers.h +++ b/src/json_parsers.h @@ -85,6 +85,23 @@ struct parse_wrapper } }; +#ifdef WITH_INDEXED_HASHES +template +struct parse_wrapper +{ + static NOINLINE bool parse(T& v, const char* name, indexed_hash& out_value) + { + hash h; + if (!parse_wrapper::parse(v, name, h)) { + return false; + } + + out_value = static_cast(h); + return true; + } +}; +#endif + template struct parse_wrapper { diff --git a/src/log.h b/src/log.h index a847b3f..9cf1196 100644 --- a/src/log.h +++ b/src/log.h @@ -259,6 +259,13 @@ template<> struct Stream::Entry } }; +#ifdef WITH_INDEXED_HASHES +template<> struct Stream::Entry +{ + static FORCEINLINE void put(const indexed_hash& data, Stream* wrapper) { Stream::Entry::put(data, wrapper); } +}; +#endif + template<> struct Stream::Entry { static NOINLINE void put(const difficulty_type& data, Stream* wrapper) @@ -586,7 +593,7 @@ struct DummyStream #define LOGWARN(level, ...) SIDE_EFFECT_CHECK(level, log_category_prefix << __VA_ARGS__) #ifdef P2POOL_ABORT_ON_LOG_ERROR -#define LOGERR(level, ...) abort() +#define LOGERR(level, ...) do { SIDE_EFFECT_CHECK(level, log_category_prefix << __VA_ARGS__); abort(); } while (0) #else #define LOGERR(level, ...) SIDE_EFFECT_CHECK(level, log_category_prefix << __VA_ARGS__) #endif diff --git a/src/main.cpp b/src/main.cpp index f6d09ea..aa8a29d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -340,6 +340,10 @@ int main(int argc, char* argv[]) p2pool::destroy_crypto_cache(); +#ifdef WITH_INDEXED_HASHES + p2pool::indexed_hash::cleanup_storage(); +#endif + p2pool::log::stop(); uv_loop_close(uv_default_loop()); diff --git a/src/merge_mining_client_tari.cpp b/src/merge_mining_client_tari.cpp index cb2a6fc..1520c50 100644 --- a/src/merge_mining_client_tari.cpp +++ b/src/merge_mining_client_tari.cpp @@ -331,12 +331,23 @@ void MergeMiningClientTari::on_external_block(const PoolBlock& block) std::vector proof; uint32_t path; - if (!merkle_hash_with_proof(block.m_transactions, 0, proof, path, root)) { +#ifdef WITH_INDEXED_HASHES + std::vector transactions; + transactions.reserve(block.m_transactions.size()); + + for (const auto& h : block.m_transactions) { + transactions.emplace_back(h); + } +#else + const std::vector& transactions = block.m_transactions; +#endif + + if (!merkle_hash_with_proof(transactions, 0, proof, path, root)) { LOGWARN(3, "on_external_block: merkle_hash_with_proof failed for coinbase transaction"); return; } - if (!verify_merkle_proof(block.m_transactions[0], proof, path, root)) { + if (!verify_merkle_proof(transactions[0], proof, path, root)) { LOGWARN(3, "on_external_block: verify_merkle_proof failed for coinbase transaction"); return; } diff --git a/src/p2p_server.cpp b/src/p2p_server.cpp index a675587..f6d909b 100644 --- a/src/p2p_server.cpp +++ b/src/p2p_server.cpp @@ -842,7 +842,7 @@ P2PServer::Broadcast::Broadcast(const PoolBlock& block, const PoolBlock* parent) // 0 outputs in the pruned blob data->pruned_blob.push_back(0); - const uint64_t total_reward = std::accumulate(block.m_outputs.begin(), block.m_outputs.end(), 0ULL, + const uint64_t total_reward = std::accumulate(block.m_outputAmounts.begin(), block.m_outputAmounts.end(), 0ULL, [](uint64_t a, const PoolBlock::TxOutput& b) { return a + b.m_reward; diff --git a/src/pool_block.cpp b/src/pool_block.cpp index b8f38f8..9789922 100644 --- a/src/pool_block.cpp +++ b/src/pool_block.cpp @@ -90,7 +90,8 @@ PoolBlock& PoolBlock::operator=(const PoolBlock& b) m_prevId = b.m_prevId; m_nonce = b.m_nonce; m_txinGenHeight = b.m_txinGenHeight; - m_outputs = b.m_outputs; + m_ephPublicKeys = b.m_ephPublicKeys; + m_outputAmounts = b.m_outputAmounts; m_txkeyPub = b.m_txkeyPub; m_extraNonceSize = b.m_extraNonceSize; m_extraNonce = b.m_extraNonce; @@ -139,7 +140,7 @@ PoolBlock& PoolBlock::operator=(const PoolBlock& b) std::vector PoolBlock::serialize_mainchain_data(size_t* header_size, size_t* miner_tx_size, int* outputs_offset, int* outputs_blob_size, const uint32_t* nonce, const uint32_t* extra_nonce) const { std::vector data; - data.reserve(128 + m_outputs.size() * 39 + m_transactions.size() * HASH_SIZE); + data.reserve(std::min(128 + m_outputAmounts.size() * 39 + m_transactions.size() * HASH_SIZE, 131072)); // Header data.push_back(m_majorVersion); @@ -169,12 +170,15 @@ std::vector PoolBlock::serialize_mainchain_data(size_t* header_size, si *outputs_offset = outputs_offset0; } - writeVarint(m_outputs.size(), data); + writeVarint(m_outputAmounts.size(), data); + + for (size_t i = 0, n = m_outputAmounts.size(); i < n; ++i) { + const TxOutput& output = m_outputAmounts[i]; - for (const TxOutput& output : m_outputs) { writeVarint(output.m_reward, data); data.push_back(TXOUT_TO_TAGGED_KEY); - data.insert(data.end(), output.m_ephPublicKey.h, output.m_ephPublicKey.h + HASH_SIZE); + const hash h = m_ephPublicKeys[i]; + data.insert(data.end(), h.h, h.h + HASH_SIZE); data.push_back(static_cast(output.m_viewTag)); } @@ -225,8 +229,16 @@ std::vector PoolBlock::serialize_mainchain_data(size_t* header_size, si } writeVarint(m_transactions.size() - 1, data); + +#ifdef WITH_INDEXED_HASHES + for (size_t i = 1, n = m_transactions.size(); i < n; ++i) { + const hash h = m_transactions[i]; + data.insert(data.end(), h.h, h.h + HASH_SIZE); + } +#else const uint8_t* t = reinterpret_cast(m_transactions.data()); data.insert(data.end(), t + HASH_SIZE, t + m_transactions.size() * HASH_SIZE); +#endif #if POOL_BLOCK_DEBUG if ((nonce == &m_nonce) && (extra_nonce == &m_extraNonce) && !m_mainChainDataDebug.empty() && (data != m_mainChainDataDebug)) { @@ -369,15 +381,27 @@ bool PoolBlock::get_pow_hash(RandomX_Hasher_Base* hasher, uint64_t height, const memcpy(hashes, tmp.h, HASH_SIZE); count = m_transactions.size(); - uint8_t* h = reinterpret_cast(m_transactions.data()); keccak(reinterpret_cast(hashes), HASH_SIZE * 3, tmp.h); // Save the coinbase tx hash into the first element of m_transactions - memcpy(h, tmp.h, HASH_SIZE); + m_transactions[0] = static_cast(tmp); root_hash tmp_root; + +#ifdef WITH_INDEXED_HASHES + std::vector transactions; + transactions.reserve(m_transactions.size()); + + for (const auto& h : m_transactions) { + transactions.emplace_back(h); + } + + merkle_hash(transactions, tmp_root); +#else merkle_hash(m_transactions, tmp_root); +#endif + memcpy(blob + blob_size, tmp_root.h, HASH_SIZE); blob_size += HASH_SIZE; } @@ -392,13 +416,14 @@ bool PoolBlock::get_pow_hash(RandomX_Hasher_Base* hasher, uint64_t height, const uint64_t PoolBlock::get_payout(const Wallet& w) const { - for (size_t i = 0, n = m_outputs.size(); i < n; ++i) { - const TxOutput& out = m_outputs[i]; + for (size_t i = 0, n = m_outputAmounts.size(); i < n; ++i) { + const TxOutput& out = m_outputAmounts[i]; + hash eph_public_key; uint8_t view_tag; const uint8_t expected_view_tag = out.m_viewTag; - if (w.get_eph_public_key(m_txkeySec, i, eph_public_key, view_tag, &expected_view_tag) && (eph_public_key == out.m_ephPublicKey)) { + if (w.get_eph_public_key(m_txkeySec, i, eph_public_key, view_tag, &expected_view_tag) && (m_ephPublicKeys[i] == eph_public_key)) { return out.m_reward; } } diff --git a/src/pool_block.h b/src/pool_block.h index f474648..12460b9 100644 --- a/src/pool_block.h +++ b/src/pool_block.h @@ -98,17 +98,17 @@ struct PoolBlock struct TxOutput { - FORCEINLINE TxOutput() : m_ephPublicKey(), m_reward(0), m_viewTag(0) {} - FORCEINLINE TxOutput(uint64_t r, const hash& k, uint8_t view_tag) : m_ephPublicKey(k), m_reward(r), m_viewTag(view_tag) {} + FORCEINLINE TxOutput() : m_reward(0), m_viewTag(0) {} + FORCEINLINE TxOutput(uint64_t r, uint8_t view_tag) : m_reward(r), m_viewTag(view_tag) {} - hash m_ephPublicKey; uint64_t m_reward : 56; uint64_t m_viewTag : 8; }; - static_assert(sizeof(TxOutput) == sizeof(hash) + sizeof(uint64_t), "TxOutput bit packing didn't work with this compiler, fix the code!"); + static_assert(sizeof(TxOutput) == sizeof(uint64_t), "TxOutput bit packing didn't work with this compiler, fix the code!"); - std::vector m_outputs; + std::vector m_ephPublicKeys; + std::vector m_outputAmounts; hash m_txkeyPub; uint64_t m_extraNonceSize; @@ -119,7 +119,7 @@ struct PoolBlock root_hash m_merkleRoot; // All block transaction hashes including the miner transaction hash at index 0 - std::vector m_transactions; + std::vector m_transactions; // Miner's wallet Wallet m_minerWallet{ nullptr }; diff --git a/src/pool_block_parser.inl b/src/pool_block_parser.inl index 3c2e4cf..2c2c4e5 100644 --- a/src/pool_block_parser.inl +++ b/src/pool_block_parser.inl @@ -105,11 +105,14 @@ int PoolBlock::deserialize(const uint8_t* data, size_t size, const SideChain& si if (num_outputs > std::numeric_limits::max() / MIN_OUTPUT_SIZE) return __LINE__; if (static_cast(data_end - data) < num_outputs * MIN_OUTPUT_SIZE) return __LINE__; - m_outputs.resize(num_outputs); - m_outputs.shrink_to_fit(); + m_ephPublicKeys.resize(num_outputs); + m_outputAmounts.resize(num_outputs); + + m_ephPublicKeys.shrink_to_fit(); + m_outputAmounts.shrink_to_fit(); for (uint64_t i = 0; i < num_outputs; ++i) { - TxOutput& t = m_outputs[i]; + TxOutput& t = m_outputAmounts[i]; uint64_t reward; READ_VARINT(reward); @@ -124,7 +127,9 @@ int PoolBlock::deserialize(const uint8_t* data, size_t size, const SideChain& si EXPECT_BYTE(TXOUT_TO_TAGGED_KEY); - READ_BUF(t.m_ephPublicKey.h, HASH_SIZE); + hash ephPublicKey; + READ_BUF(ephPublicKey.h, HASH_SIZE); + m_ephPublicKeys[i] = ephPublicKey; uint8_t view_tag; READ_BYTE(view_tag); @@ -221,17 +226,18 @@ int PoolBlock::deserialize(const uint8_t* data, size_t size, const SideChain& si const int transactions_offset = static_cast(data - data_begin); std::vector parent_indices; + std::vector transactions; if (compact) { if (static_cast(data_end - data) < num_transactions) return __LINE__; - m_transactions.resize(1); - parent_indices.resize(1); - // limit reserved memory size because we can't check "num_transactions" properly here const uint64_t k = std::min(num_transactions + 1, 256); - m_transactions.reserve(k); + transactions.reserve(k); parent_indices.reserve(k); + transactions.resize(1); + parent_indices.resize(1); + for (uint64_t i = 0; i < num_transactions; ++i) { uint64_t parent_index; READ_VARINT(parent_index); @@ -241,7 +247,7 @@ int PoolBlock::deserialize(const uint8_t* data, size_t size, const SideChain& si READ_BUF(id.h, HASH_SIZE); } - m_transactions.emplace_back(id); + transactions.emplace_back(id); parent_indices.emplace_back(parent_index); } } @@ -249,13 +255,13 @@ int PoolBlock::deserialize(const uint8_t* data, size_t size, const SideChain& si if (num_transactions > std::numeric_limits::max() / HASH_SIZE) return __LINE__; if (static_cast(data_end - data) < num_transactions * HASH_SIZE) return __LINE__; - m_transactions.resize(1); - m_transactions.reserve(num_transactions + 1); + transactions.reserve(num_transactions + 1); + transactions.resize(1); for (uint64_t i = 0; i < num_transactions; ++i) { hash id; READ_BUF(id.h, HASH_SIZE); - m_transactions.emplace_back(id); + transactions.emplace_back(id); } } @@ -263,8 +269,6 @@ int PoolBlock::deserialize(const uint8_t* data, size_t size, const SideChain& si const int transactions_blob_size = static_cast(num_transactions) * HASH_SIZE; const int transactions_blob_size_diff = transactions_blob_size - transactions_actual_blob_size; - m_transactions.shrink_to_fit(); - #if POOL_BLOCK_DEBUG m_mainChainDataDebug.reserve((data - data_begin) + outputs_blob_size_diff + transactions_blob_size_diff); m_mainChainDataDebug.assign(data_begin, data_begin + outputs_offset); @@ -298,22 +302,42 @@ int PoolBlock::deserialize(const uint8_t* data, size_t size, const SideChain& si READ_BUF(m_parent.h, HASH_SIZE); + m_transactions.clear(); + m_transactions.reserve(transactions.size()); + if (compact) { const PoolBlock* parent = sidechain.find_block(m_parent); if (!parent) { return __LINE__; } - for (uint64_t i = 1, n = m_transactions.size(); i < n; ++i) { + const uint64_t n = transactions.size(); + + if (n > 0) { + m_transactions.emplace_back(transactions[0]); + } + + for (uint64_t i = 1; i < n; ++i) { const uint64_t parent_index = parent_indices[i]; if (parent_index) { if (parent_index >= parent->m_transactions.size()) { return __LINE__; } - m_transactions[i] = parent->m_transactions[parent_index]; + transactions[i] = parent->m_transactions[parent_index]; + m_transactions.emplace_back(parent->m_transactions[parent_index]); + } + else { + m_transactions.emplace_back(transactions[i]); } } } + else { + for (const hash& h : transactions) { + m_transactions.emplace_back(h); + } + } + + m_transactions.shrink_to_fit(); uint64_t num_uncles; READ_VARINT(num_uncles); @@ -416,7 +440,7 @@ int PoolBlock::deserialize(const uint8_t* data, size_t size, const SideChain& si return __LINE__; } - const uint8_t* transactions_blob = reinterpret_cast(m_transactions.data() + 1); + const uint8_t* transactions_blob = reinterpret_cast(transactions.data() + 1); #if POOL_BLOCK_DEBUG memcpy(m_mainChainDataDebug.data() + outputs_offset, outputs_blob.data(), outputs_blob_size); diff --git a/src/side_chain.cpp b/src/side_chain.cpp index b14687f..dc2c810 100644 --- a/src/side_chain.cpp +++ b/src/side_chain.cpp @@ -830,19 +830,22 @@ bool SideChain::get_outputs_blob(PoolBlock* block, uint64_t total_reward, std::v auto it = block->m_sidechainId.empty() ? m_blocksById.end() : m_blocksById.find(block->m_sidechainId); if (it != m_blocksById.end()) { const PoolBlock* b = it->second; - const size_t n = b->m_outputs.size(); + const size_t n = b->m_outputAmounts.size(); blob.reserve(n * 39 + 64); writeVarint(n, blob); - for (const PoolBlock::TxOutput& output : b->m_outputs) { + for (size_t i = 0; i < n; ++i) { + const PoolBlock::TxOutput& output = b->m_outputAmounts[i]; writeVarint(output.m_reward, blob); blob.emplace_back(TXOUT_TO_TAGGED_KEY); - blob.insert(blob.end(), output.m_ephPublicKey.h, output.m_ephPublicKey.h + HASH_SIZE); + const hash h = b->m_ephPublicKeys[i]; + blob.insert(blob.end(), h.h, h.h + HASH_SIZE); blob.emplace_back(static_cast(output.m_viewTag)); } - block->m_outputs = b->m_outputs; + block->m_ephPublicKeys = b->m_ephPublicKeys; + block->m_outputAmounts = b->m_outputAmounts; return true; } @@ -887,8 +890,11 @@ bool SideChain::get_outputs_blob(PoolBlock* block, uint64_t total_reward, std::v writeVarint(n, blob); - block->m_outputs.clear(); - block->m_outputs.reserve(n); + block->m_ephPublicKeys.clear(); + block->m_outputAmounts.clear(); + + block->m_ephPublicKeys.reserve(n); + block->m_outputAmounts.reserve(n); hash eph_public_key; for (size_t i = 0; i < n; ++i) { @@ -911,10 +917,12 @@ bool SideChain::get_outputs_blob(PoolBlock* block, uint64_t total_reward, std::v blob.emplace_back(view_tag); - block->m_outputs.emplace_back(tmpRewards[i], eph_public_key, view_tag); + block->m_ephPublicKeys.emplace_back(eph_public_key); + block->m_outputAmounts.emplace_back(tmpRewards[i], view_tag); } - block->m_outputs.shrink_to_fit(); + block->m_ephPublicKeys.shrink_to_fit(); + block->m_outputAmounts.shrink_to_fit(); return true; } @@ -1082,12 +1090,12 @@ double SideChain::get_reward_share(const Wallet& w) const const PoolBlock* tip = m_chainTip; if (tip) { hash eph_public_key; - for (size_t i = 0, n = tip->m_outputs.size(); i < n; ++i) { - const PoolBlock::TxOutput& out = tip->m_outputs[i]; + for (size_t i = 0, n = tip->m_outputAmounts.size(); i < n; ++i) { + const PoolBlock::TxOutput& out = tip->m_outputAmounts[i]; if (!reward) { uint8_t view_tag; const uint8_t expected_view_tag = out.m_viewTag; - if (w.get_eph_public_key(tip->m_txkeySec, i, eph_public_key, view_tag, &expected_view_tag) && (out.m_ephPublicKey == eph_public_key)) { + if (w.get_eph_public_key(tip->m_txkeySec, i, eph_public_key, view_tag, &expected_view_tag) && (tip->m_ephPublicKeys[i] == eph_public_key)) { reward = out.m_reward; } } @@ -1699,16 +1707,16 @@ void SideChain::verify(PoolBlock* block) return; } - if (shares.size() != block->m_outputs.size()) { + if (shares.size() != block->m_outputAmounts.size()) { LOGWARN(3, "block at height = " << block->m_sidechainHeight << ", id = " << block->m_sidechainId << ", mainchain height = " << block->m_txinGenHeight - << " has invalid number of outputs: got " << block->m_outputs.size() << ", expected " << shares.size()); + << " has invalid number of outputs: got " << block->m_outputAmounts.size() << ", expected " << shares.size()); block->m_invalid = true; return; } - uint64_t total_reward = std::accumulate(block->m_outputs.begin(), block->m_outputs.end(), 0ULL, + uint64_t total_reward = std::accumulate(block->m_outputAmounts.begin(), block->m_outputAmounts.end(), 0ULL, [](uint64_t a, const PoolBlock::TxOutput& b) { return a + b.m_reward; @@ -1723,17 +1731,17 @@ void SideChain::verify(PoolBlock* block) return; } - if (rewards.size() != block->m_outputs.size()) { + if (rewards.size() != block->m_outputAmounts.size()) { LOGWARN(3, "block at height = " << block->m_sidechainHeight << ", id = " << block->m_sidechainId << ", mainchain height = " << block->m_txinGenHeight - << " has invalid number of outputs: got " << block->m_outputs.size() << ", expected " << rewards.size()); + << " has invalid number of outputs: got " << block->m_outputAmounts.size() << ", expected " << rewards.size()); block->m_invalid = true; return; } for (size_t i = 0, n = rewards.size(); i < n; ++i) { - const PoolBlock::TxOutput& out = block->m_outputs[i]; + const PoolBlock::TxOutput& out = block->m_outputAmounts[i]; if (rewards[i] != out.m_reward) { LOGWARN(3, "block at height = " << block->m_sidechainHeight << @@ -1764,7 +1772,7 @@ void SideChain::verify(PoolBlock* block) return; } - if (eph_public_key != out.m_ephPublicKey) { + if (eph_public_key != block->m_ephPublicKeys[i]) { LOGWARN(3, "block at height = " << block->m_sidechainHeight << ", id = " << block->m_sidechainId << ", mainchain height = " << block->m_txinGenHeight << diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e20c662..e677c40 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -6,7 +6,8 @@ include(cmake/standard.cmake) option(STATIC_LIBS "Use locally built libuv and libzmq static libs" OFF) option(WITH_LTO "Use link-time compiler optimization (if linking fails for you, run cmake with -DWITH_LTO=OFF)" ON) option(WITH_COVERAGE "Generate code coverage data" OFF) -option(DEV_DEBUG "[Developer only] Compile a debug build" OFF) +option(WITH_INDEXED_HASHES "Save memory used for storing transaction hashes and public keys (a bit slower block verification, use it only if you compile for a very low memory system)" ON) +option(DEV_DEBUG "[Developer only] Compile a debug build" ON) if (DEV_DEBUG) add_definitions(-DDEV_DEBUG) @@ -84,6 +85,11 @@ set(SOURCES ../src/wallet.cpp ) +if (WITH_INDEXED_HASHES) + add_compile_definitions(WITH_INDEXED_HASHES) + set(SOURCES ${SOURCES} ../src/indexed_hash.cpp) +endif() + if (AMD64) set(SOURCES ${SOURCES} ../src/keccak_bmi.cpp) if (CMAKE_C_COMPILER_ID MATCHES GNU OR CMAKE_C_COMPILER_ID MATCHES Clang) diff --git a/tests/src/block_template_tests.cpp b/tests/src/block_template_tests.cpp index 98b3fb4..5cfe1f7 100644 --- a/tests/src/block_template_tests.cpp +++ b/tests/src/block_template_tests.cpp @@ -36,7 +36,7 @@ static hash H(const char* s) TEST(block_template, update) { init_crypto_cache(); - + { SideChain sidechain(nullptr, NetworkType::Mainnet); BlockTemplate tpl(&sidechain, nullptr); tpl.rng().seed(123); @@ -83,8 +83,11 @@ TEST(block_template, update) // Test 2: mempool with high fee and low fee transactions, it must choose high fee transactions for (uint64_t i = 0; i < 513; ++i) { + hash h; + h.u64()[0] = i; + TxMempoolData tx; - *reinterpret_cast(tx.id.h) = i; + tx.id = static_cast(h); tx.fee = (i < 256) ? 30000000 : 60000000; tx.weight = 1500; mempool.add(tx); @@ -115,7 +118,7 @@ TEST(block_template, update) ASSERT_EQ(b->m_transactions.size(), 203); for (size_t i = 1; i < b->m_transactions.size(); ++i) { - ASSERT_GE(*reinterpret_cast(b->m_transactions[i].h), 256); + ASSERT_GE(static_cast(b->m_transactions[i]).u64()[0], 256); } tpl.get_hashing_blobs(0, 1000, blobs, height, diff, aux_diff, sidechain_diff, seed_hash, nonce_offset, template_id); @@ -135,8 +138,11 @@ TEST(block_template, update) std::vector transactions; for (uint64_t i = 0; i < 10; ++i) { + hash h; + h.u64()[0] = i; + TxMempoolData tx; - *reinterpret_cast(tx.id.h) = i; + tx.id = static_cast(h); tx.fee = 30000000; tx.weight = 1500; transactions.push_back(tx); @@ -171,10 +177,11 @@ TEST(block_template, update) std::mt19937_64 rng; for (uint64_t i = 0; i < 10000; ++i) { + hash h; + h.u64()[0] = i; + TxMempoolData tx; - - *reinterpret_cast(tx.id.h) = i; - + tx.id = static_cast(h); tx.weight = 1500 + (rng() % 10007); tx.fee = 30000000 + (rng() % 100000007); @@ -199,14 +206,18 @@ TEST(block_template, update) keccak(blobs.data(), static_cast(blobs.size()), blobs_hash.h); ASSERT_EQ(blobs_hash, H("4f62562aa84400eb085f58447d8daa45257369f1ec046b2150212329c9e86ae4")); - + } destroy_crypto_cache(); + +#ifdef WITH_INDEXED_HASHES + indexed_hash::cleanup_storage(); +#endif } TEST(block_template, submit_sidechain_block) { init_crypto_cache(); - + { SideChain sidechain(nullptr, NetworkType::Mainnet, "unit_test"); ASSERT_EQ(sidechain.consensus_hash(), H("81d45b62c10afa4fdda7cebb02dd5ad82c43b577eb3fb0857824427c55fd8a8d")); @@ -270,8 +281,12 @@ TEST(block_template, submit_sidechain_block) ASSERT_EQ(tip->m_sidechainHeight, sidechain.chain_window_size() * 3 - 1); ASSERT_EQ(tip->m_sidechainId, H("12d57571a28d62d2b6dca3a647500d23ac22864138b22a133f237b459a0862da")); - + } destroy_crypto_cache(); + +#ifdef WITH_INDEXED_HASHES + indexed_hash::cleanup_storage(); +#endif } } diff --git a/tests/src/crypto_tests.cpp b/tests/src/crypto_tests.cpp index aab9fca..879c99a 100644 --- a/tests/src/crypto_tests.cpp +++ b/tests/src/crypto_tests.cpp @@ -26,7 +26,7 @@ namespace p2pool { TEST(crypto, derivation) { init_crypto_cache(); - + { hash pub, sec; generate_keys(pub, sec); ASSERT_TRUE(check_keys(pub, sec)); @@ -105,9 +105,13 @@ TEST(crypto, derivation) } } while (!f.eof()); } - + } clear_crypto_cache(0); destroy_crypto_cache(); + +#ifdef WITH_INDEXED_HASHES + indexed_hash::cleanup_storage(); +#endif } } diff --git a/tests/src/pool_block_tests.cpp b/tests/src/pool_block_tests.cpp index d33a1d0..76d29a0 100644 --- a/tests/src/pool_block_tests.cpp +++ b/tests/src/pool_block_tests.cpp @@ -39,7 +39,7 @@ static hash H(const char* s) TEST(pool_block, deserialize) { init_crypto_cache(); - + { PoolBlock b; SideChain sidechain(nullptr, NetworkType::Mainnet, "default"); @@ -93,7 +93,8 @@ TEST(pool_block, deserialize) ASSERT_EQ(b.m_timestamp, 1728813765U); ASSERT_EQ(b.m_nonce, 352454720U); ASSERT_EQ(b.m_txinGenHeight, 3258099U); - ASSERT_EQ(b.m_outputs.size(), 27U); + ASSERT_EQ(b.m_ephPublicKeys.size(), 27U); + ASSERT_EQ(b.m_outputAmounts.size(), 27U); ASSERT_EQ(b.m_extraNonceSize, 4U); ASSERT_EQ(b.m_extraNonce, 2983923783U); ASSERT_EQ(b.m_transactions.size(), 21U); @@ -128,14 +129,18 @@ TEST(pool_block, deserialize) ASSERT_EQ(b.serialize_mainchain_data(), mainchain_data); ASSERT_EQ(b.serialize_sidechain_data(), sidechain_data); - + } destroy_crypto_cache(); + +#ifdef WITH_INDEXED_HASHES + indexed_hash::cleanup_storage(); +#endif } TEST(pool_block, verify) { init_crypto_cache(); - + { struct STest { const char* m_poolName; @@ -151,7 +156,7 @@ TEST(pool_block, verify) { "mini", "sidechain_dump_mini.dat", 3456189, 11207082, 578, false, H("08debd1378bae899017eb58362f4c638d78e5218558025142dcbc2651c76b27e") }, { "mini", "sidechain_dump_mini.dat", 3456189, 11207082, 578, true, H("08debd1378bae899017eb58362f4c638d78e5218558025142dcbc2651c76b27e") }, { "nano", "sidechain_dump_nano.dat", 3456189, 188542, 115, false, H("dd667c41eb15ffb0eb662065545dc0dfbbcac8393348a4fc0a7367040319b0d5") }, - { "nano", "sidechain_dump_nano.dat", 3456189, 188542, 115, true, H("dd667c41eb15ffb0eb662065545dc0dfbbcac8393348a4fc0a7367040319b0d5") }, + { "nano", "sidechain_dump_nano.dat", 3456189, 188542, 115, true, H("dd667c41eb15ffb0eb662065545dc0dfbbcac8393348a4fc0a7367040319b0d5") }, }; for (const STest& t : tests) @@ -231,8 +236,11 @@ TEST(pool_block, verify) Mempool mempool; for (uint64_t i = 0; i < 8192; ++i) { + hash h; + h.u64()[0] = i; + TxMempoolData tx; - *reinterpret_cast(tx.id.h) = i; + tx.id = static_cast(h); tx.fee = (r() % 1'000'000'000) + 30'000'000; tx.weight = (r() % 20'000) + 1'500; mempool.add(tx); @@ -302,8 +310,12 @@ TEST(pool_block, verify) ASSERT_EQ(v1, tip_full_blob); } } - + } destroy_crypto_cache(); + +#ifdef WITH_INDEXED_HASHES + indexed_hash::cleanup_storage(); +#endif } }