De-duplicate tx hashes and pub keys to save memory (off by default) (#382)

P2Pool-main: 8.2 MB saved
P2Pool-mini: 66 MB saved
P2Pool-nano: 25.2 MB saved

The feature is available only when building from source and is intended for use on low-memory systems (for example, a VPS server with < 1 GB RAM).

It only makes sense to use with `--no-cache --no-randomx` in the command line because cache and RandomX hasher take much more memory.

tests/src/block_template_tests.cpp

@@ -36,7 +36,7 @@ static hash H(const char* s)
 TEST(block_template, update)
 {
 	init_crypto_cache();
-
+	{
 	SideChain sidechain(nullptr, NetworkType::Mainnet);
 	BlockTemplate tpl(&sidechain, nullptr);
 	tpl.rng().seed(123);
@@ -83,8 +83,11 @@ TEST(block_template, update)
 
 	// Test 2: mempool with high fee and low fee transactions, it must choose high fee transactions
 	for (uint64_t i = 0; i < 513; ++i) {
+		hash h;
+		h.u64()[0] = i;
+
 		TxMempoolData tx;
-		*reinterpret_cast<uint64_t*>(tx.id.h) = i;
+		tx.id = static_cast<indexed_hash>(h);
 		tx.fee = (i < 256) ? 30000000 : 60000000;
 		tx.weight = 1500;
 		mempool.add(tx);
@@ -115,7 +118,7 @@ TEST(block_template, update)
 	ASSERT_EQ(b->m_transactions.size(), 203);
 
 	for (size_t i = 1; i < b->m_transactions.size(); ++i) {
-		ASSERT_GE(*reinterpret_cast<const uint64_t*>(b->m_transactions[i].h), 256);
+		ASSERT_GE(static_cast<hash>(b->m_transactions[i]).u64()[0], 256);
 	}
 
 	tpl.get_hashing_blobs(0, 1000, blobs, height, diff, aux_diff, sidechain_diff, seed_hash, nonce_offset, template_id);
@@ -135,8 +138,11 @@ TEST(block_template, update)
 	std::vector<TxMempoolData> transactions;
 
 	for (uint64_t i = 0; i < 10; ++i) {
+		hash h;
+		h.u64()[0] = i;
+
 		TxMempoolData tx;
-		*reinterpret_cast<uint64_t*>(tx.id.h) = i;
+		tx.id = static_cast<indexed_hash>(h);
 		tx.fee = 30000000;
 		tx.weight = 1500;
 		transactions.push_back(tx);
@@ -171,10 +177,11 @@ TEST(block_template, update)
 	std::mt19937_64 rng;
 
 	for (uint64_t i = 0; i < 10000; ++i) {
+		hash h;
+		h.u64()[0] = i;
+
 		TxMempoolData tx;
-
-		*reinterpret_cast<uint64_t*>(tx.id.h) = i;
-
+		tx.id = static_cast<indexed_hash>(h);
 		tx.weight = 1500 + (rng() % 10007);
 		tx.fee = 30000000 + (rng() % 100000007);
 
@@ -199,14 +206,18 @@ TEST(block_template, update)
 
 	keccak(blobs.data(), static_cast<int>(blobs.size()), blobs_hash.h);
 	ASSERT_EQ(blobs_hash, H("4f62562aa84400eb085f58447d8daa45257369f1ec046b2150212329c9e86ae4"));
-
+	}
 	destroy_crypto_cache();
+
+#ifdef WITH_INDEXED_HASHES
+	indexed_hash::cleanup_storage();
+#endif
 }
 
 TEST(block_template, submit_sidechain_block)
 {
 	init_crypto_cache();
-
+	{
 	SideChain sidechain(nullptr, NetworkType::Mainnet, "unit_test");
 
 	ASSERT_EQ(sidechain.consensus_hash(), H("81d45b62c10afa4fdda7cebb02dd5ad82c43b577eb3fb0857824427c55fd8a8d"));
@@ -270,8 +281,12 @@ TEST(block_template, submit_sidechain_block)
 	ASSERT_EQ(tip->m_sidechainHeight, sidechain.chain_window_size() * 3 - 1);
 
 	ASSERT_EQ(tip->m_sidechainId, H("12d57571a28d62d2b6dca3a647500d23ac22864138b22a133f237b459a0862da"));
-
+	}
 	destroy_crypto_cache();
+
+#ifdef WITH_INDEXED_HASHES
+	indexed_hash::cleanup_storage();
+#endif
 }
 
 }

tests/src/crypto_tests.cpp

@@ -26,7 +26,7 @@ namespace p2pool {
 TEST(crypto, derivation)
 {
 	init_crypto_cache();
-
+	{
 	hash pub, sec;
 	generate_keys(pub, sec);
 	ASSERT_TRUE(check_keys(pub, sec));
@@ -105,9 +105,13 @@ TEST(crypto, derivation)
 			}
 		} while (!f.eof());
 	}
-
+	}
 	clear_crypto_cache(0);
 	destroy_crypto_cache();
+
+#ifdef WITH_INDEXED_HASHES
+	indexed_hash::cleanup_storage();
+#endif
 }
 
 }

tests/src/pool_block_tests.cpp

@@ -39,7 +39,7 @@ static hash H(const char* s)
 TEST(pool_block, deserialize)
 {
 	init_crypto_cache();
-
+	{
 	PoolBlock b;
 	SideChain sidechain(nullptr, NetworkType::Mainnet, "default");
 
@@ -93,7 +93,8 @@ TEST(pool_block, deserialize)
 	ASSERT_EQ(b.m_timestamp, 1728813765U);
 	ASSERT_EQ(b.m_nonce, 352454720U);
 	ASSERT_EQ(b.m_txinGenHeight, 3258099U);
-	ASSERT_EQ(b.m_outputs.size(), 27U);
+	ASSERT_EQ(b.m_ephPublicKeys.size(), 27U);
+	ASSERT_EQ(b.m_outputAmounts.size(), 27U);
 	ASSERT_EQ(b.m_extraNonceSize, 4U);
 	ASSERT_EQ(b.m_extraNonce, 2983923783U);
 	ASSERT_EQ(b.m_transactions.size(), 21U);
@@ -128,14 +129,18 @@ TEST(pool_block, deserialize)
 
 	ASSERT_EQ(b.serialize_mainchain_data(), mainchain_data);
 	ASSERT_EQ(b.serialize_sidechain_data(), sidechain_data);
-
+	}
 	destroy_crypto_cache();
+
+#ifdef WITH_INDEXED_HASHES
+	indexed_hash::cleanup_storage();
+#endif
 }
 
 TEST(pool_block, verify)
 {
 	init_crypto_cache();
-
+	{
 	struct STest
 	{
 		const char* m_poolName;
@@ -151,7 +156,7 @@ TEST(pool_block, verify)
 		{ "mini", "sidechain_dump_mini.dat", 3456189, 11207082, 578, false, H("08debd1378bae899017eb58362f4c638d78e5218558025142dcbc2651c76b27e") },
 		{ "mini", "sidechain_dump_mini.dat", 3456189, 11207082, 578, true, H("08debd1378bae899017eb58362f4c638d78e5218558025142dcbc2651c76b27e") },
 		{ "nano", "sidechain_dump_nano.dat", 3456189, 188542, 115, false, H("dd667c41eb15ffb0eb662065545dc0dfbbcac8393348a4fc0a7367040319b0d5") },
-	{ "nano", "sidechain_dump_nano.dat", 3456189, 188542, 115, true, H("dd667c41eb15ffb0eb662065545dc0dfbbcac8393348a4fc0a7367040319b0d5") },
+		{ "nano", "sidechain_dump_nano.dat", 3456189, 188542, 115, true, H("dd667c41eb15ffb0eb662065545dc0dfbbcac8393348a4fc0a7367040319b0d5") },
 	};
 
 	for (const STest& t : tests)
@@ -231,8 +236,11 @@ TEST(pool_block, verify)
 			Mempool mempool;
 
 			for (uint64_t i = 0; i < 8192; ++i) {
+				hash h;
+				h.u64()[0] = i;
+
 				TxMempoolData tx;
-				*reinterpret_cast<uint64_t*>(tx.id.h) = i;
+				tx.id = static_cast<indexed_hash>(h);
 				tx.fee = (r() % 1'000'000'000) + 30'000'000;
 				tx.weight = (r() % 20'000) + 1'500;
 				mempool.add(tx);
@@ -302,8 +310,12 @@ TEST(pool_block, verify)
 			ASSERT_EQ(v1, tip_full_blob);
 		}
 	}
-
+	}
 	destroy_crypto_cache();
+
+#ifdef WITH_INDEXED_HASHES
+	indexed_hash::cleanup_storage();
+#endif
 }
 
 }