Remove compact/pruned blob code - it displeases the carrot

This commit is contained in:
Matt Hess
2025-12-10 15:56:49 +00:00
parent 4a2bec3f29
commit d5a8bbcc4a
7 changed files with 98 additions and 271 deletions

View File

@@ -234,7 +234,7 @@ void BlockCache::load_all(const SideChain& side_chain, P2PServer& server)
continue;
}
if (block.deserialize(data + sizeof(uint32_t), n, side_chain, uv_default_loop_checked(), false) == 0) {
if (block.deserialize(data + sizeof(uint32_t), n, side_chain, uv_default_loop_checked()) == 0) {
server.add_cached_block(block);
++blocks_loaded;
}

View File

@@ -810,7 +810,7 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const
}
}
PoolBlock check;
const int result = check.deserialize(m_fullDataBlob.data(), m_fullDataBlob.size(), *m_sidechain, nullptr, false);
const int result = check.deserialize(m_fullDataBlob.data(), m_fullDataBlob.size(), *m_sidechain, nullptr);
if (result != 0) {
LOGERR(1, "pool block blob generation and/or parsing is broken, error " << result);
}
@@ -1802,7 +1802,7 @@ bool BlockTemplate::submit_sidechain_block(uint32_t template_id, uint32_t nonce,
buf.insert(buf.end(), sidechain_data.begin(), sidechain_data.end());
PoolBlock check;
const int result = check.deserialize(buf.data(), buf.size(), *m_sidechain, nullptr, false);
const int result = check.deserialize(buf.data(), buf.size(), *m_sidechain, nullptr);
if (result != 0) {
LOGERR(1, "pool block blob generation and/or parsing is broken, error " << result);
}

View File

@@ -961,85 +961,24 @@ void P2PServer::Peer::normalize()
}
}
P2PServer::Broadcast::Broadcast(const PoolBlock& block, const PoolBlock* parent)
: id(block.m_sidechainId)
, received_timestamp(block.m_receivedTimestamp)
P2PServer::Broadcast::Broadcast(const PoolBlock& block)
: id(block.m_sidechainId)
, received_timestamp(block.m_receivedTimestamp)
{
Broadcast* data = this;
Broadcast* data = this;
int outputs_offset, outputs_blob_size;
const std::vector<uint8_t> mainchain_data = block.serialize_mainchain_data(nullptr, nullptr, &outputs_offset, &outputs_blob_size);
const std::vector<uint8_t> sidechain_data = block.serialize_sidechain_data();
data->blob.reserve(mainchain_data.size() + sidechain_data.size());
data->blob = mainchain_data;
data->blob.insert(data->blob.end(), sidechain_data.begin(), sidechain_data.end());
int outputs_offset, outputs_blob_size;
const std::vector<uint8_t> mainchain_data = block.serialize_mainchain_data(nullptr, nullptr, &outputs_offset, &outputs_blob_size);
const std::vector<uint8_t> sidechain_data = block.serialize_sidechain_data();
data->blob.reserve(mainchain_data.size() + sidechain_data.size());
data->blob = mainchain_data;
data->blob.insert(data->blob.end(), sidechain_data.begin(), sidechain_data.end());
data->pruned_blob.reserve(mainchain_data.size() + sidechain_data.size() + 16 - outputs_blob_size);
data->pruned_blob.assign(mainchain_data.begin(), mainchain_data.begin() + outputs_offset);
// 0 outputs in the pruned blob
data->pruned_blob.push_back(0);
const uint64_t total_reward = std::accumulate(block.m_outputAmounts.begin(), block.m_outputAmounts.end(), 0ULL,
[](uint64_t a, const PoolBlock::TxOutput& b)
{
return a + b.m_reward;
});
writeVarint(total_reward, data->pruned_blob);
writeVarint(outputs_blob_size, data->pruned_blob);
data->pruned_blob.insert(data->pruned_blob.end(), block.m_sidechainId.h, block.m_sidechainId.h + HASH_SIZE);
data->pruned_blob.insert(data->pruned_blob.end(), mainchain_data.begin() + outputs_offset + outputs_blob_size, mainchain_data.end());
const size_t N = block.m_transactions.size();
if ((N > 1) && parent && (parent->m_transactions.size() > 1)) {
unordered_map<hash, size_t> parent_transactions;
parent_transactions.reserve(parent->m_transactions.size());
for (size_t i = 1; i < parent->m_transactions.size(); ++i) {
parent_transactions.emplace(parent->m_transactions[i], i);
}
// Reserve 1 additional byte per transaction to be ready for the worst case (all transactions are different in the parent block)
data->compact_blob.reserve(data->pruned_blob.capacity() + (N - 1));
// Copy pruned_blob without the transaction list
data->compact_blob.assign(data->pruned_blob.begin(), data->pruned_blob.end() - static_cast<uint32_t>((N - 1) * HASH_SIZE));
// Process transaction hashes one by one
size_t num_found = 0;
for (size_t i = 1; i < N; ++i) {
const hash& tx = block.m_transactions[i];
auto it = parent_transactions.find(tx);
if (it != parent_transactions.end()) {
writeVarint(it->second, data->compact_blob);
++num_found;
}
else {
data->compact_blob.push_back(0);
data->compact_blob.insert(data->compact_blob.end(), tx.h, tx.h + HASH_SIZE);
}
}
LOGINFO(6, "compact blob: " << num_found << '/' << (N - 1) << " transactions were found in the parent block");
data->compact_blob.insert(data->compact_blob.end(), sidechain_data.begin(), sidechain_data.end());
}
data->pruned_blob.insert(data->pruned_blob.end(), sidechain_data.begin(), sidechain_data.end());
// Carrot v1 blocks have encrypted anchors that can't be deterministically reconstructed
// Force full blob broadcasts by clearing pruned/compact blobs
data->pruned_blob.clear();
data->compact_blob.clear();
data->ancestor_hashes.reserve(block.m_uncles.size() + 1);
data->ancestor_hashes = block.m_uncles;
data->ancestor_hashes.push_back(block.m_parent);
data->ancestor_hashes.reserve(block.m_uncles.size() + 1);
data->ancestor_hashes = block.m_uncles;
data->ancestor_hashes.push_back(block.m_parent);
}
void P2PServer::broadcast(const PoolBlock& block, const PoolBlock* parent)
void P2PServer::broadcast(const PoolBlock& block)
{
// Don't broadcast blocks when shutting down
if (m_finished.load()) {
@@ -1058,9 +997,9 @@ void P2PServer::broadcast(const PoolBlock& block, const PoolBlock* parent)
return;
}
Broadcast* data = new Broadcast(block, parent);
Broadcast* data = new Broadcast(block);
LOGINFO(5, "Broadcasting block " << block.m_sidechainId << " (height " << block.m_sidechainHeight << "): " << data->compact_blob.size() << '/' << data->pruned_blob.size() << '/' << data->blob.size() << " bytes (compact/pruned/full)");
LOGINFO(5, "Broadcasting block " << block.m_sidechainId << " (height " << block.m_sidechainHeight << "): " << data->blob.size() << " bytes");
MutexLock lock(m_broadcastLock);
@@ -1097,31 +1036,15 @@ void P2PServer::on_broadcast()
return;
}
if (pool_block_debug()) {
for (Broadcast* data : broadcast_queue) {
if (!data->compact_blob.empty()) {
PoolBlock check;
const int result = check.deserialize(data->compact_blob.data(), data->compact_blob.size(), m_pool->side_chain(), nullptr, true);
if (result != 0) {
LOGERR(1, "compact blob broadcast is broken, error " << result);
}
}
{
PoolBlock check;
const int result = check.deserialize(data->pruned_blob.data(), data->pruned_blob.size(), m_pool->side_chain(), nullptr, false);
if (result != 0) {
LOGERR(1, "pruned blob broadcast is broken, error " << result);
}
}
{
PoolBlock check;
const int result = check.deserialize(data->blob.data(), data->blob.size(), m_pool->side_chain(), nullptr, false);
if (result != 0) {
LOGERR(1, "full blob broadcast is broken, error " << result);
}
}
}
}
if (pool_block_debug()) {
for (Broadcast* data : broadcast_queue) {
PoolBlock check;
const int result = check.deserialize(data->blob.data(), data->blob.size(), m_pool->side_chain(), nullptr);
if (result != 0) {
LOGERR(1, "full blob broadcast is broken, error " << result);
}
}
}
for (P2PClient* client = static_cast<P2PClient*>(m_connectedClientsList->m_next); client != m_connectedClientsList; client = static_cast<P2PClient*>(client->m_next)) {
if (!client->is_good()) {
@@ -1129,80 +1052,43 @@ void P2PServer::on_broadcast()
}
for (Broadcast* data : broadcast_queue) {
const bool result = send(client, [client, data](uint8_t* buf, size_t buf_size) -> size_t
{
uint8_t* p = buf;
const bool result = send(client, [client, data](uint8_t* buf, size_t buf_size) -> size_t
{
uint8_t* p = buf;
const hash* a = client->m_broadcastedHashes;
const hash* b = client->m_broadcastedHashes + array_size(&P2PClient::m_broadcastedHashes);
const hash* a = client->m_broadcastedHashes;
const hash* b = client->m_broadcastedHashes + array_size(&P2PClient::m_broadcastedHashes);
// If this peer already broadcasted this block to us, we don't need to broadcast it back, we just need to notify the peer
if ((client->m_protocolVersion >= PROTOCOL_VERSION_1_2) && (std::find(a, b, data->id) != b)) {
LOGINFO(6, "sending BLOCK_NOTIFY to " << log::Gray() << static_cast<char*>(client->m_addrString));
// If this peer already broadcasted this block to us, just notify
if ((client->m_protocolVersion >= PROTOCOL_VERSION_1_2) && (std::find(a, b, data->id) != b)) {
LOGINFO(6, "sending BLOCK_NOTIFY to " << log::Gray() << static_cast<char*>(client->m_addrString));
if (buf_size < 1 + HASH_SIZE) {
return 0;
}
if (buf_size < 1 + HASH_SIZE) {
return 0;
}
*(p++) = static_cast<uint8_t>(MessageId::BLOCK_NOTIFY);
*(p++) = static_cast<uint8_t>(MessageId::BLOCK_NOTIFY);
memcpy(p, data->id.h, HASH_SIZE);
p += HASH_SIZE;
memcpy(p, data->id.h, HASH_SIZE);
p += HASH_SIZE;
return p - buf;
}
return p - buf;
}
bool send_pruned = !data->pruned_blob.empty();
bool send_compact = (client->m_protocolVersion >= PROTOCOL_VERSION_1_1) && !data->compact_blob.empty() && (data->compact_blob.size() < data->pruned_blob.size());
for (const hash& id : data->ancestor_hashes) {
if (std::find(a, b, id) == b) {
send_pruned = false;
send_compact = false;
break;
}
}
if (send_pruned) {
LOGINFO(6, "sending BLOCK_BROADCAST " << (send_compact ? "(compact)" : "(pruned) ") << " to " << log::Gray() << static_cast<char*>(client->m_addrString));
const std::vector<uint8_t>& blob = send_compact ? data->compact_blob : data->pruned_blob;
const uint32_t len = static_cast<uint32_t>(blob.size());
if (buf_size < 1 + sizeof(uint32_t) + len) {
return 0;
}
*(p++) = static_cast<uint8_t>(send_compact ? MessageId::BLOCK_BROADCAST_COMPACT : MessageId::BLOCK_BROADCAST);
memcpy(p, &len, sizeof(uint32_t));
p += sizeof(uint32_t);
if (len) {
memcpy(p, blob.data(), len);
p += len;
}
}
else {
LOGINFO(5, "sending BLOCK_BROADCAST (full) to " << log::Gray() << static_cast<char*>(client->m_addrString));
const uint32_t len = static_cast<uint32_t>(data->blob.size());
if (buf_size < 1 + sizeof(uint32_t) + len) {
return 0;
}
*(p++) = static_cast<uint8_t>(MessageId::BLOCK_BROADCAST);
memcpy(p, &len, sizeof(uint32_t));
p += sizeof(uint32_t);
if (len) {
memcpy(p, data->blob.data(), len);
p += len;
}
}
return p - buf;
});
// Always send full blob (Carrot v1 requires it)
LOGINFO(5, "sending BLOCK_BROADCAST to " << log::Gray() << static_cast<char*>(client->m_addrString));
const uint32_t len = static_cast<uint32_t>(data->blob.size());
if (buf_size < 1 + sizeof(uint32_t) + len) {
return 0;
}
*(p++) = static_cast<uint8_t>(MessageId::BLOCK_BROADCAST);
memcpy(p, &len, sizeof(uint32_t));
p += sizeof(uint32_t);
if (len) {
memcpy(p, data->blob.data(), len);
p += len;
}
return p - buf;
});
if (!result) {
LOGWARN(5, "failed to broadcast a block to " << static_cast<char*>(client->m_addrString) << ", disconnecting");
client->close();
@@ -1290,7 +1176,7 @@ int P2PServer::external_listen_port() const
return params.m_p2pExternalPort ? params.m_p2pExternalPort : m_listenPort;
}
int P2PServer::deserialize_block(const uint8_t* buf, uint32_t size, bool compact, uint64_t received_timestamp)
int P2PServer::deserialize_block(const uint8_t* buf, uint32_t size, uint64_t received_timestamp)
{
int result;
@@ -1299,7 +1185,7 @@ int P2PServer::deserialize_block(const uint8_t* buf, uint32_t size, bool compact
result = m_blockDeserializeResult;
}
else {
result = m_block->deserialize(buf, size, m_pool->side_chain(), &m_loop, compact);
result = m_block->deserialize(buf, size, m_pool->side_chain(), &m_loop);
m_blockDeserializeBuf.assign(buf, buf + size);
m_blockDeserializeResult = result;
m_lookForMissingBlocks = true;
@@ -2145,25 +2031,22 @@ bool P2PServer::P2PClient::on_read(const char* data, uint32_t size)
}
break;
case MessageId::BLOCK_BROADCAST:
case MessageId::BLOCK_BROADCAST_COMPACT:
{
const bool compact = (id == MessageId::BLOCK_BROADCAST_COMPACT);
LOGINFO(6, "peer " << log::Gray() << static_cast<char*>(m_addrString) << log::NoColor() << " sent " << (compact ? "BLOCK_BROADCAST_COMPACT" : "BLOCK_BROADCAST"));
if (bytes_left >= 1 + sizeof(uint32_t)) {
const uint32_t block_size = read_unaligned(reinterpret_cast<uint32_t*>(buf + 1));
if (bytes_left >= 1 + sizeof(uint32_t) + block_size) {
bytes_read = 1 + sizeof(uint32_t) + block_size;
if (!on_block_broadcast(buf + 1 + sizeof(uint32_t), block_size, compact)) {
ban(DEFAULT_BAN_TIME);
server->remove_peer_from_list(this);
return false;
}
}
}
}
break;
case MessageId::BLOCK_BROADCAST:
{
LOGINFO(6, "peer " << log::Gray() << static_cast<char*>(m_addrString) << log::NoColor() << " sent BLOCK_BROADCAST");
if (bytes_left >= 1 + sizeof(uint32_t)) {
const uint32_t block_size = read_unaligned(reinterpret_cast<uint32_t*>(buf + 1));
if (bytes_left >= 1 + sizeof(uint32_t) + block_size) {
bytes_read = 1 + sizeof(uint32_t) + block_size;
if (!on_block_broadcast(buf + 1 + sizeof(uint32_t), block_size)) {
ban(DEFAULT_BAN_TIME);
server->remove_peer_from_list(this);
return false;
}
}
}
}
break;
case MessageId::PEER_LIST_REQUEST:
LOGINFO(6, "peer " << log::Gray() << static_cast<char*>(m_addrString) << log::NoColor() << " sent PEER_LIST_REQUEST");
@@ -2742,7 +2625,7 @@ bool P2PServer::P2PClient::on_block_response(const uint8_t* buf, uint32_t size,
MutexLock lock(server->m_blockLock);
const int result = server->deserialize_block(buf, size, false, received_timestamp);
const int result = server->deserialize_block(buf, size, received_timestamp);
if (result != 0) {
LOGWARN(3, "peer " << static_cast<char*>(m_addrString) << " sent an invalid block, error " << result);
return false;
@@ -2778,7 +2661,7 @@ bool P2PServer::P2PClient::on_block_response(const uint8_t* buf, uint32_t size,
return handle_incoming_block_async(block, max_time_delta);
}
bool P2PServer::P2PClient::on_block_broadcast(const uint8_t* buf, uint32_t size, bool compact)
bool P2PServer::P2PClient::on_block_broadcast(const uint8_t* buf, uint32_t size)
{
if (!size) {
LOGWARN(3, "peer " << static_cast<char*>(m_addrString) << " broadcasted an empty block");
@@ -2791,7 +2674,7 @@ bool P2PServer::P2PClient::on_block_broadcast(const uint8_t* buf, uint32_t size,
MutexLock lock(server->m_blockLock);
const int result = server->deserialize_block(buf, size, compact, received_timestamp);
const int result = server->deserialize_block(buf, size, received_timestamp);
if (result != 0) {
LOGWARN(3, "peer " << static_cast<char*>(m_addrString) << " sent an invalid block, error " << result);
return false;

View File

@@ -58,7 +58,6 @@ public:
BLOCK_BROADCAST,
PEER_LIST_REQUEST,
PEER_LIST_RESPONSE,
BLOCK_BROADCAST_COMPACT,
BLOCK_NOTIFY,
// Donation messages are signed by author's private keys to prevent their abuse/misuse.
AUX_JOB_DONATION,
@@ -121,7 +120,7 @@ public:
[[nodiscard]] bool on_listen_port(const uint8_t* buf);
[[nodiscard]] bool on_block_request(const uint8_t* buf);
[[nodiscard]] bool on_block_response(const uint8_t* buf, uint32_t size, uint64_t expected_id);
[[nodiscard]] bool on_block_broadcast(const uint8_t* buf, uint32_t size, bool compact);
[[nodiscard]] bool on_block_broadcast(const uint8_t* buf, uint32_t size);
[[nodiscard]] bool on_peer_list_request(const uint8_t* buf);
void on_peer_list_response(const uint8_t* buf);
void on_block_notify(const uint8_t* buf);
@@ -182,18 +181,16 @@ public:
struct Broadcast
{
Broadcast(const PoolBlock& block, const PoolBlock* parent);
Broadcast(const PoolBlock& block);
hash id;
uint64_t received_timestamp;
std::vector<uint8_t> blob;
std::vector<uint8_t> pruned_blob;
std::vector<uint8_t> compact_blob;
std::vector<hash> ancestor_hashes;
};
void broadcast(const PoolBlock& block, const PoolBlock* parent);
void broadcast(const PoolBlock& block);
[[nodiscard]] uint64_t get_random64();
[[nodiscard]] uint64_t get_peerId(bool is_tor) const { return is_tor ? m_peerId_TOR : m_peerId; }
@@ -209,7 +206,7 @@ public:
void set_max_outgoing_peers(uint32_t n) { m_maxOutgoingPeers = std::min(std::max(n, 10U), 450U); }
void set_max_incoming_peers(uint32_t n) { m_maxIncomingPeers = std::min(std::max(n, 10U), 450U); }
[[nodiscard]] int deserialize_block(const uint8_t* buf, uint32_t size, bool compact, uint64_t received_timestamp);
[[nodiscard]] int deserialize_block(const uint8_t* buf, uint32_t size, uint64_t received_timestamp);
[[nodiscard]] const PoolBlock* get_block() const { return m_block; }
[[nodiscard]] const PoolBlock* find_block(const hash& id) const;

View File

@@ -198,7 +198,7 @@ struct PoolBlock
std::vector<uint8_t> serialize_mainchain_data(size_t* header_size = nullptr, size_t* miner_tx_size = nullptr, int* outputs_offset = nullptr, int* outputs_blob_size = nullptr, const uint32_t* nonce = nullptr, const uint32_t* extra_nonce = nullptr, bool include_tx_hashes = true) const;
std::vector<uint8_t> serialize_sidechain_data() const;
[[nodiscard]] int deserialize(const uint8_t* data, size_t size, const SideChain& sidechain, uv_loop_t* loop, bool compact);
[[nodiscard]] int deserialize(const uint8_t* data, size_t size, const SideChain& sidechain, uv_loop_t* loop);
void reset_offchain_data();
bool get_pow_hash(RandomX_Hasher_Base* hasher, uint64_t height, const hash& seed_hash, hash& pow_hash, bool force_light_mode = false);

View File

@@ -23,7 +23,7 @@ namespace p2pool {
// Since data here can come from external and possibly malicious sources, check everything
// Only the syntax (i.e. the serialized block binary format) and the keccak hash are checked here
// Semantics must also be checked elsewhere before accepting the block (PoW, reward split between miners, difficulty calculation and so on)
int PoolBlock::deserialize(const uint8_t* data, size_t size, const SideChain& sidechain, uv_loop_t* loop, bool compact)
int PoolBlock::deserialize(const uint8_t* data, size_t size, const SideChain& sidechain, uv_loop_t* loop)
{
try {
// Sanity check
@@ -353,44 +353,15 @@ skip_protocol_tx:
const int transactions_offset = static_cast<int>(data - data_begin);
std::vector<uint64_t> parent_indices;
std::vector<hash> transactions;
if (compact) {
if (static_cast<uint64_t>(data_end - data) < num_transactions) return __LINE__;
// limit reserved memory size because we can't check "num_transactions" properly here
const uint64_t k = std::min<uint64_t>(num_transactions + 1, 256);
transactions.reserve(k);
parent_indices.reserve(k);
transactions.resize(1);
parent_indices.resize(1);
for (uint64_t i = 0; i < num_transactions; ++i) {
uint64_t parent_index;
READ_VARINT(parent_index);
hash id;
if (parent_index == 0) {
READ_BUF(id.h, HASH_SIZE);
}
transactions.emplace_back(id);
parent_indices.emplace_back(parent_index);
}
}
else {
if (num_transactions > std::numeric_limits<uint64_t>::max() / HASH_SIZE) return __LINE__;
if (static_cast<uint64_t>(data_end - data) < num_transactions * HASH_SIZE) return __LINE__;
transactions.reserve(num_transactions + 1);
transactions.resize(1);
for (uint64_t i = 0; i < num_transactions; ++i) {
hash id;
READ_BUF(id.h, HASH_SIZE);
transactions.emplace_back(id);
}
if (num_transactions > std::numeric_limits<uint64_t>::max() / HASH_SIZE) return __LINE__;
if (static_cast<uint64_t>(data_end - data) < num_transactions * HASH_SIZE) return __LINE__;
transactions.reserve(num_transactions + 1);
transactions.resize(1);
for (uint64_t i = 0; i < num_transactions; ++i) {
hash id;
READ_BUF(id.h, HASH_SIZE);
transactions.emplace_back(id);
}
const int transactions_actual_blob_size = static_cast<int>(data - data_begin) - transactions_offset;
@@ -438,32 +409,8 @@ skip_protocol_tx:
m_transactions.reserve(transactions.size() + 1);
m_transactions.emplace_back(hash{});
if (compact) {
const PoolBlock* parent = sidechain.find_block(m_parent);
if (!parent) {
return __LINE__;
}
const uint64_t n = transactions.size();
for (uint64_t i = 1; i < n; ++i) {
const uint64_t parent_index = parent_indices[i];
if (parent_index) {
if (parent_index >= parent->m_transactions.size()) {
return __LINE__;
}
transactions[i] = parent->m_transactions[parent_index];
m_transactions.emplace_back(parent->m_transactions[parent_index]);
}
else {
m_transactions.emplace_back(transactions[i]);
}
}
}
else {
for (size_t i = 1; i < transactions.size(); ++i) {
m_transactions.emplace_back(transactions[i]);
}
for (size_t i = 1; i < transactions.size(); ++i) {
m_transactions.emplace_back(transactions[i]);
}
m_transactions.shrink_to_fit();

View File

@@ -1603,7 +1603,7 @@ void SideChain::verify_loop(PoolBlock* block)
if (m_pool && (block->m_minerWallet == m_pool->params().m_miningWallet)) {
LOGINFO(0, log::Green() << "SHARE ADDED: height = " << block->m_sidechainHeight << ", id = " << block->m_sidechainId << ", mainchain height = " << block->m_txinGenHeight);
}
server->broadcast(*block, get_parent(block));
server->broadcast(*block);
}
}
@@ -2070,7 +2070,7 @@ void SideChain::update_chain_tip(PoolBlock* block)
if (p2pServer() && block->m_wantBroadcast && !block->m_broadcasted) {
block->m_broadcasted = true;
p2pServer()->broadcast(*block, get_parent(block));
p2pServer()->broadcast(*block);
}
}