This is an automated email from the ASF dual-hosted git repository.
erose pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2db082e0644 HDDS-13237. Container data checksum should contain block
IDs. (#8773)
2db082e0644 is described below
commit 2db082e0644f4dcd96fc9c4b5dab52395f1f13f1
Author: Aswin Shakil Balasubramanian <[email protected]>
AuthorDate: Tue Jul 22 15:36:23 2025 -0700
HDDS-13237. Container data checksum should contain block IDs. (#8773)
---
.../checksum/ContainerMerkleTreeWriter.java | 5 +-
.../checksum/TestContainerMerkleTreeWriter.java | 105 ++++++++++++++++++++-
2 files changed, 105 insertions(+), 5 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeWriter.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeWriter.java
index 3fc1f4fb951..69921885915 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeWriter.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeWriter.java
@@ -172,7 +172,10 @@ public void addChunks(ChunkMerkleTreeWriter... chunks) {
public ContainerProtos.BlockMerkleTree toProto() {
ContainerProtos.BlockMerkleTree.Builder blockTreeBuilder =
ContainerProtos.BlockMerkleTree.newBuilder();
ChecksumByteBuffer checksumImpl = CHECKSUM_BUFFER_SUPPLIER.get();
- ByteBuffer blockChecksumBuffer = ByteBuffer.allocate(Long.BYTES *
offset2Chunk.size());
+ // Allocate space for block ID + all chunk checksums
+ ByteBuffer blockChecksumBuffer = ByteBuffer.allocate(Long.BYTES * (1 +
offset2Chunk.size()));
+ // Hash the block ID into the beginning of the block checksum calculation
+ blockChecksumBuffer.putLong(blockID);
for (ChunkMerkleTreeWriter chunkTree: offset2Chunk.values()) {
// Ordering of checksums within a chunk is assumed to be in the order
they are written.
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerMerkleTreeWriter.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerMerkleTreeWriter.java
index 2eaad40dc5f..a3069997387 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerMerkleTreeWriter.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerMerkleTreeWriter.java
@@ -113,6 +113,103 @@ public void testBuildTreeWithMissingChunks() {
assertTreesSortedAndMatch(expectedTree, actualTreeProto);
}
+ @Test
+ public void testBlockIdIncludedInChecksum() {
+ // Create a set of chunks to be used in different blocks with identical
content.
+ ContainerProtos.ChunkInfo chunk1 = buildChunk(config, 0,
ByteBuffer.wrap(new byte[]{1, 2, 3}));
+ ContainerProtos.ChunkInfo chunk2 = buildChunk(config, 1,
ByteBuffer.wrap(new byte[]{4, 5, 6}));
+
+ // Create two blocks with different IDs but identical chunk data
+ final long blockID1 = 1;
+ final long blockID2 = 2;
+
+ ContainerMerkleTreeWriter tree1 = new ContainerMerkleTreeWriter();
+ tree1.addChunks(blockID1, true, chunk1, chunk2);
+
+ ContainerMerkleTreeWriter tree2 = new ContainerMerkleTreeWriter();
+ tree2.addChunks(blockID2, true, chunk1, chunk2);
+
+ ContainerProtos.ContainerMerkleTree tree1Proto = tree1.toProto();
+ ContainerProtos.ContainerMerkleTree tree2Proto = tree2.toProto();
+
+ // Even though the chunks are identical, the block checksums should be
different
+ // because the block IDs are different
+ ContainerProtos.BlockMerkleTree block1 = tree1Proto.getBlockMerkleTree(0);
+ ContainerProtos.BlockMerkleTree block2 = tree2Proto.getBlockMerkleTree(0);
+
+ assertEquals(blockID1, block1.getBlockID());
+ assertEquals(blockID2, block2.getBlockID());
+ assertNotEquals(block1.getDataChecksum(), block2.getDataChecksum(),
+ "Blocks with identical chunks but different IDs should have different
checksums");
+
+ // Consequently, the container checksums should also be different
+ assertNotEquals(tree1Proto.getDataChecksum(), tree2Proto.getDataChecksum(),
+ "Containers with blocks having identical chunks but different IDs
should have different checksums");
+ }
+
+ @Test
+ public void testIdenticalBlocksHaveSameChecksum() {
+ // Create a set of chunks to be used in different blocks with identical
content.
+ ContainerProtos.ChunkInfo chunk1 = buildChunk(config, 0,
ByteBuffer.wrap(new byte[]{1, 2, 3}));
+ ContainerProtos.ChunkInfo chunk2 = buildChunk(config, 1,
ByteBuffer.wrap(new byte[]{4, 5, 6}));
+
+ // Create two blocks with the same ID and identical chunk data
+ final long blockID = 1;
+
+ ContainerMerkleTreeWriter tree1 = new ContainerMerkleTreeWriter();
+ tree1.addChunks(blockID, true, chunk1, chunk2);
+
+ ContainerMerkleTreeWriter tree2 = new ContainerMerkleTreeWriter();
+ tree2.addChunks(blockID, true, chunk1, chunk2);
+
+ ContainerProtos.ContainerMerkleTree tree1Proto = tree1.toProto();
+ ContainerProtos.ContainerMerkleTree tree2Proto = tree2.toProto();
+
+ // Blocks with same ID and identical chunks should have same checksums
+ ContainerProtos.BlockMerkleTree block1 = tree1Proto.getBlockMerkleTree(0);
+ ContainerProtos.BlockMerkleTree block2 = tree2Proto.getBlockMerkleTree(0);
+
+ assertEquals(blockID, block1.getBlockID());
+ assertEquals(blockID, block2.getBlockID());
+ assertEquals(block1.getDataChecksum(), block2.getDataChecksum(),
+ "Blocks with same ID and identical chunks should have same checksums");
+
+ // Container checksums should also be the same
+ assertEquals(tree1Proto.getDataChecksum(), tree2Proto.getDataChecksum(),
+ "Containers with identical blocks should have same checksums");
+ }
+
+ @Test
+ public void
testContainerReplicasWithDifferentMissingBlocksHaveDifferentChecksums() {
+ // Create identical chunk data that will be used across all blocks
+ ContainerProtos.ChunkInfo chunk1 = buildChunk(config, 0,
ByteBuffer.wrap(new byte[]{1, 2, 3}));
+ ContainerProtos.ChunkInfo chunk2 = buildChunk(config, 1,
ByteBuffer.wrap(new byte[]{4, 5, 6}));
+
+ // Scenario: Container has 5 identical blocks, but different replicas are
missing different blocks
+ // Replica 1 is missing block 1 (has blocks 2,3,4,5)
+ ContainerMerkleTreeWriter replica1 = new ContainerMerkleTreeWriter();
+ replica1.addChunks(2, true, chunk1, chunk2);
+ replica1.addChunks(3, true, chunk1, chunk2);
+ replica1.addChunks(4, true, chunk1, chunk2);
+ replica1.addChunks(5, true, chunk1, chunk2);
+
+ // Replica 2 is missing block 5 (has blocks 1,2,3,4)
+ ContainerMerkleTreeWriter replica2 = new ContainerMerkleTreeWriter();
+ replica2.addChunks(1, true, chunk1, chunk2);
+ replica2.addChunks(2, true, chunk1, chunk2);
+ replica2.addChunks(3, true, chunk1, chunk2);
+ replica2.addChunks(4, true, chunk1, chunk2);
+
+ ContainerProtos.ContainerMerkleTree replica1Proto = replica1.toProto();
+ ContainerProtos.ContainerMerkleTree replica2Proto = replica2.toProto();
+ assertNotEquals(replica1Proto.getDataChecksum(),
replica2Proto.getDataChecksum(),
+ "Container replicas with identical blocks but different missing blocks
should have different checksums");
+
+ // Verify both replicas have the same number of blocks
+ assertEquals(4, replica1Proto.getBlockMerkleTreeCount());
+ assertEquals(4, replica2Proto.getBlockMerkleTreeCount());
+ }
+
@Test
public void testBuildTreeWithEmptyBlock() {
final long blockID = 1;
@@ -274,12 +371,12 @@ private ContainerProtos.ContainerMerkleTree
buildExpectedContainerTree(List<Cont
private ContainerProtos.BlockMerkleTree buildExpectedBlockTree(long blockID,
List<ContainerProtos.ChunkMerkleTree> chunks) {
+ List<Long> itemsToChecksum =
chunks.stream().map(ContainerProtos.ChunkMerkleTree::getDataChecksum)
+ .collect(Collectors.toList());
+ itemsToChecksum.add(0, blockID);
return ContainerProtos.BlockMerkleTree.newBuilder()
.setBlockID(blockID)
- .setDataChecksum(computeExpectedChecksum(
- chunks.stream()
- .map(ContainerProtos.ChunkMerkleTree::getDataChecksum)
- .collect(Collectors.toList())))
+ .setDataChecksum(computeExpectedChecksum(itemsToChecksum))
.addAllChunkMerkleTree(chunks)
.build();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
