This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 250d793a60 [common] Harden blob metadata deserialization (#8443)
250d793a60 is described below
commit 250d793a60e86979f7982044903bf47ef5f08659
Author: QuakeWang <[email protected]>
AuthorDate: Fri Jul 3 20:13:53 2026 +0800
[common] Harden blob metadata deserialization (#8443)
`BlobDescriptor` and `BlobViewStruct` deserialization read fields
directly from the input buffer. Malformed descriptor/view bytes could
therefore fail with low-level exceptions such as
`BufferUnderflowException` or `NegativeArraySizeException`, and length
fields were trusted before checking that the remaining payload was
complete.
This change adds narrow structural validation for malformed payloads:
- minimum payload length
- URI / identifier length fields
- complete offset/length and fieldId/rowId fields
The serialization format and normal deserialization behavior are
unchanged.
---
.../org/apache/paimon/data/BlobDescriptor.java | 26 ++++++++++++++
.../org/apache/paimon/data/BlobViewStruct.java | 29 +++++++++++++++-
.../org/apache/paimon/data/BlobDescriptorTest.java | 40 ++++++++++++++++++++++
.../org/apache/paimon/data/BlobViewStructTest.java | 36 +++++++++++++++++++
4 files changed, 130 insertions(+), 1 deletion(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/BlobDescriptor.java
b/paimon-common/src/main/java/org/apache/paimon/data/BlobDescriptor.java
index 671ac822a2..241eca120a 100644
--- a/paimon-common/src/main/java/org/apache/paimon/data/BlobDescriptor.java
+++ b/paimon-common/src/main/java/org/apache/paimon/data/BlobDescriptor.java
@@ -129,6 +129,10 @@ public class BlobDescriptor implements Serializable {
}
public static BlobDescriptor deserialize(byte[] bytes) {
+ if (bytes == null || bytes.length < Byte.BYTES) {
+ throw invalidPayload("too short");
+ }
+
ByteBuffer buffer = ByteBuffer.wrap(bytes);
buffer.order(ByteOrder.LITTLE_ENDIAN);
@@ -143,6 +147,7 @@ public class BlobDescriptor implements Serializable {
}
if (version > 1) {
+ checkRemaining(buffer, Long.BYTES, "too short");
long magic = buffer.getLong();
if (MAGIC != magic) {
throw new IllegalArgumentException(
@@ -153,7 +158,18 @@ public class BlobDescriptor implements Serializable {
}
}
+ checkRemaining(buffer, Integer.BYTES, "too short");
int uriLength = buffer.getInt();
+ if (uriLength < 0) {
+ throw invalidPayload("negative URI length: " + uriLength);
+ }
+ if (uriLength > buffer.remaining()) {
+ throw invalidPayload("URI length exceeds data size");
+ }
+ if (buffer.remaining() - uriLength < Long.BYTES + Long.BYTES) {
+ throw invalidPayload("missing offset/length");
+ }
+
byte[] uriBytes = new byte[uriLength];
buffer.get(uriBytes);
String uri = new String(uriBytes, StandardCharsets.UTF_8);
@@ -163,6 +179,16 @@ public class BlobDescriptor implements Serializable {
return new BlobDescriptor(version, uri, offset, length);
}
+ private static void checkRemaining(ByteBuffer buffer, int length, String
message) {
+ if (buffer.remaining() < length) {
+ throw invalidPayload(message);
+ }
+ }
+
+ private static IllegalArgumentException invalidPayload(String message) {
+ return new IllegalArgumentException("Invalid BlobDescriptor data: " +
message);
+ }
+
public static boolean isBlobDescriptor(byte[] bytes) {
if (bytes.length < 9) {
return false;
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/BlobViewStruct.java
b/paimon-common/src/main/java/org/apache/paimon/data/BlobViewStruct.java
index b5a98468a6..d53d49e446 100644
--- a/paimon-common/src/main/java/org/apache/paimon/data/BlobViewStruct.java
+++ b/paimon-common/src/main/java/org/apache/paimon/data/BlobViewStruct.java
@@ -84,6 +84,10 @@ public class BlobViewStruct implements Serializable {
}
public static BlobViewStruct deserialize(byte[] bytes) {
+ if (bytes == null || bytes.length < Byte.BYTES) {
+ throw invalidPayload("too short");
+ }
+
ByteBuffer buffer =
ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
byte version = buffer.get();
@@ -96,6 +100,7 @@ public class BlobViewStruct implements Serializable {
+ ".");
}
+ checkRemaining(buffer, Long.BYTES, "too short");
long magic = buffer.getLong();
if (magic != MAGIC) {
throw new IllegalArgumentException(
@@ -105,7 +110,19 @@ public class BlobViewStruct implements Serializable {
+ magic);
}
- byte[] identifierBytes = new byte[buffer.getInt()];
+ checkRemaining(buffer, Integer.BYTES, "too short");
+ int identifierLength = buffer.getInt();
+ if (identifierLength < 0) {
+ throw invalidPayload("negative identifier length: " +
identifierLength);
+ }
+ if (identifierLength > buffer.remaining()) {
+ throw invalidPayload("identifier length exceeds data size");
+ }
+ if (buffer.remaining() - identifierLength < Integer.BYTES +
Long.BYTES) {
+ throw invalidPayload("missing fieldId/rowId");
+ }
+
+ byte[] identifierBytes = new byte[identifierLength];
buffer.get(identifierBytes);
int fieldId = buffer.getInt();
@@ -114,6 +131,16 @@ public class BlobViewStruct implements Serializable {
Identifier.fromString(new String(identifierBytes, UTF_8)),
fieldId, rowId);
}
+ private static void checkRemaining(ByteBuffer buffer, int length, String
message) {
+ if (buffer.remaining() < length) {
+ throw invalidPayload(message);
+ }
+ }
+
+ private static IllegalArgumentException invalidPayload(String message) {
+ return new IllegalArgumentException("Invalid BlobViewStruct data: " +
message);
+ }
+
public static boolean isBlobViewStruct(byte[] bytes) {
if (bytes == null || bytes.length < 9) {
return false;
diff --git
a/paimon-common/src/test/java/org/apache/paimon/data/BlobDescriptorTest.java
b/paimon-common/src/test/java/org/apache/paimon/data/BlobDescriptorTest.java
index b25d603bf8..8120ce7620 100644
--- a/paimon-common/src/test/java/org/apache/paimon/data/BlobDescriptorTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/data/BlobDescriptorTest.java
@@ -23,6 +23,9 @@ import org.apache.paimon.utils.IOUtils;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Constructor;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Arrays;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -132,6 +135,32 @@ public class BlobDescriptorTest {
.isEqualTo(new BlobDescriptor("/test/path", 100L, 200L));
}
+ @Test
+ public void testRejectMalformedPayloads() {
+ byte[] serialized = new BlobDescriptor("/test/path", 1, 1).serialize();
+
+ byte[] headerOnly = Arrays.copyOf(serialized, Byte.BYTES + Long.BYTES);
+ assertThat(BlobDescriptor.isBlobDescriptor(headerOnly)).isTrue();
+ assertInvalidPayload(headerOnly, "too short");
+
+ byte[] negativeUriLength = new BlobDescriptor("", 1, 1).serialize();
+ putInt(negativeUriLength, Byte.BYTES + Long.BYTES, -1);
+ assertInvalidPayload(negativeUriLength, "negative URI length");
+
+ byte[] oversizedUriLength = new BlobDescriptor("", 1, 1).serialize();
+ putInt(oversizedUriLength, Byte.BYTES + Long.BYTES, 100);
+ assertInvalidPayload(oversizedUriLength, "URI length exceeds data
size");
+
+ byte[] missingOffsetLength = Arrays.copyOf(serialized,
serialized.length - Long.BYTES);
+ assertInvalidPayload(missingOffsetLength, "missing offset/length");
+
+ ByteBuffer v1OversizedUriLength =
+ ByteBuffer.allocate(Byte.BYTES +
Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN);
+ v1OversizedUriLength.put((byte) 1);
+ v1OversizedUriLength.putInt(16);
+ assertInvalidPayload(v1OversizedUriLength.array(), "URI length exceeds
data size");
+ }
+
private BlobDescriptor createDescriptorWithVersion(
byte version, String uri, long offset, long length) throws
Exception {
Constructor<BlobDescriptor> constructor =
@@ -140,4 +169,15 @@ public class BlobDescriptorTest {
constructor.setAccessible(true);
return constructor.newInstance(version, uri, offset, length);
}
+
+ private static void putInt(byte[] bytes, int offset, int value) {
+ ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).putInt(offset,
value);
+ }
+
+ private static void assertInvalidPayload(byte[] bytes, String message) {
+ assertThatThrownBy(() -> BlobDescriptor.deserialize(bytes))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid BlobDescriptor data:")
+ .hasMessageContaining(message);
+ }
}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/data/BlobViewStructTest.java
b/paimon-common/src/test/java/org/apache/paimon/data/BlobViewStructTest.java
index 0d40c1fe30..de68d482c2 100644
--- a/paimon-common/src/test/java/org/apache/paimon/data/BlobViewStructTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/data/BlobViewStructTest.java
@@ -22,6 +22,10 @@ import org.apache.paimon.catalog.Identifier;
import org.junit.jupiter.api.Test;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -85,4 +89,36 @@ public class BlobViewStructTest {
assertThat(BlobViewStruct.isBlobViewStruct(new byte[] {1, 2,
3})).isFalse();
assertThat(Blob.fromBytes(bytes, null,
null)).isEqualTo(Blob.fromView(viewStruct));
}
+
+ @Test
+ public void testRejectMalformedPayloads() {
+ byte[] serialized =
+ new BlobViewStruct(Identifier.fromString("default.source"), 7,
5L).serialize();
+
+ byte[] headerOnly = Arrays.copyOf(serialized, Byte.BYTES + Long.BYTES);
+ assertThat(BlobViewStruct.isBlobViewStruct(headerOnly)).isTrue();
+ assertInvalidPayload(headerOnly, "too short");
+
+ byte[] negativeIdentifierLength = Arrays.copyOf(serialized,
serialized.length);
+ putInt(negativeIdentifierLength, Byte.BYTES + Long.BYTES, -1);
+ assertInvalidPayload(negativeIdentifierLength, "negative identifier
length");
+
+ byte[] oversizedIdentifierLength = Arrays.copyOf(serialized,
serialized.length);
+ putInt(oversizedIdentifierLength, Byte.BYTES + Long.BYTES, 100);
+ assertInvalidPayload(oversizedIdentifierLength, "identifier length
exceeds data size");
+
+ byte[] missingFieldIdRowId = Arrays.copyOf(serialized,
serialized.length - Long.BYTES);
+ assertInvalidPayload(missingFieldIdRowId, "missing fieldId/rowId");
+ }
+
+ private static void putInt(byte[] bytes, int offset, int value) {
+ ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).putInt(offset,
value);
+ }
+
+ private static void assertInvalidPayload(byte[] bytes, String message) {
+ assertThatThrownBy(() -> BlobViewStruct.deserialize(bytes))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid BlobViewStruct data:")
+ .hasMessageContaining(message);
+ }
}
