This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 14db13496f1 [fix](compression) Preserve LZ4HC in tablet metadata
(#69574)
14db13496f1 is described below
commit 14db13496f1ef5696769d3a75f89ba1ab43491c0
Author: Gavin Chou <[email protected]>
AuthorDate: Mon Oct 26 11:19:19 2026 +0800
[fix](compression) Preserve LZ4HC in tablet metadata (#69574)
### What problem does this PR solve?
Issue Number: DORIS-28505
Problem Summary:
`compression=LZ4HC` is accepted by the FE, but the tablet metadata
conversion omitted the LZ4HC case. It therefore fell through to the LZ4F
default and silently changed the requested codec.
This PR adds explicit LZ4HC mappings in both the BE tablet metadata
conversion and the cloud FE tablet metadata builder, with regression
tests for both paths.
### Release note
Fix `compression=LZ4HC` being silently stored as LZ4F in tablet
metadata.
---
be/src/storage/tablet/tablet_meta.cpp | 3 +++
be/test/storage/tablet/tablet_meta_test.cpp | 7 +++++++
.../cloud/datasource/CloudInternalCatalog.java | 3 +++
.../cloud/datasource/CloudInternalCatalogTest.java | 24 ++++++++++++++++++----
4 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/be/src/storage/tablet/tablet_meta.cpp
b/be/src/storage/tablet/tablet_meta.cpp
index c4f27d9916a..8e912bc9b0b 100644
--- a/be/src/storage/tablet/tablet_meta.cpp
+++ b/be/src/storage/tablet/tablet_meta.cpp
@@ -446,6 +446,9 @@ void TabletMeta::init_schema_from_thrift(const
TTabletSchema& tablet_schema,
case TCompressionType::LZ4F:
tablet_schema_pb->set_compression_type(segment_v2::LZ4F);
break;
+ case TCompressionType::LZ4HC:
+ tablet_schema_pb->set_compression_type(segment_v2::LZ4HC);
+ break;
case TCompressionType::ZLIB:
tablet_schema_pb->set_compression_type(segment_v2::ZLIB);
break;
diff --git a/be/test/storage/tablet/tablet_meta_test.cpp
b/be/test/storage/tablet/tablet_meta_test.cpp
index d6b2d7dcf63..33990a7097a 100644
--- a/be/test/storage/tablet/tablet_meta_test.cpp
+++ b/be/test/storage/tablet/tablet_meta_test.cpp
@@ -83,6 +83,13 @@ TEST(TabletMetaTest, SaveAsBufferAndParse) {
}
}
+TEST(TabletMetaTest, PreserveLz4hcCompressionType) {
+ TabletMeta tablet_meta(1, 2, 3, 3, 4, 5, TTabletSchema(), 6, {{7, 8}},
UniqueId(9, 10),
+ TTabletType::TABLET_TYPE_DISK,
TCompressionType::LZ4HC);
+
+ EXPECT_EQ(segment_v2::LZ4HC,
tablet_meta.tablet_schema()->compression_type());
+}
+
TEST(TabletMetaTest, ChangelogTtlMetadataRoundTrip) {
TabletMeta tablet_meta(1, 2, 3, 3, 4, 5, TTabletSchema(), 6, {{7, 8}},
UniqueId(9, 10),
TTabletType::TABLET_TYPE_DISK,
TCompressionType::LZ4F);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index b48fa3cb59c..a61c03538d8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -556,6 +556,9 @@ public class CloudInternalCatalog extends InternalCatalog {
case LZ4F:
schemaBuilder.setCompressionType(SegmentV2.CompressionTypePB.LZ4F);
break;
+ case LZ4HC:
+
schemaBuilder.setCompressionType(SegmentV2.CompressionTypePB.LZ4HC);
+ break;
case ZLIB:
schemaBuilder.setCompressionType(SegmentV2.CompressionTypePB.ZLIB);
break;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java
index cabc592f4d3..f934089fd1d 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java
@@ -44,6 +44,7 @@ import org.apache.doris.thrift.TSortType;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TTabletType;
+import doris.segment_v2.SegmentV2;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
@@ -107,7 +108,7 @@ public class CloudInternalCatalogTest {
boolean original =
Config.enable_partition_inverted_index_storage_format_rollout;
try {
Config.enable_partition_inverted_index_storage_format_rollout =
false;
- OlapFile.TabletMetaCloudPB disabledTabletMeta =
createTabletMeta(tablet);
+ OlapFile.TabletMetaCloudPB disabledTabletMeta =
createTabletMeta(tablet, TCompressionType.LZ4F);
Assertions.assertEquals(17, disabledTabletMeta.getSchemaVersion());
Assertions.assertEquals(17,
disabledTabletMeta.getSchema().getSchemaVersion());
Assertions.assertEquals(OlapFile.InvertedIndexStorageFormatPB.SNII,
@@ -117,7 +118,7 @@ public class CloudInternalCatalogTest {
Assertions.assertFalse(disabledTabletMeta.getRsMetas(0).hasInvertedIndexStorageFormat());
Config.enable_partition_inverted_index_storage_format_rollout =
true;
- OlapFile.TabletMetaCloudPB enabledTabletMeta =
createTabletMeta(tablet);
+ OlapFile.TabletMetaCloudPB enabledTabletMeta =
createTabletMeta(tablet, TCompressionType.LZ4F);
Assertions.assertTrue(enabledTabletMeta.hasInvertedIndexStorageFormat());
Assertions.assertEquals(OlapFile.InvertedIndexStorageFormatPB.SNII,
enabledTabletMeta.getInvertedIndexStorageFormat());
@@ -129,6 +130,20 @@ public class CloudInternalCatalogTest {
}
}
+ @Test
+ public void testCreateTabletMetaPreservesLz4hcCompressionType() throws
Exception {
+ Tablet tablet = Mockito.mock(Tablet.class);
+ Replica replica = Mockito.mock(Replica.class);
+ Mockito.when(tablet.getId()).thenReturn(100L);
+
Mockito.when(tablet.getReplicas()).thenReturn(Collections.singletonList(replica));
+ Mockito.when(replica.getId()).thenReturn(200L);
+
+ OlapFile.TabletMetaCloudPB tabletMeta = createTabletMeta(tablet,
TCompressionType.LZ4HC);
+
+ Assertions.assertEquals(SegmentV2.CompressionTypePB.LZ4HC,
+ tabletMeta.getSchema().getCompressionType());
+ }
+
@Test
public void testCreateTableConflictDropsOnlyLosingIndexes() throws
Exception {
testCreateTableConflictCleanup(Cloud.MetaServiceCode.OK);
@@ -298,11 +313,12 @@ public class CloudInternalCatalogTest {
Mockito.when(systemInfoService.getBackend(backendId)).thenReturn(backend);
}
- private OlapFile.TabletMetaCloudPB createTabletMeta(Tablet tablet) throws
Exception {
+ private OlapFile.TabletMetaCloudPB createTabletMeta(Tablet tablet,
TCompressionType compressionType)
+ throws Exception {
return new CloudInternalCatalog().createTabletMetaBuilder(
1L, 2L, 3L, tablet, TTabletType.TABLET_TYPE_DISK, 23,
KeysType.DUP_KEYS, (short) 1,
Collections.emptySet(), 0.05, Collections.emptyList(),
Collections.emptyList(),
- new DataSortInfo(TSortType.LEXICAL, 0), TCompressionType.LZ4F,
TStorageFormat.DEFAULT,
+ new DataSortInfo(TSortType.LEXICAL, 0), compressionType,
TStorageFormat.DEFAULT,
"", false, false,
"table", 0L, false, false, 17, null, "size_based",
0L, 0L, 0L, 0L, 0L, false, Collections.emptyList(),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
