Copilot commented on code in PR #17028:
URL: https://github.com/apache/pinot/pull/17028#discussion_r2441388493
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java:
##########
@@ -1723,6 +1725,214 @@ public void testStarTreeCreationWithDictionaryChanges()
}
}
+ @Test
+ public void testStarTreeCreationWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 1: Adding a new startree index with invalid functionColumnPair
(SUM__*)
+ // This should fail during config creation but segment build should be fine
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__*"),
null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to new startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be created
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that no startree index was created due to invalid config
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ assertNull(segmentMetadata.getStarTreeV2MetadataList());
+ }
+
+ @Test
+ public void testStarTreeUpdateWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a valid startree index
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ // Test 2: Try to update existing startree index with invalid
functionColumnPair (SUM__*)
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*", "SUM__*"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to changed startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be updated
Review Comment:
Corrected spelling of 'startree' to 'star-tree' in comment for consistency
with other comments in the file.
```suggestion
// Process should complete without throwing exception, but star-tree
should not be updated
```
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java:
##########
@@ -1723,6 +1725,214 @@ public void testStarTreeCreationWithDictionaryChanges()
}
}
+ @Test
+ public void testStarTreeCreationWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 1: Adding a new startree index with invalid functionColumnPair
(SUM__*)
+ // This should fail during config creation but segment build should be fine
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__*"),
null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to new startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be created
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that no startree index was created due to invalid config
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ assertNull(segmentMetadata.getStarTreeV2MetadataList());
+ }
+
+ @Test
+ public void testStarTreeUpdateWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a valid startree index
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ // Test 2: Try to update existing startree index with invalid
functionColumnPair (SUM__*)
Review Comment:
Corrected spelling of 'startree' to 'star-tree' in comment for consistency
with other comments in the file.
```suggestion
// Test 2: Try to update existing star-tree index with invalid
functionColumnPair (SUM__*)
```
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java:
##########
@@ -1723,6 +1725,214 @@ public void testStarTreeCreationWithDictionaryChanges()
}
}
+ @Test
+ public void testStarTreeCreationWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 1: Adding a new startree index with invalid functionColumnPair
(SUM__*)
Review Comment:
Corrected spelling of 'startree' to 'star-tree' in comment for consistency
with other comments in the file.
```suggestion
// Test 1: Adding a new star-tree index with invalid functionColumnPair
(SUM__*)
```
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java:
##########
@@ -1723,6 +1725,214 @@ public void testStarTreeCreationWithDictionaryChanges()
}
}
+ @Test
+ public void testStarTreeCreationWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 1: Adding a new startree index with invalid functionColumnPair
(SUM__*)
+ // This should fail during config creation but segment build should be fine
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__*"),
null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to new startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be created
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that no startree index was created due to invalid config
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ assertNull(segmentMetadata.getStarTreeV2MetadataList());
+ }
+
+ @Test
+ public void testStarTreeUpdateWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a valid startree index
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ // Test 2: Try to update existing startree index with invalid
functionColumnPair (SUM__*)
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*", "SUM__*"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to changed startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be updated
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that the original startree index still exists and hasn't changed
Review Comment:
Corrected spelling of 'startree' to 'star-tree' in comment for consistency
with other comments in the file.
```suggestion
// Verify that the original star-tree index still exists and hasn't
changed
```
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java:
##########
@@ -1723,6 +1725,214 @@ public void testStarTreeCreationWithDictionaryChanges()
}
}
+ @Test
+ public void testStarTreeCreationWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 1: Adding a new startree index with invalid functionColumnPair
(SUM__*)
+ // This should fail during config creation but segment build should be fine
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__*"),
null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to new startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be created
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that no startree index was created due to invalid config
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ assertNull(segmentMetadata.getStarTreeV2MetadataList());
+ }
+
+ @Test
+ public void testStarTreeUpdateWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a valid startree index
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ // Test 2: Try to update existing startree index with invalid
functionColumnPair (SUM__*)
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*", "SUM__*"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to changed startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be updated
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that the original startree index still exists and hasn't changed
+ segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> currentStarTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(currentStarTreeMetadataList);
+ assertFalse(currentStarTreeMetadataList.isEmpty());
+ assertEquals(currentStarTreeMetadataList.size(), 1);
+
+ // Verify the metadata hasn't changed (original startree should still be
there)
+ assertEquals(currentStarTreeMetadataList.get(0).getDimensionsSplitOrder(),
+ starTreeMetadataList.get(0).getDimensionsSplitOrder());
+ assertEquals(currentStarTreeMetadataList.get(0).getFunctionColumnPairs(),
+ starTreeMetadataList.get(0).getFunctionColumnPairs());
+ }
+
+ @Test
+ public void testStarTreeCreationWithValidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 3: Adding a new startree index with valid functionColumnPair
(COUNT__*)
Review Comment:
Corrected spelling of 'startree' to 'star-tree' in comment for consistency
with other comments in the file.
```suggestion
// Test 3: Adding a new star-tree index with valid functionColumnPair
(COUNT__*)
```
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java:
##########
@@ -1723,6 +1725,214 @@ public void testStarTreeCreationWithDictionaryChanges()
}
}
+ @Test
+ public void testStarTreeCreationWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 1: Adding a new startree index with invalid functionColumnPair
(SUM__*)
+ // This should fail during config creation but segment build should be fine
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__*"),
null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to new startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be created
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that no startree index was created due to invalid config
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ assertNull(segmentMetadata.getStarTreeV2MetadataList());
+ }
+
+ @Test
+ public void testStarTreeUpdateWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a valid startree index
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ // Test 2: Try to update existing startree index with invalid
functionColumnPair (SUM__*)
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*", "SUM__*"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to changed startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be updated
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that the original startree index still exists and hasn't changed
+ segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> currentStarTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(currentStarTreeMetadataList);
+ assertFalse(currentStarTreeMetadataList.isEmpty());
+ assertEquals(currentStarTreeMetadataList.size(), 1);
+
+ // Verify the metadata hasn't changed (original startree should still be
there)
+ assertEquals(currentStarTreeMetadataList.get(0).getDimensionsSplitOrder(),
+ starTreeMetadataList.get(0).getDimensionsSplitOrder());
+ assertEquals(currentStarTreeMetadataList.get(0).getFunctionColumnPairs(),
+ starTreeMetadataList.get(0).getFunctionColumnPairs());
+ }
+
+ @Test
+ public void testStarTreeCreationWithValidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 3: Adding a new startree index with valid functionColumnPair
(COUNT__*)
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created successfully
Review Comment:
Corrected spelling of 'startree' to 'star-tree' in comment for consistency
with other comments in the file.
```suggestion
// Verify that star-tree index was created successfully
```
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java:
##########
@@ -1723,6 +1725,214 @@ public void testStarTreeCreationWithDictionaryChanges()
}
}
+ @Test
+ public void testStarTreeCreationWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 1: Adding a new startree index with invalid functionColumnPair
(SUM__*)
+ // This should fail during config creation but segment build should be fine
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__*"),
null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to new startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be created
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that no startree index was created due to invalid config
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ assertNull(segmentMetadata.getStarTreeV2MetadataList());
+ }
+
+ @Test
+ public void testStarTreeUpdateWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a valid startree index
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ // Test 2: Try to update existing startree index with invalid
functionColumnPair (SUM__*)
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*", "SUM__*"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to changed startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be updated
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that the original startree index still exists and hasn't changed
+ segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> currentStarTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(currentStarTreeMetadataList);
+ assertFalse(currentStarTreeMetadataList.isEmpty());
+ assertEquals(currentStarTreeMetadataList.size(), 1);
+
+ // Verify the metadata hasn't changed (original startree should still be
there)
Review Comment:
Corrected spelling of 'startree' to 'star-tree' in comment for consistency
with other comments in the file.
```suggestion
// Verify the metadata hasn't changed (original star-tree should still
be there)
```
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java:
##########
@@ -1723,6 +1725,214 @@ public void testStarTreeCreationWithDictionaryChanges()
}
}
+ @Test
+ public void testStarTreeCreationWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 1: Adding a new startree index with invalid functionColumnPair
(SUM__*)
+ // This should fail during config creation but segment build should be fine
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__*"),
null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to new startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be created
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that no startree index was created due to invalid config
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ assertNull(segmentMetadata.getStarTreeV2MetadataList());
+ }
+
+ @Test
+ public void testStarTreeUpdateWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a valid startree index
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ // Test 2: Try to update existing startree index with invalid
functionColumnPair (SUM__*)
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*", "SUM__*"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to changed startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be updated
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that the original startree index still exists and hasn't changed
+ segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> currentStarTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(currentStarTreeMetadataList);
+ assertFalse(currentStarTreeMetadataList.isEmpty());
+ assertEquals(currentStarTreeMetadataList.size(), 1);
+
+ // Verify the metadata hasn't changed (original startree should still be
there)
+ assertEquals(currentStarTreeMetadataList.get(0).getDimensionsSplitOrder(),
+ starTreeMetadataList.get(0).getDimensionsSplitOrder());
+ assertEquals(currentStarTreeMetadataList.get(0).getFunctionColumnPairs(),
+ starTreeMetadataList.get(0).getFunctionColumnPairs());
+ }
+
+ @Test
+ public void testStarTreeCreationWithValidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 3: Adding a new startree index with valid functionColumnPair
(COUNT__*)
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created successfully
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ StarTreeV2Metadata starTreeMetadata = starTreeMetadataList.get(0);
+ assertEquals(starTreeMetadata.getDimensionsSplitOrder(),
List.of("stringCol"));
+ assertEquals(starTreeMetadata.getFunctionColumnPairs().size(), 1);
+
assertEquals(starTreeMetadata.getFunctionColumnPairs().iterator().next().toColumnName(),
"count__*");
+ }
+
+ @Test
+ public void testStarTreeUpdateWithValidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a startree index with SUM function
Review Comment:
Corrected spelling of 'startree' to 'star-tree' in comment for consistency
with other comments in the file.
```suggestion
// First, create a star-tree index with SUM function
```
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java:
##########
@@ -1723,6 +1725,214 @@ public void testStarTreeCreationWithDictionaryChanges()
}
}
+ @Test
+ public void testStarTreeCreationWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 1: Adding a new startree index with invalid functionColumnPair
(SUM__*)
+ // This should fail during config creation but segment build should be fine
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__*"),
null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to new startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be created
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that no startree index was created due to invalid config
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ assertNull(segmentMetadata.getStarTreeV2MetadataList());
+ }
+
+ @Test
+ public void testStarTreeUpdateWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a valid startree index
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ // Test 2: Try to update existing startree index with invalid
functionColumnPair (SUM__*)
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*", "SUM__*"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to changed startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be updated
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that the original startree index still exists and hasn't changed
+ segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> currentStarTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(currentStarTreeMetadataList);
+ assertFalse(currentStarTreeMetadataList.isEmpty());
+ assertEquals(currentStarTreeMetadataList.size(), 1);
+
+ // Verify the metadata hasn't changed (original startree should still be
there)
+ assertEquals(currentStarTreeMetadataList.get(0).getDimensionsSplitOrder(),
+ starTreeMetadataList.get(0).getDimensionsSplitOrder());
+ assertEquals(currentStarTreeMetadataList.get(0).getFunctionColumnPairs(),
+ starTreeMetadataList.get(0).getFunctionColumnPairs());
+ }
+
+ @Test
+ public void testStarTreeCreationWithValidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 3: Adding a new startree index with valid functionColumnPair
(COUNT__*)
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created successfully
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ StarTreeV2Metadata starTreeMetadata = starTreeMetadataList.get(0);
+ assertEquals(starTreeMetadata.getDimensionsSplitOrder(),
List.of("stringCol"));
+ assertEquals(starTreeMetadata.getFunctionColumnPairs().size(), 1);
+
assertEquals(starTreeMetadata.getFunctionColumnPairs().iterator().next().toColumnName(),
"count__*");
+ }
+
+ @Test
+ public void testStarTreeUpdateWithValidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a startree index with SUM function
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig originalStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("SUM__longCol"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(originalStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that original startree index was created
Review Comment:
Corrected spelling of 'startree' to 'star-tree' in comment for consistency
with other comments in the file.
```suggestion
// Verify that original star-tree index was created
```
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java:
##########
@@ -1723,6 +1725,214 @@ public void testStarTreeCreationWithDictionaryChanges()
}
}
+ @Test
+ public void testStarTreeCreationWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 1: Adding a new startree index with invalid functionColumnPair
(SUM__*)
+ // This should fail during config creation but segment build should be fine
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__*"),
null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to new startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be created
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that no startree index was created due to invalid config
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ assertNull(segmentMetadata.getStarTreeV2MetadataList());
+ }
+
+ @Test
+ public void testStarTreeUpdateWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a valid startree index
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ // Test 2: Try to update existing startree index with invalid
functionColumnPair (SUM__*)
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*", "SUM__*"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to changed startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be updated
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that the original startree index still exists and hasn't changed
+ segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> currentStarTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(currentStarTreeMetadataList);
+ assertFalse(currentStarTreeMetadataList.isEmpty());
+ assertEquals(currentStarTreeMetadataList.size(), 1);
+
+ // Verify the metadata hasn't changed (original startree should still be
there)
+ assertEquals(currentStarTreeMetadataList.get(0).getDimensionsSplitOrder(),
+ starTreeMetadataList.get(0).getDimensionsSplitOrder());
+ assertEquals(currentStarTreeMetadataList.get(0).getFunctionColumnPairs(),
+ starTreeMetadataList.get(0).getFunctionColumnPairs());
+ }
+
+ @Test
+ public void testStarTreeCreationWithValidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 3: Adding a new startree index with valid functionColumnPair
(COUNT__*)
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created successfully
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ StarTreeV2Metadata starTreeMetadata = starTreeMetadataList.get(0);
+ assertEquals(starTreeMetadata.getDimensionsSplitOrder(),
List.of("stringCol"));
+ assertEquals(starTreeMetadata.getFunctionColumnPairs().size(), 1);
+
assertEquals(starTreeMetadata.getFunctionColumnPairs().iterator().next().toColumnName(),
"count__*");
+ }
+
+ @Test
+ public void testStarTreeUpdateWithValidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a startree index with SUM function
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig originalStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("SUM__longCol"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(originalStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that original startree index was created
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> originalStarTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(originalStarTreeMetadataList);
+ assertFalse(originalStarTreeMetadataList.isEmpty());
+ assertEquals(originalStarTreeMetadataList.size(), 1);
+
assertEquals(originalStarTreeMetadataList.get(0).getFunctionColumnPairs().iterator().next().toColumnName(),
+ "sum__longCol");
+
+ // Test 4: Update existing startree index with valid functionColumnPair
(COUNT__*)
+ StarTreeIndexConfig updatedStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("SUM__longCol", "COUNT__*"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(updatedStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was updated successfully
Review Comment:
Corrected spelling of 'startree' to 'star-tree' in comment for consistency
with other comments in the file.
```suggestion
// Verify that star-tree index was updated successfully
```
##########
pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java:
##########
@@ -1723,6 +1725,214 @@ public void testStarTreeCreationWithDictionaryChanges()
}
}
+ @Test
+ public void testStarTreeCreationWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 1: Adding a new startree index with invalid functionColumnPair
(SUM__*)
+ // This should fail during config creation but segment build should be fine
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null, List.of("SUM__*"),
null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to new startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be created
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that no startree index was created due to invalid config
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ assertNull(segmentMetadata.getStarTreeV2MetadataList());
+ }
+
+ @Test
+ public void testStarTreeUpdateWithInvalidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a valid startree index
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ // Test 2: Try to update existing startree index with invalid
functionColumnPair (SUM__*)
+ StarTreeIndexConfig invalidStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*", "SUM__*"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(invalidStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ // Should need processing due to changed startree config
+ assertTrue(processor.needProcess());
+ // Process should complete without throwing exception, but startree
should not be updated
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that the original startree index still exists and hasn't changed
+ segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> currentStarTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(currentStarTreeMetadataList);
+ assertFalse(currentStarTreeMetadataList.isEmpty());
+ assertEquals(currentStarTreeMetadataList.size(), 1);
+
+ // Verify the metadata hasn't changed (original startree should still be
there)
+ assertEquals(currentStarTreeMetadataList.get(0).getDimensionsSplitOrder(),
+ starTreeMetadataList.get(0).getDimensionsSplitOrder());
+ assertEquals(currentStarTreeMetadataList.get(0).getFunctionColumnPairs(),
+ starTreeMetadataList.get(0).getFunctionColumnPairs());
+ }
+
+ @Test
+ public void testStarTreeCreationWithValidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // Test 3: Adding a new startree index with valid functionColumnPair
(COUNT__*)
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig validStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("COUNT__*"), null, 1000);
+ indexingConfig.setStarTreeIndexConfigs(List.of(validStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that startree index was created successfully
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> starTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(starTreeMetadataList);
+ assertFalse(starTreeMetadataList.isEmpty());
+ assertEquals(starTreeMetadataList.size(), 1);
+
+ StarTreeV2Metadata starTreeMetadata = starTreeMetadataList.get(0);
+ assertEquals(starTreeMetadata.getDimensionsSplitOrder(),
List.of("stringCol"));
+ assertEquals(starTreeMetadata.getFunctionColumnPairs().size(), 1);
+
assertEquals(starTreeMetadata.getFunctionColumnPairs().iterator().next().toColumnName(),
"count__*");
+ }
+
+ @Test
+ public void testStarTreeUpdateWithValidFunctionColumnPair()
+ throws Exception {
+ // Build the sample segment
+ String[] stringValues = {"A", "C", "B", "C", "D", "E", "E", "E"};
+ long[] longValues = {2, 1, 2, 3, 4, 5, 3, 2};
+ TableConfig tableConfig = new
TableConfigBuilder(TableType.OFFLINE).setTableName("testTable").build();
+ Schema schema = new
Schema.SchemaBuilder().addSingleValueDimension("stringCol",
FieldSpec.DataType.STRING)
+ .addMetric("longCol", DataType.LONG)
+ .build();
+
+ // Build good segment, no need for preprocess
+ buildTestSegment(tableConfig, schema, stringValues, longValues);
+
+ // First, create a startree index with SUM function
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setEnableDynamicStarTreeCreation(true);
+ StarTreeIndexConfig originalStarTreeIndexConfig =
+ new StarTreeIndexConfig(List.of("stringCol"), null,
List.of("SUM__longCol"), null, 1000);
+
indexingConfig.setStarTreeIndexConfigs(List.of(originalStarTreeIndexConfig));
+
+ try (SegmentDirectory segmentDirectory = new
SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
+ SegmentPreProcessor processor = new
SegmentPreProcessor(segmentDirectory,
+ new IndexLoadingConfig(tableConfig, schema))) {
+ assertTrue(processor.needProcess());
+ processor.process(SEGMENT_OPERATIONS_THROTTLER);
+ }
+
+ // Verify that original startree index was created
+ SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(INDEX_DIR);
+ List<StarTreeV2Metadata> originalStarTreeMetadataList =
segmentMetadata.getStarTreeV2MetadataList();
+ assertNotNull(originalStarTreeMetadataList);
+ assertFalse(originalStarTreeMetadataList.isEmpty());
+ assertEquals(originalStarTreeMetadataList.size(), 1);
+
assertEquals(originalStarTreeMetadataList.get(0).getFunctionColumnPairs().iterator().next().toColumnName(),
+ "sum__longCol");
+
+ // Test 4: Update existing startree index with valid functionColumnPair
(COUNT__*)
Review Comment:
Corrected spelling of 'startree' to 'star-tree' in comment for consistency
with other comments in the file.
```suggestion
// Test 4: Update existing star-tree index with valid functionColumnPair
(COUNT__*)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
