This is an automated email from the ASF dual-hosted git repository.
amogh-jahagirdar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 9719db6d61 Core: V4 refactor metrics config (#18269)
9719db6d61 is described below
commit 9719db6d616ba9e2c2d922071ef8c148b9f81849
Author: Ryan Blue <[email protected]>
AuthorDate: Mon Sep 28 16:32:39 2026 -0700
Core: V4 refactor metrics config (#18269)
---
.../java/org/apache/iceberg/MetricsConfig.java | 130 ++++++++++++++-------
.../main/java/org/apache/iceberg/MetricsUtil.java | 14 ++-
.../java/org/apache/iceberg/MetricsTestUtil.java | 39 +++++++
.../org/apache/iceberg/TestFileGenerationUtil.java | 9 +-
.../test/java/org/apache/iceberg/TestMetrics.java | 24 ++--
.../java/org/apache/iceberg/TestMetricsConfig.java | 8 +-
.../java/org/apache/iceberg/TestMetricsModes.java | 12 +-
.../java/org/apache/iceberg/TestStatsUtil.java | 8 +-
.../org/apache/iceberg/TestV4ManifestReader.java | 17 ++-
.../java/org/apache/iceberg/orc/OrcMetrics.java | 3 +-
.../org/apache/iceberg/parquet/ParquetMetrics.java | 16 +--
.../org/apache/iceberg/parquet/TestParquet.java | 10 +-
.../apache/iceberg/parquet/TestVariantMetrics.java | 31 +++--
.../extensions/TestMigrateTableProcedure.java | 2 +-
.../apache/iceberg/spark/TestSparkTableUtil.java | 19 ++-
.../extensions/TestMigrateTableProcedure.java | 2 +-
.../apache/iceberg/spark/source/SparkBatch.java | 4 +-
.../apache/iceberg/spark/TestSparkTableUtil.java | 19 ++-
.../extensions/TestMigrateTableProcedure.java | 2 +-
.../apache/iceberg/spark/source/SparkBatch.java | 4 +-
.../apache/iceberg/spark/TestSparkTableUtil.java | 19 ++-
.../extensions/TestMigrateTableProcedure.java | 2 +-
.../apache/iceberg/spark/source/SparkBatch.java | 4 +-
.../apache/iceberg/spark/TestSparkTableUtil.java | 19 ++-
24 files changed, 273 insertions(+), 144 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/MetricsConfig.java
b/core/src/main/java/org/apache/iceberg/MetricsConfig.java
index 87dae4c95d..273d51f888 100644
--- a/core/src/main/java/org/apache/iceberg/MetricsConfig.java
+++ b/core/src/main/java/org/apache/iceberg/MetricsConfig.java
@@ -253,79 +253,119 @@ public final class MetricsConfig implements Serializable
{
* @param schema table schema
* @param order sort order columns, will be promoted to truncate(16)
* @return metrics configuration
+ * @deprecated will be removed in 1.14.0; use {@link #forTable(Table)}
instead.
*/
+ @Deprecated
public static MetricsConfig from(Map<String, String> props, Schema schema,
SortOrder order) {
- int maxInferredDefaultColumns = maxInferredColumnDefaults(props);
- Map<Integer, String> idToName = Maps.newHashMap();
- Map<String, MetricsMode> columnModes = Maps.newHashMap();
+ int maxDefaultColumns = maxInferredColumnDefaults(props);
- // Handle user override of default mode
- MetricsMode defaultMode;
- String configuredDefault = props.get(DEFAULT_WRITE_METRICS_MODE);
+ // Handle configured default mode
+ MetricsMode configuredDefault = configuredDefault(props);
+ Map<String, MetricsMode> defaultColumnConf = defaultColumnModes(schema,
maxDefaultColumns);
+ MetricsMode defaultMode;
if (configuredDefault != null) {
- // a user-configured default mode is applied for all columns
- defaultMode = parseMode(configuredDefault, DEFAULT_MODE, "default");
- } else if (schema == null) {
+ defaultMode = configuredDefault;
+ } else if (defaultColumnConf.size() < maxDefaultColumns) {
+ // an additional column should use the default mode
defaultMode = DEFAULT_MODE;
} else {
- Set<Integer> ids = TypeUtil.getProjectedIds(schema);
- if (ids.size() <= maxInferredDefaultColumns) {
- for (int id : ids) {
- idToName.put(id, schema.findColumnName(id));
- }
+ // an additional column should not store metrics
+ defaultMode = MetricsModes.None.get();
+ }
- // there are less than the inferred limit (including structs), so the
default is used
- // everywhere
- defaultMode = DEFAULT_MODE;
- } else {
- for (Integer id : limitFieldIds(schema, maxInferredDefaultColumns)) {
- String name = schema.findColumnName(id);
- idToName.put(id, name);
- columnModes.put(name, DEFAULT_MODE);
- }
+ Map<String, MetricsMode> columnModes = Maps.newHashMap();
+
+ if (configuredDefault == null) {
+ columnModes.putAll(defaultColumnConf);
+ }
+
+ // Default sort columns to at least truncate (overridden by config)
+ columnModes.putAll(sortColumnModes(order, configuredDefault));
+
+ // Override automatic modes with configured modes
+ columnModes.putAll(configuredColumnModes(props));
+
+ Map<Integer, String> idToName = idToName(schema, columnModes);
+
+ return new MetricsConfig(columnModes, defaultMode, idToName);
+ }
+
+ private static MetricsMode configuredDefault(Map<String, String> props) {
+ String configuredDefault = props.get(DEFAULT_WRITE_METRICS_MODE);
+ if (configuredDefault != null) {
+ // a user-configured default mode is applied for all columns
+ return parseMode(configuredDefault, null, "default");
+ }
+
+ return null;
+ }
- // all other columns don't use metrics
- defaultMode = MetricsModes.None.get();
+ private static Map<String, MetricsMode> defaultColumnModes(Schema schema,
int maxColumns) {
+ ImmutableMap.Builder<String, MetricsMode> builder = ImmutableMap.builder();
+ if (schema != null) {
+ for (int id : limitFieldIds(schema, maxColumns)) {
+ builder.put(schema.findColumnName(id), DEFAULT_MODE);
}
}
- // First set sorted column with sorted column default (can be overridden
by user)
- MetricsMode sortedColDefaultMode = sortedColumnDefaultMode(defaultMode);
- Set<String> sortedCols = SortOrderUtil.orderPreservingSortedColumns(order);
- sortedCols.forEach(
- name -> {
- columnModes.put(name, sortedColDefaultMode);
- Types.NestedField field = schema != null ? schema.findField(name) :
null;
- if (field != null) {
- idToName.put(field.fieldId(), name);
- }
- });
+ return builder.build();
+ }
+
+ private static Map<String, MetricsMode> sortColumnModes(
+ SortOrder order, MetricsMode configuredDefault) {
+ ImmutableMap.Builder<String, MetricsMode> builder = ImmutableMap.builder();
+ MetricsMode sortDefault = promoteToIncludeBounds(configuredDefault);
+ for (String name : SortOrderUtil.orderPreservingSortedColumns(order)) {
+ builder.put(name, sortDefault);
+ }
- // Handle user overrides of defaults
+ return builder.build();
+ }
+
+ private static Map<String, MetricsMode> configuredColumnModes(Map<String,
String> props) {
+ ImmutableMap.Builder<String, MetricsMode> builder = ImmutableMap.builder();
for (String key : props.keySet()) {
if (key.startsWith(METRICS_MODE_COLUMN_CONF_PREFIX)) {
String columnAlias = key.replaceFirst(METRICS_MODE_COLUMN_CONF_PREFIX,
"");
- MetricsMode mode = parseMode(props.get(key), defaultMode, "column " +
columnAlias);
- columnModes.put(columnAlias, mode);
- Types.NestedField field = schema != null ?
schema.findField(columnAlias) : null;
+ MetricsMode mode = parseMode(props.get(key), null, "column " +
columnAlias);
+ if (mode != null) {
+ builder.put(columnAlias, mode);
+ }
+ }
+ }
+
+ return builder.build();
+ }
+
+ private static Map<Integer, String> idToName(
+ Schema schema, Map<String, MetricsMode> columnModes) {
+ if (schema != null) {
+ ImmutableMap.Builder<Integer, String> builder = ImmutableMap.builder();
+ for (String name : columnModes.keySet()) {
+ Types.NestedField field = schema.findField(name);
if (field != null) {
- idToName.put(field.fieldId(), columnAlias);
+ builder.put(field.fieldId(), name);
}
}
+
+ return builder.buildKeepingLast();
}
- return new MetricsConfig(columnModes, defaultMode, idToName);
+ return null;
}
/**
- * Auto promote sorted columns to truncate(16) if default is set at Counts
or None.
+ * Mode used for automatic metrics for sort and partitioning. Uses
truncate(16) if default is
+ * Counts or None.
*
* @param defaultMode default mode
* @return mode to use
*/
- private static MetricsMode sortedColumnDefaultMode(MetricsMode defaultMode) {
- if (defaultMode == MetricsModes.None.get() || defaultMode ==
MetricsModes.Counts.get()) {
+ private static MetricsMode promoteToIncludeBounds(MetricsMode defaultMode) {
+ if (defaultMode == null
+ || defaultMode == MetricsModes.None.get()
+ || defaultMode == MetricsModes.Counts.get()) {
return MetricsModes.Truncate.withLength(16);
} else {
return defaultMode;
diff --git a/core/src/main/java/org/apache/iceberg/MetricsUtil.java
b/core/src/main/java/org/apache/iceberg/MetricsUtil.java
index 5fa6c95a85..c93523aed4 100644
--- a/core/src/main/java/org/apache/iceberg/MetricsUtil.java
+++ b/core/src/main/java/org/apache/iceberg/MetricsUtil.java
@@ -112,9 +112,7 @@ public class MetricsUtil {
return fieldMetrics
.filter(metrics -> !inMapOrList(inputSchema, parents, metrics.id()))
- .filter(
- metrics ->
- metricsMode(inputSchema, metricsConfig, metrics.id()) !=
MetricsModes.None.get())
+ .filter(metrics -> metricsConfig.columnMode(metrics.id()) !=
MetricsModes.None.get())
.collect(Collectors.toMap(FieldMetrics::id,
FieldMetrics::nanValueCount));
}
@@ -129,14 +127,18 @@ public class MetricsUtil {
return false;
}
- /** Extract MetricsMode for the given field id from metrics config. */
+ /**
+ * Extract MetricsMode for the given field id from metrics config.
+ *
+ * @deprecated will be removed in 1.14.0; use metricsConfig.columnMode(int)
instead.
+ */
+ @Deprecated
public static MetricsModes.MetricsMode metricsMode(
Schema inputSchema, MetricsConfig metricsConfig, int fieldId) {
Preconditions.checkNotNull(inputSchema, "inputSchema is required");
Preconditions.checkNotNull(metricsConfig, "metricsConfig is required");
- String columnName = inputSchema.findColumnName(fieldId);
- return metricsConfig.columnMode(columnName);
+ return metricsConfig.columnMode(fieldId);
}
public static final List<ReadableMetricColDefinition> READABLE_METRIC_COLS =
diff --git a/core/src/test/java/org/apache/iceberg/MetricsTestUtil.java
b/core/src/test/java/org/apache/iceberg/MetricsTestUtil.java
new file mode 100644
index 0000000000..db0874d111
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/MetricsTestUtil.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.util.Map;
+import org.mockito.Mockito;
+
+public class MetricsTestUtil {
+ private MetricsTestUtil() {}
+
+ public static MetricsConfig from(Map<String, String> properties, Schema
schema) {
+ return from(properties, schema, null);
+ }
+
+ public static MetricsConfig from(
+ Map<String, String> properties, Schema schema, SortOrder sortOrder) {
+ Table mock = Mockito.mock(Table.class);
+ Mockito.when(mock.properties()).thenReturn(properties);
+ Mockito.when(mock.schema()).thenReturn(schema);
+ Mockito.when(mock.sortOrder()).thenReturn(sortOrder);
+ return MetricsConfig.forTable(mock);
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java
b/core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java
index 590d4e1601..1f510be74e 100644
--- a/core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java
+++ b/core/src/test/java/org/apache/iceberg/TestFileGenerationUtil.java
@@ -106,8 +106,8 @@ public class TestFileGenerationUtil {
@ValueSource(strings = {"none", "counts", "truncate(16)", "full"})
void testBoundsForAllMetricsModes(String metricsMode) {
MetricsConfig metricsConfig =
- MetricsConfig.from(
- ImmutableMap.of(TableProperties.DEFAULT_WRITE_METRICS_MODE,
metricsMode), SCHEMA, null);
+ MetricsTestUtil.from(
+ ImmutableMap.of(TableProperties.DEFAULT_WRITE_METRICS_MODE,
metricsMode), SCHEMA);
Metrics metrics =
FileGenerationUtil.generateRandomMetrics(
SCHEMA,
@@ -122,10 +122,9 @@ public class TestFileGenerationUtil {
@ValueSource(strings = {"none", "counts", "truncate(16)", "full"})
public void testBoundsForNestedTypes(String metricsMode) {
MetricsConfig metricsConfig =
- MetricsConfig.from(
+ MetricsTestUtil.from(
ImmutableMap.of(TableProperties.DEFAULT_WRITE_METRICS_MODE,
metricsMode),
- NESTED_SCHEMA,
- null);
+ NESTED_SCHEMA);
Metrics metrics =
FileGenerationUtil.generateRandomMetrics(
NESTED_SCHEMA,
diff --git a/core/src/test/java/org/apache/iceberg/TestMetrics.java
b/core/src/test/java/org/apache/iceberg/TestMetrics.java
index 9b25825316..89d528c22e 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetrics.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetrics.java
@@ -353,7 +353,7 @@ public abstract class TestMetrics {
MetricsModes.None.get().toString(),
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"nestedStructCol.longCol",
MetricsModes.Full.get().toString());
- MetricsConfig config = MetricsConfig.from(properties, NESTED_SCHEMA, null);
+ MetricsConfig config = MetricsTestUtil.from(properties, NESTED_SCHEMA);
Metrics metrics = getMetrics(NESTED_SCHEMA, config,
buildNestedTestRecord());
assertThat(metrics.recordCount()).isEqualTo(1L);
@@ -595,8 +595,8 @@ public abstract class TestMetrics {
Metrics metrics =
getMetrics(
NESTED_SCHEMA,
- MetricsConfig.from(
- ImmutableMap.of("write.metadata.metrics.default", "none"),
NESTED_SCHEMA, null),
+ MetricsTestUtil.from(
+ ImmutableMap.of("write.metadata.metrics.default", "none"),
NESTED_SCHEMA),
buildNestedTestRecord());
assertThat(metrics.recordCount()).isEqualTo(1L);
assertThat(metrics.columnSizes()).isEmpty();
@@ -617,8 +617,8 @@ public abstract class TestMetrics {
Metrics metrics =
getMetrics(
NESTED_SCHEMA,
- MetricsConfig.from(
- ImmutableMap.of("write.metadata.metrics.default", "counts"),
NESTED_SCHEMA, null),
+ MetricsTestUtil.from(
+ ImmutableMap.of("write.metadata.metrics.default", "counts"),
NESTED_SCHEMA),
buildNestedTestRecord());
assertThat(metrics.recordCount()).isEqualTo(1L);
assertThat(metrics.columnSizes()).doesNotContainValue(null);
@@ -640,8 +640,8 @@ public abstract class TestMetrics {
Metrics metrics =
getMetrics(
NESTED_SCHEMA,
- MetricsConfig.from(
- ImmutableMap.of("write.metadata.metrics.default", "full"),
NESTED_SCHEMA, null),
+ MetricsTestUtil.from(
+ ImmutableMap.of("write.metadata.metrics.default", "full"),
NESTED_SCHEMA),
buildNestedTestRecord());
assertThat(metrics.recordCount()).isEqualTo(1L);
assertThat(metrics.columnSizes()).doesNotContainValue(null);
@@ -675,10 +675,9 @@ public abstract class TestMetrics {
Metrics metrics =
getMetrics(
singleStringColSchema,
- MetricsConfig.from(
+ MetricsTestUtil.from(
ImmutableMap.of("write.metadata.metrics.default",
"truncate(10)"),
- singleStringColSchema,
- null),
+ singleStringColSchema),
record);
CharBuffer expectedMinBound = CharBuffer.wrap("Lorem ipsu");
@@ -702,10 +701,9 @@ public abstract class TestMetrics {
Metrics metrics =
getMetrics(
singleBinaryColSchema,
- MetricsConfig.from(
+ MetricsTestUtil.from(
ImmutableMap.of("write.metadata.metrics.default",
"truncate(5)"),
- singleBinaryColSchema,
- null),
+ singleBinaryColSchema),
record);
ByteBuffer expectedMinBounds = ByteBuffer.wrap(new byte[] {0x1, 0x2, 0x3,
0x4, 0x5});
diff --git a/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java
b/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java
index 4eeada847d..cb4459472c 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java
@@ -128,7 +128,7 @@ public class TestMetricsConfig {
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", "full",
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "data", "none");
- MetricsConfig config = MetricsConfig.from(props, schema, null);
+ MetricsConfig config = MetricsTestUtil.from(props, schema);
assertThat(config.metricsFieldIds())
.as("Should track field ids for configured and defaulted columns")
@@ -151,7 +151,7 @@ public class TestMetricsConfig {
optional(2, "data", Types.StringType.get()),
optional(3, "category", Types.StringType.get()));
- MetricsConfig config = MetricsConfig.from(ImmutableMap.of(), schema, null);
+ MetricsConfig config = MetricsTestUtil.from(ImmutableMap.of(), schema);
assertThat(config.metricsFieldIds())
.as("Should track field ids for all columns that use the default mode")
@@ -174,7 +174,7 @@ public class TestMetricsConfig {
Map<String, String> limitedToTwo =
ImmutableMap.of(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS,
"2");
- MetricsConfig config = MetricsConfig.from(limitedToTwo, schema, null);
+ MetricsConfig config = MetricsTestUtil.from(limitedToTwo, schema);
// only the fields within the inferred limit are tracked, matching
limitFieldIds
assertThat(config.metricsFieldIds())
@@ -195,7 +195,7 @@ public class TestMetricsConfig {
// raising the limit expands both the tracked ids and the columns with
metrics
Map<String, String> limitedToThree =
ImmutableMap.of(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS,
"3");
- MetricsConfig wider = MetricsConfig.from(limitedToThree, schema, null);
+ MetricsConfig wider = MetricsTestUtil.from(limitedToThree, schema);
assertThat(wider.metricsFieldIds())
.as("Raising the limit should track more field ids")
diff --git a/core/src/test/java/org/apache/iceberg/TestMetricsModes.java
b/core/src/test/java/org/apache/iceberg/TestMetricsModes.java
index f96857f8fb..6c873ec21b 100644
--- a/core/src/test/java/org/apache/iceberg/TestMetricsModes.java
+++ b/core/src/test/java/org/apache/iceberg/TestMetricsModes.java
@@ -87,7 +87,9 @@ public class TestMetricsModes {
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col",
"troncate(5)");
- MetricsConfig config = MetricsConfig.from(properties, null, null);
+ Schema schema = new Schema(required(1, "col", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(properties, schema);
assertThat(config.columnMode("col"))
.as("Invalid mode should be defaulted to table default (full)")
.isEqualTo(MetricsModes.Full.get());
@@ -102,14 +104,16 @@ public class TestMetricsModes {
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col",
"troncate(5)");
- MetricsConfig config = MetricsConfig.from(properties, null, null);
+ Schema schema = new Schema(required(1, "col", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(properties, schema);
assertThat(config.columnMode("col"))
.as("Invalid mode should be defaulted to library default
(truncate(16))")
.isEqualTo(MetricsModes.Truncate.withLength(16));
}
@TestTemplate
- public void testMetricsConfigSortedColsDefault() throws Exception {
+ public void testMetricsConfigSortedColsDefault() {
Schema schema =
new Schema(
required(1, "col1", Types.IntegerType.get()),
@@ -166,7 +170,7 @@ public class TestMetricsModes {
.isEqualTo(Full.get());
assertThat(config.columnMode("col2"))
.as("Original default applies as user entered invalid mode for sorted
column")
- .isEqualTo(Counts.get());
+ .isEqualTo(Truncate.withLength(16));
}
@TestTemplate
diff --git a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
index 71ebf227af..183c4336f7 100644
--- a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
+++ b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java
@@ -462,7 +462,7 @@ public class TestStatsUtil {
Types.NestedField.optional(10_400, "data", dataStats));
Types.StructType actual =
- StatsUtil.statsWriteSchema(schema,
MetricsConfig.from(ImmutableMap.of(), schema, null));
+ StatsUtil.statsWriteSchema(schema,
MetricsTestUtil.from(ImmutableMap.of(), schema));
assertSameStructure(expected, actual);
}
@@ -499,7 +499,7 @@ public class TestStatsUtil {
Map<String, String> properties =
ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"data", "none");
Types.StructType actual =
- StatsUtil.statsWriteSchema(schema, MetricsConfig.from(properties,
schema, null));
+ StatsUtil.statsWriteSchema(schema, MetricsTestUtil.from(properties,
schema));
assertSameStructure(expected, actual);
}
@@ -541,7 +541,7 @@ public class TestStatsUtil {
Map<String, String> properties =
ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"id", "none");
Types.StructType actual =
- StatsUtil.statsWriteSchema(schema, MetricsConfig.from(properties,
schema, null));
+ StatsUtil.statsWriteSchema(schema, MetricsTestUtil.from(properties,
schema));
assertSameStructure(expected, actual);
}
@@ -559,7 +559,7 @@ public class TestStatsUtil {
Types.MapType.ofOptional(4, 5, Types.StringType.get(),
Types.StringType.get())));
Types.StructType actual =
- StatsUtil.statsWriteSchema(schema,
MetricsConfig.from(ImmutableMap.of(), schema, null));
+ StatsUtil.statsWriteSchema(schema,
MetricsTestUtil.from(ImmutableMap.of(), schema));
assertThat(actual.fields()).isEmpty();
}
diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java
b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java
index 92104ac90c..7b8e41a534 100644
--- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java
+++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java
@@ -93,7 +93,7 @@ class TestV4ManifestReader {
ImmutableMap.of(PartitionSpec.unpartitioned().specId(),
PartitionSpec.unpartitioned());
private static final MetricsConfig METRICS_CONFIG =
- MetricsConfig.from(ImmutableMap.of(), TABLE_SCHEMA, null);
+ MetricsTestUtil.from(ImmutableMap.of(), TABLE_SCHEMA);
private static final Types.StructType STATS_TYPE =
StatsUtil.statsWriteSchema(TABLE_SCHEMA, METRICS_CONFIG);
private static final Types.StructType ID_ONLY_STATS_TYPE =
@@ -802,10 +802,9 @@ class TestV4ManifestReader {
@Test
public void projectionDependsOnMetricsConfig() {
MetricsConfig metricsWithoutID =
- MetricsConfig.from(
+ MetricsTestUtil.from(
ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"id", "none"),
- TABLE_SCHEMA,
- null);
+ TABLE_SCHEMA);
Types.StructType readSchema =
V4ManifestReader.builder(UNREAD_MANIFEST_FILE, IO, TABLE_SCHEMA,
ID_PARTITIONING_SPECS)
@@ -850,10 +849,9 @@ class TestV4ManifestReader {
@Test
public void projectionForScanPlanningOverridesMetricsConfig() {
MetricsConfig metricsWithoutID =
- MetricsConfig.from(
+ MetricsTestUtil.from(
ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"id", "none"),
- TABLE_SCHEMA,
- null);
+ TABLE_SCHEMA);
Types.StructType readSchema =
V4ManifestReader.builder(UNREAD_MANIFEST_FILE, IO, TABLE_SCHEMA,
ID_PARTITIONING_SPECS)
@@ -878,10 +876,9 @@ class TestV4ManifestReader {
@Test
public void projectionForScanPlanningIncludesRequestedStatsMetricsConfig() {
MetricsConfig metricsWithoutID =
- MetricsConfig.from(
+ MetricsTestUtil.from(
ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"id", "none"),
- TABLE_SCHEMA,
- null);
+ TABLE_SCHEMA);
Types.StructType readSchema =
V4ManifestReader.builder(UNREAD_MANIFEST_FILE, IO, TABLE_SCHEMA,
ID_PARTITIONING_SPECS)
diff --git a/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java
b/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java
index 7498d6e155..6bebb1c3ff 100644
--- a/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java
+++ b/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java
@@ -163,8 +163,7 @@ public class OrcMetrics {
final Types.NestedField icebergCol = icebergColOpt.get();
final int fieldId = icebergCol.fieldId();
- final MetricsMode metricsMode =
- MetricsUtil.metricsMode(schema, effectiveMetricsConfig,
icebergCol.fieldId());
+ final MetricsMode metricsMode =
effectiveMetricsConfig.columnMode(icebergCol.fieldId());
if (metricsMode == MetricsModes.None.get() || inMapOrList(orcCol)) {
continue;
diff --git
a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
index 5aed952d9d..346931b375 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
@@ -31,7 +31,6 @@ import org.apache.iceberg.FieldMetrics;
import org.apache.iceberg.Metrics;
import org.apache.iceberg.MetricsConfig;
import org.apache.iceberg.MetricsModes;
-import org.apache.iceberg.MetricsUtil;
import org.apache.iceberg.Schema;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
@@ -93,7 +92,7 @@ class ParquetMetrics {
fields.collect(Collectors.toMap(FieldMetrics::id,
Function.identity()));
return TypeWithSchemaVisitor.visit(
- schema.asStruct(), type, new MetricsVisitor(schema, metricsConfig,
metricsById, columns));
+ schema.asStruct(), type, new MetricsVisitor(metricsConfig,
metricsById, columns));
}
private static long rowCount(ParquetMetadata metadata) {
@@ -106,7 +105,7 @@ class ParquetMetrics {
}
private static Map<Integer, Long> columnSizes(
- Schema schema, MessageType type, ParquetMetadata metadata, MetricsConfig
metricsConfig) {
+ MessageType type, ParquetMetadata metadata, MetricsConfig metricsConfig)
{
Map<Integer, Long> columnSizes = Maps.newHashMap();
for (BlockMetaData block : metadata.getBlocks()) {
for (ColumnChunkMetaData column : block.getColumns()) {
@@ -114,7 +113,7 @@ class ParquetMetrics {
type.getColumnDescription(column.getPath().toArray()).getPrimitiveType().getId();
if (id != null) {
int fieldId = id.intValue();
- MetricsModes.MetricsMode mode = MetricsUtil.metricsMode(schema,
metricsConfig, fieldId);
+ MetricsModes.MetricsMode mode = metricsConfig.columnMode(fieldId);
if (mode != MetricsModes.None.get()) {
columnSizes.put(fieldId, columnSizes.getOrDefault(fieldId, 0L) +
column.getTotalSize());
}
@@ -132,7 +131,7 @@ class ParquetMetrics {
ParquetMetadata metadata,
Stream<FieldMetrics<?>> fields) {
long rowCount = rowCount(metadata);
- Map<Integer, Long> columnSizes = columnSizes(schema, type, metadata,
metricsConfig);
+ Map<Integer, Long> columnSizes = columnSizes(type, metadata,
metricsConfig);
Map<Integer, Long> valueCounts = Maps.newHashMap();
Map<Integer, Long> nullValueCounts = Maps.newHashMap();
@@ -190,17 +189,14 @@ class ParquetMetrics {
}
private static class MetricsVisitor extends
TypeWithSchemaVisitor<Iterable<FieldMetrics<?>>> {
- private final Schema schema;
private final MetricsConfig metricsConfig;
private final Map<Integer, FieldMetrics<?>> metricsById;
private final Multimap<ColumnPath, ColumnChunkMetaData> columns;
private MetricsVisitor(
- Schema schema,
MetricsConfig metricsConfig,
Map<Integer, FieldMetrics<?>> metricsById,
Multimap<ColumnPath, ColumnChunkMetaData> columns) {
- this.schema = schema;
this.metricsConfig = metricsConfig;
this.metricsById = metricsById;
this.columns = columns;
@@ -246,7 +242,7 @@ class ParquetMetrics {
}
int fieldId = id.intValue();
- MetricsModes.MetricsMode mode = MetricsUtil.metricsMode(schema,
metricsConfig, fieldId);
+ MetricsModes.MetricsMode mode = metricsConfig.columnMode(fieldId);
if (mode == MetricsModes.None.get()) {
return ImmutableList.of();
}
@@ -397,7 +393,7 @@ class ParquetMetrics {
}
int fieldId = id.intValue();
- MetricsModes.MetricsMode mode = MetricsUtil.metricsMode(schema,
metricsConfig, fieldId);
+ MetricsModes.MetricsMode mode = metricsConfig.columnMode(fieldId);
if (mode == MetricsModes.None.get()) {
return ImmutableList.of();
}
diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java
b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java
index 1a5b540504..b778536ae5 100644
--- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java
+++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java
@@ -56,6 +56,7 @@ import org.apache.avro.generic.GenericRecordBuilder;
import org.apache.iceberg.Files;
import org.apache.iceberg.Metrics;
import org.apache.iceberg.MetricsConfig;
+import org.apache.iceberg.MetricsTestUtil;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.avro.AvroSchemaUtil;
@@ -926,10 +927,15 @@ public class TestParquet {
@Test
public void missingNullCountWithCountsMode() {
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "id", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "data", Types.StringType.get()));
+
Metrics metrics =
missingNullCountMetrics(
- MetricsConfig.fromProperties(
- Collections.singletonMap("write.metadata.metrics.default",
"counts")),
+ MetricsTestUtil.from(
+ Collections.singletonMap("write.metadata.metrics.default",
"counts"), schema),
block(statsWithoutNullCount(1, 10), 10),
block(statsWithNullCount(20, 30, 1), 10));
diff --git
a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java
b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java
index 15291a1930..3e0c088192 100644
--- a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java
+++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java
@@ -33,6 +33,7 @@ import java.util.stream.Stream;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.Metrics;
import org.apache.iceberg.MetricsConfig;
+import org.apache.iceberg.MetricsTestUtil;
import org.apache.iceberg.Schema;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.data.GenericRecord;
@@ -293,10 +294,9 @@ public class TestVariantMetrics {
VariantValue value = Variants.of(ByteBuffer.wrap(BINARY_20_BYTES));
MetricsConfig metricsConfig =
- MetricsConfig.from(
+ MetricsTestUtil.from(
ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"var", "truncate(8)"),
- SCHEMA,
- null);
+ SCHEMA);
Metrics metrics =
writeParquetWithMetricsConfig(
@@ -321,10 +321,9 @@ public class TestVariantMetrics {
VariantValue value = Variants.of(ByteBuffer.wrap(BINARY_20_BYTES));
MetricsConfig metricsConfig =
- MetricsConfig.from(
+ MetricsTestUtil.from(
ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"var", "full"),
- SCHEMA,
- null);
+ SCHEMA);
Metrics metrics =
writeParquetWithMetricsConfig(
@@ -349,10 +348,9 @@ public class TestVariantMetrics {
VariantValue value = Variants.of(ByteBuffer.wrap(BINARY_20_BYTES));
MetricsConfig metricsConfig =
- MetricsConfig.from(
+ MetricsTestUtil.from(
ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"var", "counts"),
- SCHEMA,
- null);
+ SCHEMA);
Metrics metrics =
writeParquetWithMetricsConfig(
@@ -373,10 +371,9 @@ public class TestVariantMetrics {
VariantValue value = Variants.of("iceberg_variant");
MetricsConfig metricsConfig =
- MetricsConfig.from(
+ MetricsTestUtil.from(
ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"var", "truncate(8)"),
- SCHEMA,
- null);
+ SCHEMA);
Metrics metrics =
writeParquetWithMetricsConfig(
@@ -401,10 +398,9 @@ public class TestVariantMetrics {
VariantValue value = Variants.of("iceberg_variant_full");
MetricsConfig metricsConfig =
- MetricsConfig.from(
+ MetricsTestUtil.from(
ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"var", "full"),
- SCHEMA,
- null);
+ SCHEMA);
Metrics metrics =
writeParquetWithMetricsConfig(
@@ -429,10 +425,9 @@ public class TestVariantMetrics {
VariantValue value = Variants.of("iceberg_variant");
MetricsConfig metricsConfig =
- MetricsConfig.from(
+ MetricsTestUtil.from(
ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"var", "counts"),
- SCHEMA,
- null);
+ SCHEMA);
Metrics metrics =
writeParquetWithMetricsConfig(
diff --git
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
index 39bf49ce0c..832e5376bf 100644
---
a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
+++
b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
@@ -151,7 +151,7 @@ public class TestMigrateTableProcedure extends
ExtensionsTestBase {
assertThatThrownBy(
() -> {
- String props = "map('write.metadata.metrics.column.x', 'X')";
+ String props = "map('write.metadata.metrics.column.x',
'counts')";
sql("CALL %s.system.migrate('%s', %s)", catalogName, tableName,
props);
})
.isInstanceOf(ValidationException.class)
diff --git
a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
index b57480a61e..ca37f933c6 100644
---
a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
+++
b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
@@ -25,10 +25,13 @@ import java.util.Map;
import org.apache.iceberg.KryoHelpers;
import org.apache.iceberg.MetricsConfig;
import org.apache.iceberg.MetricsModes;
+import org.apache.iceberg.MetricsTestUtil;
+import org.apache.iceberg.Schema;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.spark.SparkTableUtil.SparkPartition;
+import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;
public class TestSparkTableUtil {
@@ -65,7 +68,13 @@ public class TestSparkTableUtil {
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
"truncate(16)");
- MetricsConfig config = MetricsConfig.from(metricsConfig, null, null);
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "col1", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "col2", Types.StringType.get()),
+ Types.NestedField.optional(3, "col3", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(metricsConfig, schema);
MetricsConfig deserialized = KryoHelpers.roundTripSerialize(config);
assertThat(deserialized.columnMode("col1"))
@@ -90,7 +99,13 @@ public class TestSparkTableUtil {
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
"truncate(16)");
- MetricsConfig config = MetricsConfig.from(metricsConfig, null, null);
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "col1", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "col2", Types.StringType.get()),
+ Types.NestedField.optional(3, "col3", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(metricsConfig, schema);
MetricsConfig deserialized = TestHelpers.roundTripSerialize(config);
assertThat(deserialized.columnMode("col1"))
diff --git
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
index de1322384b..8b6aa6a722 100644
---
a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
+++
b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
@@ -151,7 +151,7 @@ public class TestMigrateTableProcedure extends
ExtensionsTestBase {
assertThatThrownBy(
() -> {
- String props = "map('write.metadata.metrics.column.x', 'X')";
+ String props = "map('write.metadata.metrics.column.x',
'counts')";
sql("CALL %s.system.migrate('%s', %s)", catalogName, tableName,
props);
})
.isInstanceOf(ValidationException.class)
diff --git
a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java
b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java
index f90423900d..b50d060955 100644
---
a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java
+++
b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java
@@ -28,7 +28,6 @@ import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.MetricsConfig;
import org.apache.iceberg.MetricsModes;
-import org.apache.iceberg.MetricsUtil;
import org.apache.iceberg.ScanTask;
import org.apache.iceberg.ScanTaskGroup;
import org.apache.iceberg.Schema;
@@ -201,8 +200,7 @@ class SparkBatch implements Batch {
}
MetricsConfig metricsConfig = MetricsConfig.forTable(table);
- MetricsModes.MetricsMode mode =
- MetricsUtil.metricsMode(table.schema(), metricsConfig,
field.fieldId());
+ MetricsModes.MetricsMode mode =
metricsConfig.columnMode(field.fieldId());
if (mode == MetricsModes.None.get() || mode ==
MetricsModes.Counts.get()) {
return false;
}
diff --git
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
index b57480a61e..ca37f933c6 100644
---
a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
+++
b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
@@ -25,10 +25,13 @@ import java.util.Map;
import org.apache.iceberg.KryoHelpers;
import org.apache.iceberg.MetricsConfig;
import org.apache.iceberg.MetricsModes;
+import org.apache.iceberg.MetricsTestUtil;
+import org.apache.iceberg.Schema;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.spark.SparkTableUtil.SparkPartition;
+import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;
public class TestSparkTableUtil {
@@ -65,7 +68,13 @@ public class TestSparkTableUtil {
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
"truncate(16)");
- MetricsConfig config = MetricsConfig.from(metricsConfig, null, null);
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "col1", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "col2", Types.StringType.get()),
+ Types.NestedField.optional(3, "col3", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(metricsConfig, schema);
MetricsConfig deserialized = KryoHelpers.roundTripSerialize(config);
assertThat(deserialized.columnMode("col1"))
@@ -90,7 +99,13 @@ public class TestSparkTableUtil {
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
"truncate(16)");
- MetricsConfig config = MetricsConfig.from(metricsConfig, null, null);
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "col1", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "col2", Types.StringType.get()),
+ Types.NestedField.optional(3, "col3", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(metricsConfig, schema);
MetricsConfig deserialized = TestHelpers.roundTripSerialize(config);
assertThat(deserialized.columnMode("col1"))
diff --git
a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
index 5fd0fe9665..80c2fa6e61 100644
---
a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
+++
b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
@@ -151,7 +151,7 @@ public class TestMigrateTableProcedure extends
ExtensionsTestBase {
assertThatThrownBy(
() -> {
- String props = "map('write.metadata.metrics.column.x', 'X')";
+ String props = "map('write.metadata.metrics.column.x',
'counts')";
sql("CALL %s.system.migrate('%s', %s)", catalogName, tableName,
props);
})
.isInstanceOf(ValidationException.class)
diff --git
a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java
b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java
index 5cafa1c9fa..81fb008963 100644
---
a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java
+++
b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java
@@ -28,7 +28,6 @@ import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.MetricsConfig;
import org.apache.iceberg.MetricsModes;
-import org.apache.iceberg.MetricsUtil;
import org.apache.iceberg.ScanTask;
import org.apache.iceberg.ScanTaskGroup;
import org.apache.iceberg.Schema;
@@ -211,8 +210,7 @@ class SparkBatch implements Batch {
}
MetricsConfig metricsConfig = MetricsConfig.forTable(table);
- MetricsModes.MetricsMode mode =
- MetricsUtil.metricsMode(table.schema(), metricsConfig,
field.fieldId());
+ MetricsModes.MetricsMode mode =
metricsConfig.columnMode(field.fieldId());
if (mode == MetricsModes.None.get() || mode ==
MetricsModes.Counts.get()) {
return false;
}
diff --git
a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
index b57480a61e..ca37f933c6 100644
---
a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
+++
b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
@@ -25,10 +25,13 @@ import java.util.Map;
import org.apache.iceberg.KryoHelpers;
import org.apache.iceberg.MetricsConfig;
import org.apache.iceberg.MetricsModes;
+import org.apache.iceberg.MetricsTestUtil;
+import org.apache.iceberg.Schema;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.spark.SparkTableUtil.SparkPartition;
+import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;
public class TestSparkTableUtil {
@@ -65,7 +68,13 @@ public class TestSparkTableUtil {
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
"truncate(16)");
- MetricsConfig config = MetricsConfig.from(metricsConfig, null, null);
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "col1", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "col2", Types.StringType.get()),
+ Types.NestedField.optional(3, "col3", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(metricsConfig, schema);
MetricsConfig deserialized = KryoHelpers.roundTripSerialize(config);
assertThat(deserialized.columnMode("col1"))
@@ -90,7 +99,13 @@ public class TestSparkTableUtil {
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
"truncate(16)");
- MetricsConfig config = MetricsConfig.from(metricsConfig, null, null);
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "col1", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "col2", Types.StringType.get()),
+ Types.NestedField.optional(3, "col3", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(metricsConfig, schema);
MetricsConfig deserialized = TestHelpers.roundTripSerialize(config);
assertThat(deserialized.columnMode("col1"))
diff --git
a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
index 5fd0fe9665..80c2fa6e61 100644
---
a/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
+++
b/spark/v4.2/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java
@@ -151,7 +151,7 @@ public class TestMigrateTableProcedure extends
ExtensionsTestBase {
assertThatThrownBy(
() -> {
- String props = "map('write.metadata.metrics.column.x', 'X')";
+ String props = "map('write.metadata.metrics.column.x',
'counts')";
sql("CALL %s.system.migrate('%s', %s)", catalogName, tableName,
props);
})
.isInstanceOf(ValidationException.class)
diff --git
a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java
b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java
index 5cafa1c9fa..81fb008963 100644
---
a/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java
+++
b/spark/v4.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkBatch.java
@@ -28,7 +28,6 @@ import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.MetricsConfig;
import org.apache.iceberg.MetricsModes;
-import org.apache.iceberg.MetricsUtil;
import org.apache.iceberg.ScanTask;
import org.apache.iceberg.ScanTaskGroup;
import org.apache.iceberg.Schema;
@@ -211,8 +210,7 @@ class SparkBatch implements Batch {
}
MetricsConfig metricsConfig = MetricsConfig.forTable(table);
- MetricsModes.MetricsMode mode =
- MetricsUtil.metricsMode(table.schema(), metricsConfig,
field.fieldId());
+ MetricsModes.MetricsMode mode =
metricsConfig.columnMode(field.fieldId());
if (mode == MetricsModes.None.get() || mode ==
MetricsModes.Counts.get()) {
return false;
}
diff --git
a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
index b57480a61e..ca37f933c6 100644
---
a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
+++
b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/TestSparkTableUtil.java
@@ -25,10 +25,13 @@ import java.util.Map;
import org.apache.iceberg.KryoHelpers;
import org.apache.iceberg.MetricsConfig;
import org.apache.iceberg.MetricsModes;
+import org.apache.iceberg.MetricsTestUtil;
+import org.apache.iceberg.Schema;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.spark.SparkTableUtil.SparkPartition;
+import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;
public class TestSparkTableUtil {
@@ -65,7 +68,13 @@ public class TestSparkTableUtil {
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
"truncate(16)");
- MetricsConfig config = MetricsConfig.from(metricsConfig, null, null);
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "col1", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "col2", Types.StringType.get()),
+ Types.NestedField.optional(3, "col3", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(metricsConfig, schema);
MetricsConfig deserialized = KryoHelpers.roundTripSerialize(config);
assertThat(deserialized.columnMode("col1"))
@@ -90,7 +99,13 @@ public class TestSparkTableUtil {
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "col2",
"truncate(16)");
- MetricsConfig config = MetricsConfig.from(metricsConfig, null, null);
+ Schema schema =
+ new Schema(
+ Types.NestedField.required(1, "col1", Types.IntegerType.get()),
+ Types.NestedField.optional(2, "col2", Types.StringType.get()),
+ Types.NestedField.optional(3, "col3", Types.StringType.get()));
+
+ MetricsConfig config = MetricsTestUtil.from(metricsConfig, schema);
MetricsConfig deserialized = TestHelpers.roundTripSerialize(config);
assertThat(deserialized.columnMode("col1"))
