drexler-sky commented on code in PR #18885:
URL: https://github.com/apache/iceberg/pull/18885#discussion_r4559699814
##########
core/src/main/java/org/apache/iceberg/SchemaUpdate.java:
##########
@@ -506,22 +506,33 @@ private TableMetadata
applyChangesToMetadata(TableMetadata metadata) {
// Transform the metrics if they exist
if (base != null && base.properties() != null) {
- Schema newSchema = newMetadata.schema();
- List<String> deletedColumns =
-
deletes.stream().map(schema::findColumnName).collect(Collectors.toList());
- Map<String, String> renamedColumns =
- updates.keySet().stream()
- .filter(id -> !addedNameToId.containsValue(id)) // remove added
columns
- .filter(id ->
!schema.findColumnName(id).equals(newSchema.findColumnName(id)))
- .collect(Collectors.toMap(schema::findColumnName,
newSchema::findColumnName));
+ Map<Integer, String> newIdToName = newMetadata.schema().idToName();
Review Comment:
Done
##########
core/src/test/java/org/apache/iceberg/TestSchemaAndMappingUpdate.java:
##########
@@ -229,6 +236,84 @@ public void testModificationWithParquetColumnCompression()
{
.doesNotContainKey(PARQUET_COLUMN_COMPRESSION_LEVEL_PREFIX + "ID");
}
+ @TestTemplate
+ public void testModificationWithParquetBloomFilterFppAndNdv() {
+ table
+ .updateProperties()
+ .set(PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX + "id", "0.05")
+ .set(PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX + "id", "1000")
+ .commit();
+
+ table.updateSchema().renameColumn("id", "ID").commit();
+ assertThat(table.properties())
+ .containsEntry(PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX + "ID", "0.05")
+ .containsEntry(PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX + "ID", "1000")
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX + "id")
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX + "id");
+
+ table.updateSchema().deleteColumn("ID").commit();
+ assertThat(table.properties())
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX + "ID")
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_NDV_PREFIX + "ID");
+ }
+
+ @TestTemplate
+ public void testModificationWithParquetDictEncoding() {
+ table
+ .updateProperties()
+ .set(PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX + "id", "false")
+ .commit();
+
+ table.updateSchema().renameColumn("id", "ID").commit();
+ assertThat(table.properties())
+ .containsEntry(PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX + "ID",
"false")
+ .doesNotContainKey(PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX + "id");
+
+ table.updateSchema().deleteColumn("ID").commit();
+ assertThat(table.properties())
+ .doesNotContainKey(PARQUET_DICT_ENCODING_ENABLED_COLUMN_PREFIX + "ID");
+ }
+
+ @TestTemplate
+ public void testRenameStructWithNestedColumnProperties() {
+ table.updateSchema().addColumn("location", LOCATION_TYPE).commit();
+ table
+ .updateProperties()
+ .set(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "location.lat",
"full")
+ .set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "location.lat",
"true")
+ .commit();
+
+ table.updateSchema().renameColumn("location", "loc").commit();
+ assertThat(table.properties())
+ .containsEntry(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"loc.lat", "full")
+ .containsEntry(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "loc.lat",
"true")
+ .doesNotContainKey(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"location.lat")
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"location.lat");
+
+ // a stale metrics property would fail validation of later property updates
+ table.updateProperties().set("key", "value").commit();
+ assertThat(table.properties()).containsEntry("key", "value");
+ }
+
+ @TestTemplate
+ public void testDeleteStructWithNestedColumnProperties() {
+ table.updateSchema().addColumn("location", LOCATION_TYPE).commit();
+ table
+ .updateProperties()
+ .set(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "location.lat",
"full")
+ .set(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX + "location.lat",
"true")
+ .commit();
+
+ table.updateSchema().deleteColumn("location").commit();
+ assertThat(table.properties())
+ .doesNotContainKey(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"location.lat")
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"location.lat");
+
+ table.updateSchema().addColumn("location", LOCATION_TYPE).commit();
+ assertThat(table.properties())
+ .doesNotContainKey(PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX +
"location.lat");
Review Comment:
Done
##########
core/src/main/java/org/apache/iceberg/SchemaUpdate.java:
##########
@@ -506,22 +506,33 @@ private TableMetadata
applyChangesToMetadata(TableMetadata metadata) {
// Transform the metrics if they exist
if (base != null && base.properties() != null) {
- Schema newSchema = newMetadata.schema();
- List<String> deletedColumns =
-
deletes.stream().map(schema::findColumnName).collect(Collectors.toList());
- Map<String, String> renamedColumns =
- updates.keySet().stream()
- .filter(id -> !addedNameToId.containsValue(id)) // remove added
columns
- .filter(id ->
!schema.findColumnName(id).equals(newSchema.findColumnName(id)))
- .collect(Collectors.toMap(schema::findColumnName,
newSchema::findColumnName));
+ Map<Integer, String> newIdToName = newMetadata.schema().idToName();
+ List<String> deletedColumns = Lists.newArrayList();
+ Map<String, String> renamedColumns = Maps.newHashMap();
+ // compare all fields so that nested fields of a deleted or renamed
struct are included
+ schema
+ .idToName()
+ .forEach(
+ (id, name) -> {
+ String newName = newIdToName.get(id);
+ if (newName == null) {
+ deletedColumns.add(name);
+ } else if (!newName.equals(name)) {
+ renamedColumns.put(name, newName);
+ }
+ });
+
if (!deletedColumns.isEmpty() || !renamedColumns.isEmpty()) {
Set<String> columnProperties =
ImmutableSet.of(
TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX,
TableProperties.PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX,
+ TableProperties.PARQUET_BLOOM_FILTER_COLUMN_FPP_PREFIX,
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
