ebyhr commented on code in PR #3630:
URL: https://github.com/apache/iceberg-python/pull/3630#discussion_r3562399785
##########
pyiceberg/table/sorting.py:
##########
@@ -107,11 +108,31 @@ def map_source_ids_onto_source_id(cls, data: Any) -> Any:
if len(source_ids) == 0:
raise ValueError("Empty source-ids is not allowed")
if len(source_ids) > 1:
- raise ValueError("Multi argument transforms are not
yet supported")
+ if data.get("transform") is None:
+ raise ValueError("Transform is required for a
multi-argument field")
+ # Multi-argument transforms cannot be evaluated; per
the spec, v3 readers
+ # must read tables with such transforms, ignoring them
+ data["transform"] =
UnknownTransform(transform=str(data["transform"]))
+ else:
+ data.pop("source-ids", None)
data["source-id"] = source_ids[0]
return data
+ @model_serializer(mode="wrap")
+ def _serialize_source_ids(self, handler: Any) -> Any:
+ serialized = handler(self)
+ # Per the spec, single-argument transforms write only source-id and
+ # multi-argument transforms write only source-ids
+ if self.source_ids is not None and len(self.source_ids) > 1:
+ serialized.pop("source-id", None)
+ serialized.pop("source_id", None)
+ else:
+ serialized.pop("source-ids", None)
+ serialized.pop("source_ids", None)
Review Comment:
Why does this block contain underscore fields that the specification doesn't
include? We should delete them in my opinion. Same for `partitioning.py`.
##########
pyiceberg/partitioning.py:
##########
@@ -115,13 +117,36 @@ def map_source_ids_onto_source_id(cls, data: Any) -> Any:
if len(source_ids) == 0:
raise ValueError("Empty source-ids is not allowed")
if len(source_ids) > 1:
- raise ValueError("Multi argument transforms are not
yet supported")
+ if data.get("transform") is None:
+ raise ValueError("Transform is required for a
multi-argument field")
+ # Multi-argument transforms cannot be evaluated; per
the spec, v3 readers
+ # must read tables with such transforms, ignoring them
+ data["transform"] =
UnknownTransform(transform=str(data["transform"]))
+ else:
+ data.pop("source-ids", None)
data["source-id"] = source_ids[0]
return data
+ @model_serializer(mode="wrap")
+ def _serialize_source_ids(self, handler: Any) -> Any:
+ serialized = handler(self)
+ # Per the spec, single-argument transforms write only source-id and
+ # multi-argument transforms write only source-ids
+ if self.source_ids is not None and len(self.source_ids) > 1:
+ serialized.pop("source-id", None)
+ serialized.pop("source_id", None)
+ else:
+ serialized.pop("source-ids", None)
+ serialized.pop("source_ids", None)
+ return serialized
+
def __str__(self) -> str:
"""Return the string representation of the PartitionField class."""
- return f"{self.field_id}: {self.name}:
{self.transform}({self.source_id})"
+ if self.source_ids is not None and len(self.source_ids) > 1:
+ sources = ", ".join(str(s) for s in self.source_ids)
+ else:
+ sources = str(self.source_id)
+ return f"{self.field_id}: {self.name}: {self.transform}({sources})"
Review Comment:
Could you also update `SortField.__str__`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
