kosiew commented on code in PR #23914:
URL: https://github.com/apache/datafusion/pull/23914#discussion_r3819539198
##########
datafusion/common/src/nested_struct.rs:
##########
@@ -340,6 +352,73 @@ fn mask_array_values(
))
}
+/// Casts Map children by their semantic positions: key at index 0 and value at
+/// index 1. Technical entry field names are taken from the target schema.
+///
+/// Nested Struct fields within keys and values are still matched by name. Key
+/// evolution is restricted by [`validate_map_key_compatibility`] so it cannot
+/// remove identity-bearing fields; sorted Maps require an unchanged key type.
+/// Entries hidden by null Map parents are compacted before casting.
+fn cast_map_column(
+ source_map: &MapArray,
+ target_entries: &FieldRef,
+ target_sorted: bool,
+ cast_options: &CastOptions,
+) -> Result<ArrayRef> {
+ let DataType::Map(source_entries, source_sorted) = source_map.data_type()
else {
+ unreachable!("MapArray data type must be Map")
+ };
+ let (target_key, target_value) = validate_map_compatibility(
+ source_entries,
+ *source_sorted,
+ target_entries,
+ target_sorted,
+ )?;
+
+ let offsets = source_map.value_offsets();
+ let has_unreachable_entries = offsets[0] != 0
+ || offsets[offsets.len() - 1] as usize != source_map.entries().len();
+ let needs_compaction = has_unreachable_entries
+ || source_map.offsets().has_non_empty_nulls(source_map.nulls());
+ let compacted_map = if needs_compaction {
+ Some(compact_map_entries(source_map)?)
+ } else {
+ None
+ };
+ let source_map = compacted_map.as_ref().unwrap_or(source_map);
+
+ let cast_keys = cast_column(source_map.keys(), target_key.data_type(),
cast_options)
+ .map_err(|error| error.context("While casting Map keys"))?;
+ let cast_values =
+ cast_column(source_map.values(), target_value.data_type(),
cast_options)
+ .map_err(|error| error.context("While casting Map values"))?;
+ let Struct(target_fields) = target_entries.data_type() else {
+ unreachable!("validated Map entries must be Struct")
+ };
+ let cast_entries =
+ StructArray::new(target_fields.clone(), vec![cast_keys, cast_values],
None);
Review Comment:
👍
[replace StructArray::new with try_new(...) and prevent panic on invalid
casts](https://github.com/apache/datafusion/pull/23914/commits/9be989b07a8fcaea3992dd8735d9d68b464abe02)
##########
datafusion/common/src/nested_struct.rs:
##########
@@ -340,6 +352,73 @@ fn mask_array_values(
))
}
+/// Casts Map children by their semantic positions: key at index 0 and value at
+/// index 1. Technical entry field names are taken from the target schema.
+///
+/// Nested Struct fields within keys and values are still matched by name. Key
+/// evolution is restricted by [`validate_map_key_compatibility`] so it cannot
+/// remove identity-bearing fields; sorted Maps require an unchanged key type.
+/// Entries hidden by null Map parents are compacted before casting.
+fn cast_map_column(
+ source_map: &MapArray,
+ target_entries: &FieldRef,
+ target_sorted: bool,
+ cast_options: &CastOptions,
+) -> Result<ArrayRef> {
+ let DataType::Map(source_entries, source_sorted) = source_map.data_type()
else {
+ unreachable!("MapArray data type must be Map")
+ };
+ let (target_key, target_value) = validate_map_compatibility(
+ source_entries,
+ *source_sorted,
+ target_entries,
+ target_sorted,
+ )?;
+
+ let offsets = source_map.value_offsets();
+ let has_unreachable_entries = offsets[0] != 0
+ || offsets[offsets.len() - 1] as usize != source_map.entries().len();
+ let needs_compaction = has_unreachable_entries
+ || source_map.offsets().has_non_empty_nulls(source_map.nulls());
+ let compacted_map = if needs_compaction {
+ Some(compact_map_entries(source_map)?)
+ } else {
+ None
+ };
+ let source_map = compacted_map.as_ref().unwrap_or(source_map);
+
+ let cast_keys = cast_column(source_map.keys(), target_key.data_type(),
cast_options)
+ .map_err(|error| error.context("While casting Map keys"))?;
+ let cast_values =
+ cast_column(source_map.values(), target_value.data_type(),
cast_options)
+ .map_err(|error| error.context("While casting Map values"))?;
+ let Struct(target_fields) = target_entries.data_type() else {
+ unreachable!("validated Map entries must be Struct")
+ };
+ let cast_entries =
+ StructArray::new(target_fields.clone(), vec![cast_keys, cast_values],
None);
Review Comment:
👍
[replaced StructArray::new with try_new(...) and prevent panic on invalid
casts](https://github.com/apache/datafusion/pull/23914/commits/9be989b07a8fcaea3992dd8735d9d68b464abe02)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
