kosiew commented on code in PR #23914:
URL: https://github.com/apache/datafusion/pull/23914#discussion_r3819557343
##########
datafusion/common/src/nested_struct.rs:
##########
@@ -340,6 +352,73 @@ fn mask_array_values(
))
}
+/// Casts Map children by their semantic positions: key at index 0 and value at
+/// index 1. Technical entry field names are taken from the target schema.
+///
+/// Nested Struct fields within keys and values are still matched by name. Key
+/// evolution is restricted by [`validate_map_key_compatibility`] so it cannot
+/// remove identity-bearing fields; sorted Maps require an unchanged key type.
+/// Entries hidden by null Map parents are compacted before casting.
+fn cast_map_column(
+ source_map: &MapArray,
+ target_entries: &FieldRef,
+ target_sorted: bool,
+ cast_options: &CastOptions,
+) -> Result<ArrayRef> {
+ let DataType::Map(source_entries, source_sorted) = source_map.data_type()
else {
+ unreachable!("MapArray data type must be Map")
+ };
+ let (target_key, target_value) = validate_map_compatibility(
+ source_entries,
+ *source_sorted,
+ target_entries,
+ target_sorted,
+ )?;
+
+ let offsets = source_map.value_offsets();
+ let has_unreachable_entries = offsets[0] != 0
+ || offsets[offsets.len() - 1] as usize != source_map.entries().len();
+ let needs_compaction = has_unreachable_entries
+ || source_map.offsets().has_non_empty_nulls(source_map.nulls());
+ let compacted_map = if needs_compaction {
+ Some(compact_map_entries(source_map)?)
+ } else {
+ None
+ };
+ let source_map = compacted_map.as_ref().unwrap_or(source_map);
+
+ let cast_keys = cast_column(source_map.keys(), target_key.data_type(),
cast_options)
+ .map_err(|error| error.context("While casting Map keys"))?;
+ let cast_values =
+ cast_column(source_map.values(), target_value.data_type(),
cast_options)
+ .map_err(|error| error.context("While casting Map values"))?;
+ let Struct(target_fields) = target_entries.data_type() else {
+ unreachable!("validated Map entries must be Struct")
+ };
+ let cast_entries =
+ StructArray::new(target_fields.clone(), vec![cast_keys, cast_values],
None);
+
+ Ok(Arc::new(MapArray::try_new(
+ Arc::clone(target_entries),
+ source_map.offsets().clone(),
+ cast_entries,
+ source_map.nulls().cloned(),
+ target_sorted,
+ )?))
+}
+
+/// Returns an equivalent MapArray whose entries contain only values reachable
+/// from visible Map rows.
+///
+/// Arrow Map arrays can contain unreachable entries after slicing, or entries
+/// hidden behind null parent rows. An identity `take` rebuilds the Map through
+/// Arrow's selection kernel, normalizing offsets and dropping those
unreachable
+/// child entries before recursive key/value casts are applied.
+fn compact_map_entries(map: &MapArray) -> Result<MapArray> {
+ let indices = UInt64Array::from_iter_values(0..map.len() as u64);
+ Ok(take(map, &indices, None)?.as_map().clone())
+}
Review Comment:
[Added Map-vs-List-family rationale
comment](https://github.com/apache/datafusion/pull/23914/commits/7a70827fcbfc031b7972d4e4ee473d031f5a9168)
Created tracking issue #24506
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
