emkornfield commented on code in PR #1851: URL: https://github.com/apache/iceberg-rust/pull/1851#discussion_r2530482656
########## crates/iceberg/src/spec/avro_util.rs: ########## @@ -0,0 +1,280 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Utilities for working with Apache Avro in Iceberg. + +use apache_avro::Codec; +use log::warn; + +/// Settings for compression codec and level. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompressionSettings { + /// The compression codec name (e.g., "gzip", "zstd", "deflate", "none") + pub codec: String, + /// The compression level + pub level: u8, +} + +impl CompressionSettings { + /// Create a new CompressionSettings with the specified codec and level. + pub fn new(codec: String, level: u8) -> Self { + Self { codec, level } + } + + /// Convert to apache_avro::Codec using the codec_from_str helper function. + pub(crate) fn to_codec(&self) -> Codec { + codec_from_str(Some(&self.codec), self.level) + } +} + +impl Default for CompressionSettings { + fn default() -> Self { + use crate::spec::TableProperties; + Self { + codec: TableProperties::PROPERTY_AVRO_COMPRESSION_CODEC_DEFAULT.to_string(), + level: TableProperties::PROPERTY_AVRO_COMPRESSION_LEVEL_DEFAULT, + } + } +} + +/// Convert codec name and level to apache_avro::Codec. +/// Returns Codec::Null for unknown or unsupported codecs. +/// +/// # Arguments +/// +/// * `codec` - The name of the compression codec (e.g., "gzip", "zstd", "deflate", "none") +/// * `level` - The compression level. For deflate/gzip: +/// - 0: NoCompression +/// - 1: BestSpeed +/// - 9: BestCompression +/// - 10: UberCompression +/// - Other values: DefaultLevel (6) +/// +/// # Supported Codecs +/// +/// - `gzip` or `deflate`: Uses Deflate compression with specified level +/// - `zstd`: Uses Zstandard compression (level clamped to valid zstd range) +/// - `none` or `None`: No compression +/// - Any other value: Defaults to no compression (Codec::Null) +/// +/// # Compression Levels +/// +/// The compression level mapping is based on miniz_oxide's CompressionLevel enum: +/// - Level 0: No compression +/// - Level 1: Best speed (fastest) +/// - Level 9: Best compression (slower, better compression) +/// - Level 10: Uber compression (slowest, best compression) +/// - Other: Default level (balanced speed/compression) +pub(crate) fn codec_from_str(codec: Option<&str>, level: u8) -> Codec { + use apache_avro::{DeflateSettings, ZstandardSettings}; + + match codec { + Some("gzip") | Some("deflate") => { + // Map compression level to miniz_oxide::deflate::CompressionLevel + // Reference: https://docs.rs/miniz_oxide/latest/miniz_oxide/deflate/enum.CompressionLevel.html + use miniz_oxide::deflate::CompressionLevel; + + let compression_level = match level { + 0 => CompressionLevel::NoCompression, + 1 => CompressionLevel::BestSpeed, + 9 => CompressionLevel::BestCompression, + 10 => CompressionLevel::UberCompression, + _ => CompressionLevel::DefaultLevel, + }; + + Codec::Deflate(DeflateSettings::new(compression_level)) + } + Some("zstd") => { + // Zstandard supports levels 0-22, clamp to valid range + let zstd_level = level.min(22); Review Comment: Yes, it does look like the default for ZSTD is 1, not sure on why the divergence here though, happy to change it we want to match exactly with Java, but I agree it probably isn't critical. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
