liurenjie1024 commented on code in PR #285: URL: https://github.com/apache/iceberg-rust/pull/285#discussion_r1531431294
########## crates/catalog/hms/src/error.rs: ########## @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::anyhow; +use iceberg::{Error, ErrorKind}; +use std::fmt::Debug; +use std::io; + +/// Format a thrift error into iceberg error. +pub fn from_thrift_error<T>(error: volo_thrift::error::ResponseError<T>) -> Error +where + T: Debug, +{ + Error::new( + ErrorKind::Unexpected, + "operation failed for hitting thrift error".to_string(), Review Comment: ```suggestion "Operation failed for hitting thrift error".to_string(), ``` ########## crates/catalog/hms/src/catalog.rs: ########## @@ -287,31 +307,203 @@ impl Catalog for HmsCatalog { Ok(tables) } + /// Creates a new table within a specified namespace using the provided + /// table creation settings. + /// + /// # Returns + /// A `Result` wrapping a `Table` object representing the newly created + /// table. + /// + /// # Errors + /// This function may return an error in several cases, including invalid + /// namespace identifiers, failure to determine a default storage location, + /// issues generating or writing table metadata, and errors communicating + /// with the Hive Metastore. async fn create_table( &self, - _namespace: &NamespaceIdent, - _creation: TableCreation, + namespace: &NamespaceIdent, + creation: TableCreation, ) -> Result<Table> { - todo!() + let db_name = validate_namespace(namespace)?; + let table_name = creation.name.clone(); + + let location = match &creation.location { + Some(location) => location.clone(), + None => { + let ns = self.get_namespace(namespace).await?; + get_default_table_location(&ns, &table_name)? + } + }; + + let metadata = TableMetadataBuilder::from_table_creation(creation)?.build()?; + let metadata_location = create_metadata_location(&location, 0)?; + + let file_io = FileIO::from_path(&metadata_location)? + .with_props(&self.config.props) + .build()?; Review Comment: I think it's better practice to keep `FileIO` in catalog instance? cc @Xuanwo ########## crates/catalog/hms/src/schema.rs: ########## @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use hive_metastore::FieldSchema; +use iceberg::spec::{visit_schema, NestedFieldRef, PrimitiveType, Schema, SchemaVisitor}; +use iceberg::Result; + +type HiveSchema = Vec<FieldSchema>; + +#[derive(Debug, Default)] +pub(crate) struct HiveSchemaBuilder { + schema: HiveSchema, + context: Vec<NestedFieldRef>, +} + +impl HiveSchemaBuilder { + /// Creates a new `HiveSchemaBuilder` from iceberg `Schema` + pub fn from_iceberg(schema: &Schema) -> Result<HiveSchemaBuilder> { + let mut builder = Self::default(); + visit_schema(schema, &mut builder)?; + Ok(builder) + } + + /// Returns the newly converted `HiveSchema` + pub fn build(self) -> HiveSchema { + self.schema + } + + /// Check if is in `StructType` while traversing schema + fn is_inside_struct(&self) -> bool { + !self.context.is_empty() + } +} + +impl SchemaVisitor for HiveSchemaBuilder { + type T = String; + + fn schema( + &mut self, + _schema: &iceberg::spec::Schema, + value: Self::T, Review Comment: Though this is correct, it's better to replace these with actual type, e.g. `String` ########## crates/catalog/hms/src/error.rs: ########## @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::anyhow; +use iceberg::{Error, ErrorKind}; +use std::fmt::Debug; +use std::io; + +/// Format a thrift error into iceberg error. +pub fn from_thrift_error<T>(error: volo_thrift::error::ResponseError<T>) -> Error +where + T: Debug, +{ + Error::new( + ErrorKind::Unexpected, + "operation failed for hitting thrift error".to_string(), + ) + .with_source(anyhow!("thrift error: {:?}", error)) +} + +/// Format an io error into iceberg error. +pub fn from_io_error(error: io::Error) -> Error { + Error::new( + ErrorKind::Unexpected, + "operation failed for hitting io error".to_string(), Review Comment: ```suggestion "Operation failed for hitting io error".to_string(), ``` ########## crates/catalog/hms/src/schema.rs: ########## @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use hive_metastore::FieldSchema; +use iceberg::spec::{visit_schema, NestedFieldRef, PrimitiveType, Schema, SchemaVisitor}; +use iceberg::Result; + +type HiveSchema = Vec<FieldSchema>; + +#[derive(Debug, Default)] +pub(crate) struct HiveSchemaBuilder { + schema: HiveSchema, + context: Vec<NestedFieldRef>, +} + +impl HiveSchemaBuilder { + /// Creates a new `HiveSchemaBuilder` from iceberg `Schema` + pub fn from_iceberg(schema: &Schema) -> Result<HiveSchemaBuilder> { + let mut builder = Self::default(); + visit_schema(schema, &mut builder)?; + Ok(builder) + } + + /// Returns the newly converted `HiveSchema` + pub fn build(self) -> HiveSchema { + self.schema + } + + /// Check if is in `StructType` while traversing schema + fn is_inside_struct(&self) -> bool { + !self.context.is_empty() + } +} + +impl SchemaVisitor for HiveSchemaBuilder { + type T = String; + + fn schema( + &mut self, + _schema: &iceberg::spec::Schema, + value: Self::T, + ) -> iceberg::Result<Self::T> { + Ok(value) + } + + fn before_struct_field( + &mut self, + field: &iceberg::spec::NestedFieldRef, + ) -> iceberg::Result<()> { + self.context.push(field.clone()); + Ok(()) + } + + fn r#struct( + &mut self, + r#_struct: &iceberg::spec::StructType, + results: Vec<Self::T>, + ) -> iceberg::Result<Self::T> { + Ok(format!("struct<{}>", results.join(", "))) + } + + fn after_struct_field( + &mut self, + _field: &iceberg::spec::NestedFieldRef, + ) -> iceberg::Result<()> { + self.context.pop(); + Ok(()) + } + + fn field( + &mut self, + field: &iceberg::spec::NestedFieldRef, + value: Self::T, + ) -> iceberg::Result<Self::T> { + if self.is_inside_struct() { + return Ok(format!("{}:{}", field.name, value)); + } + + self.schema.push(FieldSchema { + name: Some(field.name.clone().into()), + r#type: Some(value.clone().into()), + comment: field.doc.clone().map(|doc| doc.into()), + }); + + Ok(value) + } + + fn list( + &mut self, + _list: &iceberg::spec::ListType, + value: Self::T, + ) -> iceberg::Result<Self::T> { + Ok(format!("array<{}>", value)) + } + + fn map( + &mut self, + _map: &iceberg::spec::MapType, + key_value: Self::T, + value: Self::T, + ) -> iceberg::Result<Self::T> { + Ok(format!("map<{},{}>", key_value, value)) + } + + fn primitive(&mut self, p: &iceberg::spec::PrimitiveType) -> iceberg::Result<Self::T> { + let hive_type = match p { + PrimitiveType::Boolean => "boolean".to_string(), + PrimitiveType::Int => "int".to_string(), + PrimitiveType::Long => "bigint".to_string(), + PrimitiveType::Float => "float".to_string(), + PrimitiveType::Double => "double".to_string(), + PrimitiveType::Date => "date".to_string(), + PrimitiveType::Timestamp | PrimitiveType::Timestamptz => "timestamp".to_string(), Review Comment: SHould we exclude `Timestampz`? cc @Fokko ########## crates/catalog/hms/src/schema.rs: ########## @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use hive_metastore::FieldSchema; +use iceberg::spec::{visit_schema, NestedFieldRef, PrimitiveType, Schema, SchemaVisitor}; +use iceberg::Result; + +type HiveSchema = Vec<FieldSchema>; + +#[derive(Debug, Default)] +pub(crate) struct HiveSchemaBuilder { + schema: HiveSchema, + context: Vec<NestedFieldRef>, +} + +impl HiveSchemaBuilder { + /// Creates a new `HiveSchemaBuilder` from iceberg `Schema` + pub fn from_iceberg(schema: &Schema) -> Result<HiveSchemaBuilder> { + let mut builder = Self::default(); + visit_schema(schema, &mut builder)?; + Ok(builder) + } + + /// Returns the newly converted `HiveSchema` + pub fn build(self) -> HiveSchema { + self.schema + } + + /// Check if is in `StructType` while traversing schema + fn is_inside_struct(&self) -> bool { + !self.context.is_empty() + } +} + +impl SchemaVisitor for HiveSchemaBuilder { + type T = String; + + fn schema( + &mut self, + _schema: &iceberg::spec::Schema, + value: Self::T, + ) -> iceberg::Result<Self::T> { + Ok(value) + } + + fn before_struct_field( + &mut self, + field: &iceberg::spec::NestedFieldRef, + ) -> iceberg::Result<()> { + self.context.push(field.clone()); + Ok(()) + } + + fn r#struct( + &mut self, + r#_struct: &iceberg::spec::StructType, + results: Vec<Self::T>, + ) -> iceberg::Result<Self::T> { + Ok(format!("struct<{}>", results.join(", "))) + } + + fn after_struct_field( + &mut self, + _field: &iceberg::spec::NestedFieldRef, + ) -> iceberg::Result<()> { + self.context.pop(); + Ok(()) + } + + fn field( + &mut self, + field: &iceberg::spec::NestedFieldRef, + value: Self::T, + ) -> iceberg::Result<Self::T> { + if self.is_inside_struct() { + return Ok(format!("{}:{}", field.name, value)); + } + + self.schema.push(FieldSchema { + name: Some(field.name.clone().into()), + r#type: Some(value.clone().into()), + comment: field.doc.clone().map(|doc| doc.into()), + }); + + Ok(value) Review Comment: See [this line](https://github.com/apache/iceberg-python/blob/afdfa351119090f09d38ef72857d6303e691f5ad/pyiceberg/catalog/hive.py#L157) ########## crates/catalog/hms/src/utils.rs: ########## @@ -183,6 +227,65 @@ pub(crate) fn validate_namespace(namespace: &NamespaceIdent) -> Result<String> { Ok(name) } +/// Get default table location from `Namespace` properties +pub(crate) fn get_default_table_location( + namespace: &Namespace, + table_name: impl AsRef<str> + Display, +) -> Result<String> { + let properties = namespace.properties(); + properties + .get(LOCATION) + .or_else(|| properties.get(WAREHOUSE_LOCATION)) + .map(|location| format!("{}/{}", location, table_name)) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + "No default path is set, please specify a location when creating a table", + ) + }) +} + +/// Create metadata location from `location` and `version` +pub(crate) fn create_metadata_location( + location: impl AsRef<str> + Display, Review Comment: Ditto. ########## crates/catalog/hms/src/utils.rs: ########## @@ -183,6 +227,65 @@ pub(crate) fn validate_namespace(namespace: &NamespaceIdent) -> Result<String> { Ok(name) } +/// Get default table location from `Namespace` properties +pub(crate) fn get_default_table_location( + namespace: &Namespace, + table_name: impl AsRef<str> + Display, Review Comment: ```suggestion table_name: impl AsRef<str>, ``` You can display it with `table_name.as_str()` ########## crates/catalog/hms/src/catalog.rs: ########## @@ -47,6 +56,8 @@ pub enum HmsThriftTransport { pub struct HmsCatalogConfig { address: String, thrift_transport: HmsThriftTransport, + #[builder(default, setter(prefix = "with_"))] Review Comment: To make the api consistent, I would suggest not adding this `with_` prefix. ########## crates/catalog/hms/src/schema.rs: ########## @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use hive_metastore::FieldSchema; +use iceberg::spec::{visit_schema, NestedFieldRef, PrimitiveType, Schema, SchemaVisitor}; +use iceberg::Result; + +type HiveSchema = Vec<FieldSchema>; + +#[derive(Debug, Default)] +pub(crate) struct HiveSchemaBuilder { + schema: HiveSchema, + context: Vec<NestedFieldRef>, +} + +impl HiveSchemaBuilder { + /// Creates a new `HiveSchemaBuilder` from iceberg `Schema` + pub fn from_iceberg(schema: &Schema) -> Result<HiveSchemaBuilder> { + let mut builder = Self::default(); + visit_schema(schema, &mut builder)?; + Ok(builder) + } + + /// Returns the newly converted `HiveSchema` + pub fn build(self) -> HiveSchema { + self.schema + } + + /// Check if is in `StructType` while traversing schema + fn is_inside_struct(&self) -> bool { + !self.context.is_empty() + } +} + +impl SchemaVisitor for HiveSchemaBuilder { + type T = String; + + fn schema( + &mut self, + _schema: &iceberg::spec::Schema, + value: Self::T, + ) -> iceberg::Result<Self::T> { + Ok(value) + } + + fn before_struct_field( + &mut self, + field: &iceberg::spec::NestedFieldRef, + ) -> iceberg::Result<()> { + self.context.push(field.clone()); + Ok(()) + } + + fn r#struct( + &mut self, + r#_struct: &iceberg::spec::StructType, + results: Vec<Self::T>, + ) -> iceberg::Result<Self::T> { + Ok(format!("struct<{}>", results.join(", "))) + } + + fn after_struct_field( + &mut self, + _field: &iceberg::spec::NestedFieldRef, + ) -> iceberg::Result<()> { + self.context.pop(); + Ok(()) + } + + fn field( + &mut self, + field: &iceberg::spec::NestedFieldRef, + value: Self::T, + ) -> iceberg::Result<Self::T> { + if self.is_inside_struct() { + return Ok(format!("{}:{}", field.name, value)); + } + + self.schema.push(FieldSchema { + name: Some(field.name.clone().into()), + r#type: Some(value.clone().into()), + comment: field.doc.clone().map(|doc| doc.into()), + }); + + Ok(value) + } + + fn list( + &mut self, + _list: &iceberg::spec::ListType, + value: Self::T, + ) -> iceberg::Result<Self::T> { + Ok(format!("array<{}>", value)) + } + + fn map( + &mut self, + _map: &iceberg::spec::MapType, + key_value: Self::T, + value: Self::T, + ) -> iceberg::Result<Self::T> { + Ok(format!("map<{},{}>", key_value, value)) + } + + fn primitive(&mut self, p: &iceberg::spec::PrimitiveType) -> iceberg::Result<Self::T> { + let hive_type = match p { + PrimitiveType::Boolean => "boolean".to_string(), + PrimitiveType::Int => "int".to_string(), + PrimitiveType::Long => "bigint".to_string(), + PrimitiveType::Float => "float".to_string(), + PrimitiveType::Double => "double".to_string(), + PrimitiveType::Date => "date".to_string(), + PrimitiveType::Timestamp | PrimitiveType::Timestamptz => "timestamp".to_string(), + PrimitiveType::Time | PrimitiveType::String | PrimitiveType::Uuid => { + "string".to_string() + } + PrimitiveType::Binary | PrimitiveType::Fixed(_) => "binary".to_string(), + PrimitiveType::Decimal { precision, scale } => { + format!("decimal({},{})", precision, scale) + } + }; + + Ok(hive_type) + } +} + +#[cfg(test)] +mod tests { + use iceberg::{ + spec::{ListType, MapType, NestedField, Schema, StructType, Type}, + Result, + }; + + use super::*; + + #[test] + fn test_schema_with_nested_maps() -> Result<()> { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![NestedField::required( + 1, + "quux", + Type::Map(MapType { + key_field: NestedField::map_key_element( + 2, + Type::Primitive(PrimitiveType::String), + ) + .into(), + value_field: NestedField::map_value_element( + 3, + Type::Map(MapType { + key_field: NestedField::map_key_element( + 4, + Type::Primitive(PrimitiveType::String), + ) + .into(), + value_field: NestedField::map_value_element( + 5, + Type::Primitive(PrimitiveType::Int), + true, + ) + .into(), + }), + true, + ) + .into(), + }), + ) + .into()]) + .build()?; + + let result = HiveSchemaBuilder::from_iceberg(&schema)?.build(); + + let expected = vec![FieldSchema { + name: Some("quux".into()), + r#type: Some("map<string,map<string,int>>".into()), + comment: None, + }]; + + assert_eq!(result, expected); + + Ok(()) + } + + #[test] + fn test_schema_with_struct_inside_list() -> Result<()> { Review Comment: Thanks for the tests! ########## crates/catalog/hms/src/catalog.rs: ########## @@ -287,31 +307,203 @@ impl Catalog for HmsCatalog { Ok(tables) } + /// Creates a new table within a specified namespace using the provided + /// table creation settings. + /// + /// # Returns + /// A `Result` wrapping a `Table` object representing the newly created + /// table. + /// + /// # Errors + /// This function may return an error in several cases, including invalid + /// namespace identifiers, failure to determine a default storage location, + /// issues generating or writing table metadata, and errors communicating + /// with the Hive Metastore. async fn create_table( &self, - _namespace: &NamespaceIdent, - _creation: TableCreation, + namespace: &NamespaceIdent, + creation: TableCreation, ) -> Result<Table> { - todo!() + let db_name = validate_namespace(namespace)?; + let table_name = creation.name.clone(); + + let location = match &creation.location { + Some(location) => location.clone(), + None => { + let ns = self.get_namespace(namespace).await?; + get_default_table_location(&ns, &table_name)? + } + }; + + let metadata = TableMetadataBuilder::from_table_creation(creation)?.build()?; + let metadata_location = create_metadata_location(&location, 0)?; + + let file_io = FileIO::from_path(&metadata_location)? + .with_props(&self.config.props) + .build()?; + let mut file = file_io.new_output(&metadata_location)?.writer().await?; + file.write_all(&serde_json::to_vec(&metadata)?).await?; + file.shutdown().await?; Review Comment: Though it's correct, I prefer to use `flush`, as we can keep underlying connection? cc @Xuanwo ########## crates/catalog/hms/src/schema.rs: ########## @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use hive_metastore::FieldSchema; +use iceberg::spec::{visit_schema, NestedFieldRef, PrimitiveType, Schema, SchemaVisitor}; +use iceberg::Result; + +type HiveSchema = Vec<FieldSchema>; + +#[derive(Debug, Default)] +pub(crate) struct HiveSchemaBuilder { + schema: HiveSchema, + context: Vec<NestedFieldRef>, +} + +impl HiveSchemaBuilder { + /// Creates a new `HiveSchemaBuilder` from iceberg `Schema` + pub fn from_iceberg(schema: &Schema) -> Result<HiveSchemaBuilder> { + let mut builder = Self::default(); + visit_schema(schema, &mut builder)?; + Ok(builder) + } + + /// Returns the newly converted `HiveSchema` + pub fn build(self) -> HiveSchema { + self.schema + } + + /// Check if is in `StructType` while traversing schema + fn is_inside_struct(&self) -> bool { + !self.context.is_empty() + } +} + +impl SchemaVisitor for HiveSchemaBuilder { + type T = String; + + fn schema( + &mut self, + _schema: &iceberg::spec::Schema, + value: Self::T, + ) -> iceberg::Result<Self::T> { + Ok(value) + } + + fn before_struct_field( + &mut self, + field: &iceberg::spec::NestedFieldRef, + ) -> iceberg::Result<()> { + self.context.push(field.clone()); + Ok(()) + } + + fn r#struct( + &mut self, + r#_struct: &iceberg::spec::StructType, + results: Vec<Self::T>, + ) -> iceberg::Result<Self::T> { + Ok(format!("struct<{}>", results.join(", "))) + } + + fn after_struct_field( + &mut self, + _field: &iceberg::spec::NestedFieldRef, + ) -> iceberg::Result<()> { + self.context.pop(); + Ok(()) + } + + fn field( + &mut self, + field: &iceberg::spec::NestedFieldRef, + value: Self::T, + ) -> iceberg::Result<Self::T> { + if self.is_inside_struct() { + return Ok(format!("{}:{}", field.name, value)); + } + + self.schema.push(FieldSchema { + name: Some(field.name.clone().into()), + r#type: Some(value.clone().into()), + comment: field.doc.clone().map(|doc| doc.into()), + }); + + Ok(value) + } + + fn list( + &mut self, + _list: &iceberg::spec::ListType, + value: Self::T, + ) -> iceberg::Result<Self::T> { + Ok(format!("array<{}>", value)) + } + + fn map( + &mut self, + _map: &iceberg::spec::MapType, + key_value: Self::T, + value: Self::T, + ) -> iceberg::Result<Self::T> { + Ok(format!("map<{},{}>", key_value, value)) + } + + fn primitive(&mut self, p: &iceberg::spec::PrimitiveType) -> iceberg::Result<Self::T> { + let hive_type = match p { + PrimitiveType::Boolean => "boolean".to_string(), + PrimitiveType::Int => "int".to_string(), + PrimitiveType::Long => "bigint".to_string(), + PrimitiveType::Float => "float".to_string(), + PrimitiveType::Double => "double".to_string(), + PrimitiveType::Date => "date".to_string(), + PrimitiveType::Timestamp | PrimitiveType::Timestamptz => "timestamp".to_string(), + PrimitiveType::Time | PrimitiveType::String | PrimitiveType::Uuid => { + "string".to_string() + } + PrimitiveType::Binary | PrimitiveType::Fixed(_) => "binary".to_string(), + PrimitiveType::Decimal { precision, scale } => { + format!("decimal({},{})", precision, scale) + } + }; + + Ok(hive_type) + } +} + +#[cfg(test)] +mod tests { + use iceberg::{ + spec::{ListType, MapType, NestedField, Schema, StructType, Type}, + Result, + }; + + use super::*; + + #[test] + fn test_schema_with_nested_maps() -> Result<()> { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![NestedField::required( + 1, + "quux", + Type::Map(MapType { + key_field: NestedField::map_key_element( + 2, + Type::Primitive(PrimitiveType::String), + ) + .into(), + value_field: NestedField::map_value_element( + 3, + Type::Map(MapType { + key_field: NestedField::map_key_element( + 4, + Type::Primitive(PrimitiveType::String), + ) + .into(), + value_field: NestedField::map_value_element( + 5, + Type::Primitive(PrimitiveType::Int), + true, + ) + .into(), + }), + true, + ) + .into(), + }), + ) + .into()]) + .build()?; + + let result = HiveSchemaBuilder::from_iceberg(&schema)?.build(); + + let expected = vec![FieldSchema { + name: Some("quux".into()), + r#type: Some("map<string,map<string,int>>".into()), + comment: None, + }]; + + assert_eq!(result, expected); + + Ok(()) + } + + #[test] + fn test_schema_with_struct_inside_list() -> Result<()> { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![NestedField::required( + 1, + "location", + Type::List(ListType { + element_field: NestedField::list_element( + 2, + Type::Struct(StructType::new(vec![ + NestedField::optional( + 3, + "latitude", + Type::Primitive(PrimitiveType::Float), + ) + .into(), + NestedField::optional( + 4, + "longitude", + Type::Primitive(PrimitiveType::Float), + ) + .into(), + ])), + true, + ) + .into(), + }), + ) + .into()]) + .build()?; + + let result = HiveSchemaBuilder::from_iceberg(&schema)?.build(); + + let expected = vec![FieldSchema { + name: Some("location".into()), + r#type: Some("array<struct<latitude:float, longitude:float>>".into()), + comment: None, + }]; + + assert_eq!(result, expected); + + Ok(()) + } + + #[test] + fn test_schema_with_structs() -> Result<()> { + let schema = Schema::builder() + .with_schema_id(1) + .with_fields(vec![NestedField::required( + 1, + "person", + Type::Struct(StructType::new(vec![ + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(3, "age", Type::Primitive(PrimitiveType::Int)).into(), + ])), + ) + .into()]) + .build()?; + + let result = HiveSchemaBuilder::from_iceberg(&schema)?.build(); + + let expected = vec![FieldSchema { + name: Some("person".into()), + r#type: Some("struct<name:string, age:int>".into()), + comment: None, + }]; + + assert_eq!(result, expected); + + Ok(()) + } + + #[test] + fn test_schema_with_simple_fields() -> Result<()> { Review Comment: Could we cover all primitive types here? ########## crates/catalog/hms/src/utils.rs: ########## @@ -157,6 +150,57 @@ pub(crate) fn convert_to_database( Ok(db) } +pub(crate) fn convert_to_hive_table( + db_name: String, + schema: &Schema, + table_name: String, + location: String, + metadata_location: String, + properties: &HashMap<String, String>, +) -> Result<hive_metastore::Table> { + let serde_info = SerDeInfo { + serialization_lib: Some(SERIALIZATION_LIB.into()), + ..Default::default() + }; + + let hive_schema = HiveSchemaBuilder::from_iceberg(schema)?.build(); + + let storage_descriptor = StorageDescriptor { + location: Some(location.into()), + cols: Some(hive_schema), + input_format: Some(INPUT_FORMAT.into()), + output_format: Some(OUTPUT_FORMAT.into()), + serde_info: Some(serde_info), + ..Default::default() + }; + + let parameters = AHashMap::from([ + (FastStr::from(EXTERNAL), FastStr::from("TRUE")), + (FastStr::from(TABLE_TYPE), FastStr::from("ICEBERG")), + ( + FastStr::from(METADATA_LOCATION), + FastStr::from(metadata_location), + ), + ]); + + let current_time_ms = get_current_time()?; + let owner = properties + .get("owner") Review Comment: Make this a constant? ########## crates/catalog/hms/src/utils.rs: ########## @@ -183,6 +227,65 @@ pub(crate) fn validate_namespace(namespace: &NamespaceIdent) -> Result<String> { Ok(name) } +/// Get default table location from `Namespace` properties +pub(crate) fn get_default_table_location( + namespace: &Namespace, + table_name: impl AsRef<str> + Display, +) -> Result<String> { + let properties = namespace.properties(); + properties + .get(LOCATION) + .or_else(|| properties.get(WAREHOUSE_LOCATION)) Review Comment: This part is incorrect, pyiceberg uses property from catalog, see: https://github.com/apache/iceberg-python/blob/44948cd9bab78b509b6befc960b5f248b37f53fe/pyiceberg/catalog/__init__.py#L565 ########## crates/catalog/hms/src/schema.rs: ########## @@ -0,0 +1,300 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use hive_metastore::FieldSchema; +use iceberg::spec::{visit_schema, NestedFieldRef, PrimitiveType, Schema, SchemaVisitor}; +use iceberg::Result; + +type HiveSchema = Vec<FieldSchema>; + +#[derive(Debug, Default)] +pub(crate) struct HiveSchemaBuilder { + schema: HiveSchema, + context: Vec<NestedFieldRef>, +} + +impl HiveSchemaBuilder { + /// Creates a new `HiveSchemaBuilder` from iceberg `Schema` + pub fn from_iceberg(schema: &Schema) -> Result<HiveSchemaBuilder> { + let mut builder = Self::default(); + visit_schema(schema, &mut builder)?; + Ok(builder) + } + + /// Returns the newly converted `HiveSchema` + pub fn build(self) -> HiveSchema { + self.schema + } + + /// Check if is in `StructType` while traversing schema + fn is_inside_struct(&self) -> bool { + !self.context.is_empty() + } +} + +impl SchemaVisitor for HiveSchemaBuilder { + type T = String; + + fn schema( + &mut self, + _schema: &iceberg::spec::Schema, + value: Self::T, + ) -> iceberg::Result<Self::T> { + Ok(value) + } + + fn before_struct_field( + &mut self, + field: &iceberg::spec::NestedFieldRef, + ) -> iceberg::Result<()> { + self.context.push(field.clone()); + Ok(()) + } + + fn r#struct( + &mut self, + r#_struct: &iceberg::spec::StructType, + results: Vec<Self::T>, + ) -> iceberg::Result<Self::T> { + Ok(format!("struct<{}>", results.join(", "))) + } + + fn after_struct_field( + &mut self, + _field: &iceberg::spec::NestedFieldRef, + ) -> iceberg::Result<()> { + self.context.pop(); + Ok(()) + } + + fn field( + &mut self, + field: &iceberg::spec::NestedFieldRef, + value: Self::T, + ) -> iceberg::Result<Self::T> { + if self.is_inside_struct() { + return Ok(format!("{}:{}", field.name, value)); + } + + self.schema.push(FieldSchema { + name: Some(field.name.clone().into()), + r#type: Some(value.clone().into()), + comment: field.doc.clone().map(|doc| doc.into()), + }); + + Ok(value) Review Comment: I think we can always returns `Ok(format!("{}:{}", field.name, value))` here, since hive schema only requires field name in outmost layer. We can add outmost field names by iterating the struct fields in schema. ########## crates/catalog/hms/tests/hms_catalog_test.rs: ########## @@ -56,19 +63,182 @@ async fn set_test_fixture(func: &str) -> TestFixture { } } + let props = HashMap::from([ + ( + S3_ENDPOINT.to_string(), + format!("http://{}:{}";, minio_ip, MINIO_PORT), + ), + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + ]); + + let file_io = FileIOBuilder::new("s3a") + .with_props(&props) + .build() + .unwrap(); + let config = HmsCatalogConfig::builder() .address(format!("{}:{}", hms_catalog_ip, HMS_CATALOG_PORT)) .thrift_transport(HmsThriftTransport::Buffered) + .with_props(props) .build(); let hms_catalog = HmsCatalog::new(config).unwrap(); TestFixture { _docker_compose: docker_compose, hms_catalog, + file_io, } } +fn set_table_creation(location: impl ToString, name: impl ToString) -> Result<TableCreation> { + let schema = Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "bar", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build()?; + + let creation = TableCreation::builder() + .location(location.to_string()) + .name(name.to_string()) + .properties(HashMap::new()) + .schema(schema) + .build(); + + Ok(creation) +} + +#[tokio::test] +async fn test_rename_table() -> Result<()> { + let fixture = set_test_fixture("test_rename_table").await; + let creation = set_table_creation("s3a://warehouse/hive", "my_table")?; + let namespace = Namespace::new(NamespaceIdent::new("default".into())); + + let table = fixture + .hms_catalog + .create_table(namespace.name(), creation) + .await?; + + let dest = TableIdent::new(namespace.name().clone(), "my_table_rename".to_string()); + + fixture + .hms_catalog + .rename_table(table.identifier(), &dest) + .await?; + + let result = fixture.hms_catalog.table_exists(&dest).await?; + + assert!(result); + + Ok(()) +} + +#[tokio::test] +async fn test_table_exists() -> Result<()> { + let fixture = set_test_fixture("test_table_exists").await; + let creation = set_table_creation("s3a://warehouse/hive", "my_table")?; + let namespace = Namespace::new(NamespaceIdent::new("default".into())); + + let table = fixture + .hms_catalog + .create_table(namespace.name(), creation) + .await?; + + let result = fixture.hms_catalog.table_exists(table.identifier()).await?; + + assert!(result); + + Ok(()) +} + +#[tokio::test] +async fn test_drop_table() -> Result<()> { + let fixture = set_test_fixture("test_drop_table").await; + let creation = set_table_creation("s3a://warehouse/hive", "my_table")?; + let namespace = Namespace::new(NamespaceIdent::new("default".into())); + + let table = fixture + .hms_catalog + .create_table(namespace.name(), creation) + .await?; + + fixture.hms_catalog.drop_table(table.identifier()).await?; + + let result = fixture.hms_catalog.table_exists(table.identifier()).await?; + + assert!(!result); + + Ok(()) +} + +#[tokio::test] +async fn test_load_table() -> Result<()> { + let fixture = set_test_fixture("test_load_table").await; + let creation = set_table_creation("s3a://warehouse/hive", "my_table")?; + let namespace = Namespace::new(NamespaceIdent::new("default".into())); + + let expected = fixture + .hms_catalog + .create_table(namespace.name(), creation) + .await?; + + let result = fixture + .hms_catalog + .load_table(&TableIdent::new( + namespace.name().clone(), + "my_table".to_string(), + )) + .await?; + + assert_eq!(result.identifier(), expected.identifier()); + assert_eq!(result.metadata_location(), expected.metadata_location()); + assert_eq!(result.metadata(), expected.metadata()); + + Ok(()) +} + +#[tokio::test] +async fn test_create_table() -> Result<()> { + let fixture = set_test_fixture("test_create_table").await; + let creation = set_table_creation("s3a://warehouse/hive", "my_table")?; + let namespace = Namespace::new(NamespaceIdent::new("default".into())); + + let result = fixture + .hms_catalog + .create_table(namespace.name(), creation) + .await?; + + assert_eq!(result.identifier().name(), "my_table"); + assert!(result + .metadata_location() + .is_some_and(|location| location.starts_with("s3a://warehouse/hive/metadata/00000-"))); + assert!( + fixture + .file_io + .is_exist("s3a://warehouse/hive/metadata/") + .await? + ); + + Ok(()) +} + +#[tokio::test] +async fn test_list_tables() -> Result<()> { + let fixture = set_test_fixture("test_list_tables").await; + + let ns = Namespace::new(NamespaceIdent::new("default".into())); + + let result = fixture.hms_catalog.list_tables(ns.name()).await?; + + assert_eq!(result, vec![]); Review Comment: Could we add a test where `list_table` returns some actual values? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For additional commands, e-mail: issues-h...@iceberg.apache.org
