twuebi commented on PR #331: URL: https://github.com/apache/iceberg-rust/pull/331#issuecomment-2205383749
Hi @nastra, this script: ```python3 import pyspark from pyspark.conf import SparkConf from pyspark.sql import SparkSession import pandas as pd CATALOG_URL = "http://server:8080/catalog"; MANAGEMENT_URL = "http://server:8080/management"; DEMO_WAREHOUSE = "demo" SPARK_VERSION = pyspark.__version__ SPARK_MINOR_VERSION = '.'.join(SPARK_VERSION.split('.')[:2]) ICEBERG_VERSION = "1.5.2" config = { "spark.sql.catalog.demo-catalog": "org.apache.iceberg.spark.SparkCatalog", "spark.sql.catalog.demo-catalog.type": "rest", "spark.sql.catalog.demo-catalog.uri": CATALOG_URL, "spark.sql.catalog.demo-catalog.warehouse": DEMO_WAREHOUSE, "spark.sql.catalog.demo-catalog.io-impl": "org.apache.iceberg.aws.s3.S3FileIO", "spark.sql.extensions": "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", "spark.sql.defaultCatalog": "demo-catalog", "spark.jars.packages": "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.5.2,org.apache.iceberg:iceberg-aws-bundle:1.5.2", } spark_config = SparkConf().setMaster('local').setAppName("Iceberg-REST") for k, v in config.items(): spark_config = spark_config.set(k, v) spark = SparkSession.builder.config(conf=spark_config).getOrCreate() spark.sql("CREATE NAMESPACE IF NOT EXISTS spark_demo") data = pd.DataFrame([[1, 'a-string', 2.2]], columns=['id', 'strings', 'floats']) sdf = spark.createDataFrame(data) sdf.writeTo("spark_demo.my_table").createOrReplace() spark.sql("CREATE view spark_demo.vv as select * from spark_demo.my_table") ``` ends up sending that POST to `/catalog/v1/df910b7e-3912-11ef-a048-6b2680efd54d/namespaces/spark_demo/views` ```json { "name": "vv", "schema": { "schema-id": 0, "type": "struct", "fields": [ { "id": 0, "name": "id", "required": false, "type": "long" }, { "id": 1, "name": "strings", "required": false, "type": "string" }, { "id": 2, "name": "floats", "required": false, "type": "double" } ] }, "view-version": { "version-id": 1, "schema-id": 0, "timestamp-ms": 1719994069884, "summary": { "app-id": "local-1719994018459", "engine-name": "spark", "iceberg-version": "Apache Iceberg 1.5.2 (commit cbb853073e681b4075d7c8707610dceecbee3a82)", "engine-version": "3.5.1" }, "representations": [ { "type": "sql", "sql": "select * from spark_demo.my_table", "dialect": "spark" } ], "default-namespace": [] }, "properties": { "create_engine_version": "Spark 3.5.1", "spark.query-column-names": "id,strings,floats", "engine_version": "Spark 3.5.1" } } ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For additional commands, e-mail: issues-h...@iceberg.apache.org
