xanderbailey commented on code in PR #3011: URL: https://github.com/apache/iceberg-rust/pull/3011#discussion_r4205830062
########## crates/catalog/rest/src/scan_planning.rs: ########## @@ -0,0 +1,1744 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! REST server-side scan planning client. +//! +//! Implements the plan / fetch-result / cancel / fetch-tasks endpoints and a +//! [`RestCatalog::wait_for_plan`] poller. Task decoding and `TableScan` +//! auto-routing are follow-ups: [`RestCatalog::supports_remote_scan_planning`] +//! stays `false` until those land. + +use std::time::Duration; + +use iceberg::{Error, ErrorKind, Result, TableIdent}; +use rand::Rng; +use reqwest::{Method, StatusCode}; +use serde::de::{self, Deserializer}; +use serde::{Deserialize, Serialize}; +use uuid::{Uuid, Variant, Version}; + +use crate::catalog::RestCatalog; +use crate::client::{deserialize_catalog_response, deserialize_unexpected_catalog_error}; +use crate::endpoint::{ + Endpoint, V1_CANCEL_PLANNING, V1_FETCH_PLAN_RESULT, V1_FETCH_SCAN_TASKS, V1_PLAN_TABLE_SCAN, +}; +use crate::request::HttpRequest; +use crate::response::HttpResponse; +use crate::types::{ErrorModel, ErrorResponse, StorageCredential}; + +const HEADER_IDEMPOTENCY_KEY: &str = "Idempotency-Key"; +const HEADER_ACCESS_DELEGATION: &str = "X-Iceberg-Access-Delegation"; + +const MSG_PLAN_EXPIRED: &str = "scan plan expired"; +const MSG_PLAN_FAILED: &str = "scan plan failed"; +const MSG_PLAN_CANCELLED: &str = "scan plan cancelled"; +const MSG_NO_SUCH_PLAN_TASK: &str = "scan plan task not found"; +const MSG_PLAN_POLL_EXHAUSTED: &str = "scan plan polling exhausted retries"; + +const ERR_TYPE_NO_SUCH_PLAN_ID: &str = "NoSuchPlanIdException"; +const ERR_TYPE_NO_SUCH_PLAN_TASK: &str = "NoSuchPlanTaskException"; +const ERR_TYPE_NO_SUCH_TABLE: &str = "NoSuchTableException"; +const ERR_TYPE_NO_SUCH_NAMESPACE: &str = "NoSuchNamespaceException"; + +/// Status of a server-side scan plan. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum PlanStatus { + /// Planning finished and tasks (or plan-task handles) are available. + Completed, + /// Planning is still running; poll [`RestCatalog::fetch_planning_result`]. + Submitted, + /// The plan was cancelled. Valid on fetch-result, not on planTableScan. + Cancelled, + /// Planning failed. The error detail is on the failed arm. + Failed, +} + +/// Task payload shared by completed planning responses and fetchScanTasks. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "kebab-case")] +pub struct ScanTasks { + /// Opaque plan-task handles that still need [`RestCatalog::fetch_scan_tasks`]. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub plan_tasks: Vec<String>, + /// File scan tasks. `data-file` is left as JSON until a decoder lands. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub file_scan_tasks: Vec<RestFileScanTask>, + /// Delete files referenced by the scan tasks, as raw REST JSON. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub delete_files: Vec<serde_json::Value>, +} + +/// REST `FileScanTask` wire payload. Nested content-files stay opaque. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct RestFileScanTask { + /// REST ContentFile JSON for the data file. + pub data_file: serde_json::Value, Review Comment: Is there a reason not to strongly type this? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
