Fokko commented on code in PR #60: URL: https://github.com/apache/iceberg-cpp/pull/60#discussion_r2037651164
########## src/iceberg/snapshot.h: ########## @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include <optional> +#include <string> +#include <string_view> +#include <unordered_map> +#include <variant> +#include <vector> + +#include "iceberg/iceberg_export.h" + +namespace iceberg { + +/// \brief The type of snapshot reference +enum class SnapshotRefType { + /// Branches are mutable named references that can be updated by committing a new + /// snapshot as the branch’s referenced snapshot using the Commit Conflict Resolution + /// and Retry procedures. + kBranch, + /// Tags are labels for individual snapshots + kTag, +}; + +/// \brief A reference to a snapshot, either a branch or a tag. +struct ICEBERG_EXPORT SnapshotRef { + /// A reference's snapshot ID. The tagged snapshot or latest snapshot of a branch. + int64_t snapshot_id; + /// Type of the reference, tag or branch + SnapshotRefType type; + /// For branch type only, a positive number for the minimum number of snapshots to keep + /// in a branch while expiring snapshots. Defaults to table property + /// history.expire.min-snapshots-to-keep. + std::optional<int32_t> min_snapshots_to_keep; + /// For branch type only, a positive number for the max age of snapshots to keep when + /// expiring, including the latest snapshot. Defaults to table property + /// history.expire.max-snapshot-age-ms. + std::optional<int64_t> max_snapshot_age_ms; + /// For snapshot references except the main branch, a positive number for the max age of + /// the snapshot reference to keep while expiring snapshots. Defaults to table property + /// history.expire.max-ref-age-ms. The main branch never expires. + std::optional<int64_t> max_ref_age_ms; +}; + +/// \brief Optional Snapshot Summary Fields +struct SnapshotSummaryFields { + /// \brief The operation field key + static const std::string kOperation; + + /// Metrics, see https://iceberg.apache.org/spec/#metrics + + /// \brief Number of data files added in the snapshot + static const std::string kAddedDataFiles; + /// \brief Number of data files deleted in the snapshot + static const std::string kDeletedDataFiles; + /// \brief Total number of live data files in the snapshot + static const std::string kTotalDataFiles; + /// \brief Number of positional/equality delete files and deletion vectors added in the + /// snapshot + static const std::string kAddedDeleteFiles; + /// \brief Number of equality delete files added in the snapshot + static const std::string kAddedEqDeleteFiles; + /// \brief Number of equality delete files removed in the snapshot + static const std::string kRemovedEqDeleteFiles; + /// \brief Number of position delete files added in the snapshot + static const std::string kAddedPosDeleteFiles; + /// \brief Number of position delete files removed in the snapshot + static const std::string kRemovedPosDeleteFiles; + /// \brief Number of deletion vectors added in the snapshot + static const std::string kAddedDVS; + /// \brief Number of deletion vectors removed in the snapshot + static const std::string kRemovedDVS; + /// \brief Number of positional/equality delete files and deletion vectors removed in + /// the snapshot + static const std::string kRemovedDeleteFiles; + /// \brief Total number of live positional/equality delete files and deletion vectors in + /// the snapshot + static const std::string kTotalDeleteFiles; + /// \brief Number of records added in the snapshot + static const std::string kAddedRecords; + /// \brief Number of records deleted in the snapshot + static const std::string kDeletedRecords; + /// \brief Total number of records in the snapshot + static const std::string kTotalRecords; + /// \brief The size of files added in the snapshot + static const std::string kAddedFileSize; + /// \brief The size of files removed in the snapshot + static const std::string kRemovedFileSize; + /// \brief Total size of live files in the snapshot + static const std::string kTotalFileSize; + /// \brief Number of position delete records added in the snapshot + static const std::string kAddedPosDeletes; + /// \brief Number of position delete records removed in the snapshot + static const std::string kRemovedPosDeletes; + /// \brief Total number of position delete records in the snapshot + static const std::string kTotalPosDeletes; + /// \brief Number of equality delete records added in the snapshot + static const std::string kAddedEqDeletes; + /// \brief Number of equality delete records removed in the snapshot + static const std::string kRemovedEqDeletes; + /// \brief Total number of equality delete records in the snapshot + static const std::string kTotalEqDeletes; + /// \brief Number of duplicate files deleted (duplicates are files recorded more than + /// once in the manifest) + static const std::string kDeletedDuplicatedFiles; + /// \brief Number of partitions with files added or removed in the snapshot + static const std::string kChangedPartitionCountProp; + + /// Other Fields, see https://iceberg.apache.org/spec/#other-fields + + /// \brief The Write-Audit-Publish id of a staged snapshot + static const std::string kWAPID; + /// \brief The Write-Audit-Publish id of a snapshot already been published + static const std::string kPublishedWAPID; + /// \brief The original id of a cherry-picked snapshot + static const std::string kSourceSnapshotID; + /// \brief Name of the engine that created the snapshot + static const std::string kEngineName; + /// \brief Version of the engine that created the snapshot + static const std::string kEngineVersion; +}; + +/// \brief Data operation that produce snapshots. +/// +/// A snapshot can return the operation that created the snapshot to help other components +/// ignore snapshots that are not needed for some tasks. For example, snapshot expiration +/// does not need to clean up deleted files for appends, which have no deleted files. +struct ICEBERG_EXPORT DataOperation { + /// \brief Only data files were added and no files were removed. + static constexpr std::string kAppend = "append"; + /// \brief Data and delete files were added and removed without changing table data; + /// i.e. compaction, change the data file format, or relocating data files. + static constexpr std::string kReplace = "replace"; + /// \brief Data and delete files were added and removed in a logical overwrite + /// operation. + static constexpr std::string kOverwrite = "overwrite"; + /// \brief Data files were removed and their contents logically deleted and/or delete + /// files were added to delete rows. + static constexpr std::string kDelete = "delete"; +}; + +/// \brief The location of a manifest list for this snapshot that tracks manifest files +/// with additional metadata +struct ICEBERG_EXPORT ManifestList { + std::string manifest_list_path; +}; + +/// \brief A list of manifest file locations. +struct ICEBERG_EXPORT Manifests { + std::vector<std::string> manifest_paths; +}; + +/// \brief A snapshot of the data in a table at a point in time. +/// +/// A snapshot consist of one or more file manifests, and the complete table contents is +/// the union of all the data files in those manifests. +/// +/// Snapshots are created by table operations. +struct ICEBERG_EXPORT Snapshot { + /// A unqiue long ID. + int64_t snapshot_id; + /// The snapshot ID of the snapshot's parent. Omitted for any snapshot with no parent. + std::optional<int64_t> parent_snapshot_id; + /// A monotonically increasing long that tracks the order of changes to a table. + int64_t sequence_number; + /// A timestamp when the snapshot was created, used for garbage collection and table + /// inspection. + int64_t timestamp_ms; + /// The location of a manifest list for this snapshot that tracks manifest files with + /// additional metadata(v2) or a list of manifest file locations(v1). Review Comment: I think what this is referring to is the `manifests field`: <img width="784" alt="image" src="https://github.com/user-attachments/assets/a990d1c8-a8e9-4da7-a20e-501afcd4dff6"; /> I think we should leave this out entirely, like we did with PyIceberg. ```suggestion /// The location of a manifest list for this snapshot ``` The `manifests` is used only used before it was open source, but it quickly turned out that using the manifest-list is the better option to keep the `manifest.json` within reasonable size. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For additional commands, e-mail: issues-h...@iceberg.apache.org
