This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 2906a89f93a [Iceberg CDC sink] Top API layer (#40161)
add d1ebefb6f63 [Spanner Change Streams] Claim last processed timestamp
instead of artificial query end timestamp for unbounded queries (#40209)
add 00e322188bf Revert "Bump org.codehaus.plexus:plexus-xml from 3.0.2 to
4.2.0" (#40237)
add c462e49b115 Address wrap-up items for Iceberg side input table cache
(#40222)
add 7a20b37023f AddFiles: Add dry_run (#40167)
add a40a3e32ef8 Revert "fix Bigtable naming (#40206)" (#40245)
add dbd6ca07d32 update BigTable to Bigtable for non-yaml related files
(#40207)
add aeea078636f Clean up ruff breakage findings (#40238)
No new revisions were added by this update.
Summary of changes:
.../IO_Iceberg_Integration_Tests.json | 2 +-
.../workflows/beam_StressTests_Java_BigTableIO.yml | 2 +-
.test-infra/tools/refresh_looker_metrics.py | 4 +-
buildSrc/build.gradle.kts | 2 +-
.../datatokenization/DataTokenization.java | 16 +-
.../examples/complete/datatokenization/README.md | 14 +-
.../transforms/io/TokenizationBigQueryIO.java | 2 +-
.../transforms/io/TokenizationBigTableIO.java | 16 +-
.../apache/beam/it/gcp/bigtable/BigTableIOLT.java | 4 +-
.../apache/beam/it/gcp/bigtable/BigTableIOST.java | 10 +-
learning/prompts/code-generation/04_io_bigtable.md | 2 +-
.../prompts/code-generation/java/04_io_bigtable.md | 4 +-
.../org/apache/beam/sdk/io/range/RangeTracker.java | 2 +-
.../provider/bigtable/BigtableTableProvider.java | 2 +-
.../sql/meta/provider/bigtable/package-info.java | 2 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 2 +-
.../beam/sdk/io/gcp/bigtable/BigtableConfig.java | 4 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 2 +-
.../BigtableReadSchemaTransformProvider.java | 2 +-
.../BigtableWriteSchemaTransformProvider.java | 2 +-
.../action/QueryChangeStreamAction.java | 10 +-
.../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 4 +-
.../action/QueryChangeStreamActionTest.java | 7 +-
sdks/java/io/iceberg/build.gradle | 1 +
.../org/apache/beam/sdk/io/iceberg/AddFiles.java | 119 ++--
.../iceberg/AddFilesSchemaTransformProvider.java | 41 +-
.../beam/sdk/io/iceberg/CommitSchemaOnce.java | 25 +-
.../beam/sdk/io/iceberg/CommitSchemaUnion.java | 671 +++++++--------------
.../apache/beam/sdk/io/iceberg/DryRunReport.java | 434 +++++++++++++
.../beam/sdk/io/iceberg/IcebergCatalogConfig.java | 24 +-
.../beam/sdk/io/iceberg/ReadFooterSchema.java | 33 +-
.../beam/sdk/io/iceberg/RecordWriterManager.java | 5 +-
.../apache/beam/sdk/io/iceberg/SchemaDelta.java | 9 +-
.../beam/sdk/io/iceberg/SchemaEvolutionConfig.java | 34 +-
.../org/apache/beam/sdk/io/iceberg/SchemaPlan.java | 618 +++++++++++++++++++
.../beam/sdk/io/iceberg/SerializableTableSpec.java | 33 +-
.../apache/beam/sdk/io/iceberg/SideInputTable.java | 33 +-
.../beam/sdk/io/iceberg/TableMetadataDriver.java | 83 ++-
.../io/iceberg/WritePartitionedRowsToFiles.java | 5 +-
.../AddFilesSchemaTransformProviderTest.java | 14 +
.../apache/beam/sdk/io/iceberg/AddFilesTest.java | 347 ++++++++++-
.../beam/sdk/io/iceberg/CommitSchemaOnceTest.java | 14 +-
.../beam/sdk/io/iceberg/CommitSchemaUnionTest.java | 345 +++++------
.../sdk/io/iceberg/IcebergCatalogConfigTest.java | 56 ++
.../iceberg/IcebergIOSideInputTableCacheTest.java | 6 +
.../beam/sdk/io/iceberg/ReadFooterSchemaTest.java | 2 +-
.../sdk/io/iceberg/SchemaEvolutionConfigTest.java | 10 +
.../apache/beam/sdk/io/iceberg/SchemaPlanTest.java | 325 ++++++++++
.../sdk/io/iceberg/SerializableTableSpecTest.java | 77 +++
.../beam/sdk/io/iceberg/SideInputTableTest.java | 48 ++
.../sdk/io/iceberg/TableMetadataDriverTest.java | 223 ++++++-
sdks/python/apache_beam/coders/row_coder_test.py | 6 +-
.../snippets/transforms/elementwise/enrichment.py | 2 +-
sdks/python/apache_beam/io/aws/s3io.py | 1 +
.../io/external/xlang_snowflakeio_it_test.py | 2 +-
sdks/python/apache_beam/io/gcp/bigtableio.py | 10 +-
.../apache_beam/io/gcp/bigtableio_it_test.py | 4 +-
sdks/python/apache_beam/io/gcp/bigtableio_test.py | 2 +-
.../io/gcp/gcsfilesystem_integration_test.py | 2 +-
.../apache_beam/io/gcp/pubsub_io_perf_test.py | 2 +-
sdks/python/apache_beam/io/iobase.py | 2 +-
.../ml/inference/onnx_inference_test.py | 2 +-
.../ml/inference/pytorch_inference_test.py | 2 +-
.../ml/inference/sklearn_inference_it_test.py | 2 +-
.../ml/inference/tensorrt_inference_test.py | 2 +-
.../src/yaml/EmojiMap.ts | 3 +-
.../kafka_streams_java_job_server_test.py | 2 +-
.../testing/analyzers/io_tests_config.yaml | 4 +-
.../apache_beam/testing/pubsub_test_context.py | 2 +-
.../testing/pubsub_test_context_test.py | 3 +-
.../transforms/enrichment_handlers/bigtable.py | 24 +-
.../enrichment_handlers/vertex_ai_feature_store.py | 4 +-
sdks/python/apache_beam/transforms/sql_test.py | 2 +-
sdks/python/apache_beam/transforms/util.py | 4 +-
sdks/python/apache_beam/transforms/util_test.py | 6 +-
.../typehints/native_type_compatibility_test.py | 2 +-
sdks/python/apache_beam/utils/secret.py | 14 +-
sdks/python/apache_beam/yaml/standard_io.yaml | 16 +-
sdks/python/apache_beam/yaml/tests/bigtable.yaml | 8 +-
.../yaml/tests/iceberg_add_files_dry_run.yaml | 185 ++++++
.../tensorrt_runinference/build_test_engines.py | 6 +-
.../site/content/en/documentation/io/connectors.md | 2 +-
.../en/documentation/io/developing-io-java.md | 2 +-
.../content/en/documentation/io/io-standards.md | 4 +-
website/www/site/content/en/performance/_index.md | 2 +-
.../site/content/en/performance/bigtable/_index.md | 18 +-
website/www/site/data/performance.yaml | 4 +-
website/www/site/i18n/navbar/en.yaml | 2 +-
88 files changed, 3158 insertions(+), 954 deletions(-)
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/DryRunReport.java
create mode 100644
sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/SchemaPlan.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergCatalogConfigTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/SchemaPlanTest.java
create mode 100644
sdks/python/apache_beam/yaml/tests/iceberg_add_files_dry_run.yaml
