This is an automated email from the ASF dual-hosted git repository.
github-actions[bot] pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from aeea078636f Clean up ruff breakage findings (#40238)
add e3b3e7ca1f7 [Iceberg CDC sink] SchemaTransform (ManagedIO) layer
(#40229)
add 3afe19a1a4c Merge pull request #40146: Remove blanket nullness
suppressions in google-cloud-platform-core
add 5d32ff75b4c Round float seconds to nearest subsecond unit in Timestamp
constructor (#40257)
add 404bde62095 Time out RestrictionTrackers.trySplit after 5 minutes on
lock contention (#40251)
add 0793f1dd10f Merge pull request #38854 from
reuvenlax/schema_update_handle_unknown_fields
add 2c2df5a66b6 Rename Iceberg table cache options for clarity (#40253)
add 720326829a7 Fill gaps between GcsUtilV2 and GcsUtilV1 (#40244)
No new revisions were added by this update.
Summary of changes:
CHANGES.md | 2 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 1 +
.../sdk/fn/splittabledofn/RestrictionTrackers.java | 43 +-
.../fn/splittabledofn/RestrictionTrackersTest.java | 57 +-
.../google-cloud-platform-core/build.gradle | 1 +
.../extensions/gcp/auth/NoopCredentialFactory.java | 13 +-
.../sdk/extensions/gcp/options/GcpOptions.java | 19 +-
.../gcp/options/GoogleApiDebugOptions.java | 13 +-
.../sdk/extensions/gcp/storage/GcsFileSystem.java | 12 +-
.../sdk/extensions/gcp/storage/GcsResourceId.java | 8 +-
.../sdk/extensions/gcp/util/CustomHttpErrors.java | 6 +-
.../beam/sdk/extensions/gcp/util/GcpSecret.java | 2 +
.../beam/sdk/extensions/gcp/util/GcsUtil.java | 10 +-
.../beam/sdk/extensions/gcp/util/GcsUtilV1.java | 81 ++-
.../beam/sdk/extensions/gcp/util/GcsUtilV2.java | 276 +++++++-
.../gcp/util/RetryHttpRequestInitializer.java | 17 +-
.../gcp/util/UploadIdResponseInterceptor.java | 3 -
.../sdk/extensions/gcp/util/gcsfs/GcsPath.java | 44 +-
.../gcp/util/GcsUtilParameterizedIT.java | 155 +++++
.../sdk/extensions/gcp/util/GcsUtilV2Test.java | 418 ++++++++++++
.../beam/sdk/io/gcp/bigquery/AppendClientInfo.java | 27 +-
.../beam/sdk/io/gcp/bigquery/AppendRowsPacket.java | 155 +++++
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 63 +-
.../sdk/io/gcp/bigquery/BigQueryIOTranslation.java | 15 +
.../beam/sdk/io/gcp/bigquery/BigQueryOptions.java | 20 +
.../sdk/io/gcp/bigquery/BigQuerySinkMetrics.java | 1 +
.../sdk/io/gcp/bigquery/BufferMismatchedRows.java | 501 ++++++++++++++
.../sdk/io/gcp/bigquery/DynamicDestinations.java | 38 ++
.../gcp/bigquery/DynamicDestinationsHelpers.java | 13 +
.../gcp/bigquery/SchemaChangeDetectorHelper.java | 236 +++++++
.../sdk/io/gcp/bigquery/SchemaUpdateHoldingFn.java | 85 ++-
.../sdk/io/gcp/bigquery/SplittingIterable.java | 175 ++---
.../bigquery/StorageApiDynamicDestinations.java | 18 +-
.../StorageApiDynamicDestinationsTableRow.java | 30 +-
.../beam/sdk/io/gcp/bigquery/StorageApiLoads.java | 10 +-
.../io/gcp/bigquery/StorageApiWritePayload.java | 16 +
.../StorageApiWriteRecordsInconsistent.java | 117 +++-
.../bigquery/StorageApiWriteUnshardedRecords.java | 740 +++++++++++++--------
.../bigquery/StorageApiWritesShardedRecords.java | 662 +++++++++++++++---
.../gcp/bigquery/StoragePayloadWithDeadline.java | 90 +++
.../io/gcp/bigquery/TableRowToStorageApiProto.java | 48 +-
.../sdk/io/gcp/bigquery/UpgradeTableSchema.java | 120 +++-
.../sdk/io/gcp/bigquery/AppendRowsPacketTest.java | 268 ++++++++
.../io/gcp/bigquery/BigQueryIOTranslationTest.java | 2 +
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 70 +-
.../SchemaChangeDetectorHelperBufferingTest.java | 431 ++++++++++++
.../bigquery/SchemaChangeDetectorHelperTest.java | 307 +++++++++
.../sdk/io/gcp/bigquery/SplittingIterableTest.java | 242 +++++++
.../StorageApiSchemaMismatchDrainTest.java | 641 ++++++++++++++++++
.../bigquery/StorageApiSinkSchemaUpdateITBase.java | 345 +++++++---
...torageApiSinkSchemaUpdateWithInputSchemaIT.java | 5 +-
...ageApiSinkSchemaUpdateWithoutInputSchemaIT.java | 5 +-
.../io/gcp/bigquery/UpgradeTableSchemaTest.java | 212 +++++-
.../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 72 +-
.../IcebergWriteSchemaTransformProvider.java | 401 ++++++++++-
.../sdk/io/iceberg/cdc/sink/CdcWriteConfig.java | 7 +-
.../beam/sdk/io/iceberg/cdc/sink/WriteCdcRows.java | 5 +-
.../iceberg/IcebergIOSideInputTableCacheTest.java | 50 +-
.../IcebergSchemaTransformTranslationTest.java | 27 +
...IcebergWriteSchemaTransformProviderCdcTest.java | 696 +++++++++++++++++++
.../IcebergWriteSchemaTransformProviderTest.java | 46 +-
.../sdk/io/iceberg/cdc/sink/AssignCdcKeysTest.java | 3 +-
.../io/iceberg/cdc/sink/CdcWriteConfigTest.java | 4 +-
sdks/python/apache_beam/utils/timestamp.py | 7 +-
sdks/python/apache_beam/utils/timestamp_test.py | 16 +
sdks/python/apache_beam/yaml/yaml_io.py | 75 ++-
66 files changed, 7376 insertions(+), 922 deletions(-)
create mode 100644
sdks/java/extensions/google-cloud-platform-core/src/test/java/org/apache/beam/sdk/extensions/gcp/util/GcsUtilV2Test.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AppendRowsPacket.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BufferMismatchedRows.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/SchemaChangeDetectorHelper.java
create mode 100644
sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/StoragePayloadWithDeadline.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AppendRowsPacketTest.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/SchemaChangeDetectorHelperBufferingTest.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/SchemaChangeDetectorHelperTest.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/SplittingIterableTest.java
create mode 100644
sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/StorageApiSchemaMismatchDrainTest.java
create mode 100644
sdks/java/io/iceberg/src/test/java/org/apache/beam/sdk/io/iceberg/IcebergWriteSchemaTransformProviderCdcTest.java
