nssalian commented on code in PR #2601:
URL: https://github.com/apache/iceberg-go/pull/2601#discussion_r4604697678


##########
website/src/api.md:
##########
@@ -853,6 +853,21 @@ if err != nil { /* ... */ }
 newTbl, err := txn.Commit(ctx)
 ```
 
+Snapshot expiration needs complete ancestry to honor branch retention and 
snapshot
+references. A table loaded by REST with `snapshot-loading-mode=refs` may 
contain
+only directly referenced snapshots. Calling `Transaction.ExpireSnapshots` on
+such a table fails before staging any updates; `errors.Is(err,
+table.ErrInvalidOperation)` returns true. The rejected operation does not
+prevent other operations, such as appends, from using the transaction.
+
+If a concurrent writer forces a commit retry or a second `Commit` attempt
+to reload refs-only history, the expiration cannot be replayed safely.
+`Transaction.Commit` then returns `table.ErrTransactionUnusable` and
+does not resubmit the expiration; it leaves the catalog's existing snapshots

Review Comment:
   For library users `snapshot-loading-mode` is a client catalog property that 
wins over server defaults (catalog/rest/rest.go:1301-1306), and reloading 
through the same catalog stays in refs mode (rest.go:2027). The note at 
api.md:684 already says "Load with snapshot-loading-mode=all instead". Suggest: 
"Create the REST catalog with `snapshot-loading-mode=all` (or omit it), reload, 
and start a new transaction. If the server forces `refs` through config 
overrides, expiration is not available through that catalog." Same wording as 
suggested on https://github.com/apache/iceberg-go/pull/2599.



##########
table/expire_replay_test.go:
##########
@@ -167,6 +201,130 @@ func TestExpireReplayPlansWhereExpiryWasStaged(t 
*testing.T) {
        })
 }
 
+// TestFailedRefsOnlyExpirationLeavesAppendUsable verifies that rejecting a
+// history-dependent operation does not poison an otherwise valid transaction.
+// Even a no-op expiration is unsafe to plan from refs-only metadata.
+func TestFailedRefsOnlyExpirationLeavesAppendUsable(t *testing.T) {
+       fs, cat, tbl := newExpireReplayTable(t, nil)
+       tbl.snapshotHistoryRefsOnly = true
+       tx := tbl.NewTransaction()
+
+       err := tx.ExpireSnapshots(WithOlderThan(0), WithPostCommit(false))
+       require.ErrorIs(t, err, ErrInvalidOperation)
+       require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration)
+       staged, err := tx.StagedTable()
+       require.NoError(t, err)
+       require.Empty(t, staged.Metadata().Snapshots(), "rejected expiration 
must not stage updates")

Review Comment:
   nit: the fixture starts with no snapshots, so this can't fail. Comparing the 
transaction's staged updates and requirements before and after the call would 
check what the message says.



##########
table/expire_replay_test.go:
##########
@@ -167,6 +201,130 @@ func TestExpireReplayPlansWhereExpiryWasStaged(t 
*testing.T) {
        })
 }
 
+// TestFailedRefsOnlyExpirationLeavesAppendUsable verifies that rejecting a
+// history-dependent operation does not poison an otherwise valid transaction.
+// Even a no-op expiration is unsafe to plan from refs-only metadata.
+func TestFailedRefsOnlyExpirationLeavesAppendUsable(t *testing.T) {
+       fs, cat, tbl := newExpireReplayTable(t, nil)
+       tbl.snapshotHistoryRefsOnly = true
+       tx := tbl.NewTransaction()
+
+       err := tx.ExpireSnapshots(WithOlderThan(0), WithPostCommit(false))
+       require.ErrorIs(t, err, ErrInvalidOperation)
+       require.ErrorIs(t, err, errSnapshotHistoryIncompleteForExpiration)
+       staged, err := tx.StagedTable()
+       require.NoError(t, err)
+       require.Empty(t, staged.Metadata().Snapshots(), "rejected expiration 
must not stage updates")
+
+       stageFiles(t, tx, "after-rejected-expiration")
+       committed, err := tx.Commit(t.Context())
+       require.NoError(t, err)
+       require.Len(t, committed.Metadata().Snapshots(), 1)
+       require.Equal(t, []string{"after-rejected-expiration"},
+               branchFiles(t, fs, committed.Metadata(), MainBranch))
+       require.Equal(t, int32(1), cat.attempts.Load(), "append should commit 
once")
+}
+
+func TestExpireReplayRejectsRefsOnlyHistoryAfterRetry(t *testing.T) {
+       synctest.Test(t, func(t *testing.T) {
+               fs, inner, tbl := newExpireReplayTable(t, nil)
+               var ids [4]int64
+               for i, name := range []string{"s1", "s2", "s3", "s4"} {
+                       tbl, ids[i] = appendOnBranch(t, tbl, MainBranch, name)
+               }
+               peerUpdate(t, inner, NewSetSnapshotRefUpdate("expired-tag", 
ids[0], TagRef, time.Hour.Milliseconds(), 0, 0))
+               time.Sleep(2 * time.Hour)
+
+               cat := &refsOnlyReloadCatalog{reqValidatingCatalog: inner}
+               tx := newOCCTable(t, inner.metadata, fs, cat).NewTransaction()
+               require.NoError(t, tx.ExpireSnapshots(
+                       WithOlderThan(0), WithRetainLast(5), 
WithPostCommit(false),
+               ))
+               staged, err := tx.StagedTable()
+               require.NoError(t, err)
+               assert.Nil(t, staged.Metadata().SnapshotByName("expired-tag"))
+               assert.Equal(t, sortedIDs(ids[:]...), 
survivors(staged.Metadata()))
+
+               peer := peerAppend(t, fs, inner, MainBranch, "s5")
+               partial := refsOnlySnapshotMetadata(t, inner.metadata, ids[0], 
peer)
+               assert.Equal(t, sortedIDs(ids[0], peer), survivors(partial))
+               cat.partial = partial
+               attemptsBefore := inner.attempts.Load()
+
+               _, err = tx.Commit(t.Context())
+               require.ErrorIs(t, err, ErrTransactionUnusable)
+               require.ErrorIs(t, err, 
errSnapshotHistoryIncompleteForExpiration)
+               assert.Equal(t, 1, cat.loads)
+               assert.Equal(t, attemptsBefore+1, inner.attempts.Load(), "retry 
must stop before resubmitting expiration")
+               assert.Equal(t, sortedIDs(ids[0], ids[1], ids[2], ids[3], 
peer), survivors(inner.metadata))
+               assert.NotNil(t, inner.metadata.SnapshotByID(ids[0]), "retained 
history must remain in catalog metadata")
+               assert.NotNil(t, inner.metadata.SnapshotByName("expired-tag"), 
"failed expiry must not remove the tag")
+               assert.Equal(t, peer, refHead(t, inner.metadata, MainBranch))
+       })
+}
+
+func TestExpireReplayRejectsRefsOnlyHistoryOnResume(t *testing.T) {
+       synctest.Test(t, func(t *testing.T) {
+               fs, inner, tbl := newExpireReplayTable(t, 
iceberg.Properties{CommitNumRetriesKey: "0"})

Review Comment:
   minor: the setup through the expired-tag assertion is the same 14 lines as 
...AfterRetry, and the tail assertions match too. A `stage := func(t 
*testing.T, numRetries string)` closure like 
TestExpireReplayKeepsExpiryThatReplannedToNothing (:580) would let both run as 
subtests.



##########
table/transaction.go:
##########
@@ -827,6 +835,9 @@ func (t *Transaction) ExpireSnapshots(opts 
...ExpireSnapshotsOpt) error {
        if err := validateGCEnabledForSnapshotExpiration(meta.props); err != 
nil {
                return err
        }
+       if t.tbl.snapshotHistoryRefsOnly {

Review Comment:
   not blocking: a table returned by Commit keeps the refs-only flag 
(table.go:1029 copies it) even though REST commit responses are parsed with the 
full snapshot list (rest.go:1892-1894), so ExpireSnapshots on the committed 
table is rejected too. Failing closed is fine for 0.8.0 or do we want to clean 
it up here. CC: @zeroshade @laskoviymishka 



##########
table/staged_replay.go:
##########
@@ -52,6 +52,13 @@ func newStagedReplay(branch string, updates []Update, 
records []expirationRecord
        }
 }
 
+// A staged expiration may have yielded no metadata updates but can produce
+// removals after a concurrent commit. Check the recorded operations, not the
+// length of staged updates, before replaying against incomplete history.
+func (r *stagedReplay) hasExpirationRecords() bool {
+       return r != nil && len(r.records) > 0

Review Comment:
   The comment above is the reason for checking records, but nothing pins it: 
with this returning `len(r.walkedRefs) > 0` (only expiries that produced 
updates) all of ./table/ still passes. A test that stages an ExpireSnapshots 
that plans nothing plus an append, then retries into refs-only history and 
expects ErrTransactionUnusable, would help cover that gap.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to