This is an automated email from the ASF dual-hosted git repository.
chenli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/texera.git
The following commit(s) were added to refs/heads/main by this push:
new 9ac58cf60b fix(backend): Update PHYSICAL_ADDRESS_EXPIRATION_TIME_HRS
to 24hrs (#4269)
9ac58cf60b is described below
commit 9ac58cf60bd7be153432b3837ae28411b620b46e
Author: carloea2 <[email protected]>
AuthorDate: Tue Mar 17 23:35:42 2026 -0700
fix(backend): Update PHYSICAL_ADDRESS_EXPIRATION_TIME_HRS to 24hrs (#4269)
<!--
Thanks for sending a pull request (PR)! Here are some tips for you:
1. If this is your first time, please read our contributor guidelines:
[Contributing to
Texera](https://github.com/apache/texera/blob/main/CONTRIBUTING.md)
2. Ensure you have added or run the appropriate tests for your PR
3. If the PR is work in progress, mark it a draft on GitHub.
4. Please write your PR title to summarize what this PR proposes, we
are following Conventional Commits style for PR titles as well.
5. Be sure to keep the PR description updated to reflect all changes.
-->
### What changes were proposed in this PR?
<!--
Please clarify what changes you are proposing. The purpose of this
section
is to outline the changes. Here are some tips for you:
1. If you propose a new API, clarify the use case for a new API.
2. If you fix a bug, you can clarify why it is a bug.
3. If it is a refactoring, clarify what has been changed.
3. It would be helpful to include a before-and-after comparison using
screenshots or GIFs.
4. Please consider writing useful notes for better and faster reviews.
-->
Update PHYSICAL_ADDRESS_EXPIRATION_TIME_HRS from 6 hrs to 24 hrs to keep
on sync with lakefs recent PR
https://github.com/treeverse/lakeFS/pull/10180
### Any related issues, documentation, discussions?
<!--
Please use this section to link other resources if not mentioned
already.
1. If this PR fixes an issue, please include `Fixes #1234`, `Resolves
#1234`
or `Closes #1234`. If it is only related, simply mention the issue
number.
2. If there is design documentation, please add the link.
3. If there is a discussion in the mailing list, please add the link.
-->
No
Related to https://github.com/apache/texera/discussions/4184
### How was this PR tested?
<!--
If tests were added, say they were added here. Or simply mention that if
the PR
is tested with existing test cases. Make sure to include/update test
cases that
check the changes thoroughly including negative and positive cases if
possible.
If it was tested in a way different from regular unit tests, please
clarify how
you tested step by step, ideally copy and paste-able, so that other
reviewers can
test and check, and descendants can verify in the future. If tests were
not added,
please describe why they were not added and/or why it was difficult to
add.
-->
Backend testing already applies.
### Was this PR authored or co-authored using generative AI tooling?
<!--
If generative AI tooling has been used in the process of authoring this
PR,
please include the phrase: 'Generated-by: ' followed by the name of the
tool
and its version. If no, write 'No'.
Please refer to the [ASF Generative Tooling
Guidance](https://www.apache.org/legal/generative-tooling.html) for
details.
-->
No
---------
Co-authored-by: Lacia7u7 <[email protected]>
---
.../texera/service/util/S3StorageClient.scala | 3 +-
.../service/resource/DatasetResourceSpec.scala | 41 ++++++++++------------
2 files changed, 20 insertions(+), 24 deletions(-)
diff --git
a/common/workflow-core/src/main/scala/org/apache/texera/service/util/S3StorageClient.scala
b/common/workflow-core/src/main/scala/org/apache/texera/service/util/S3StorageClient.scala
index f3d252d413..d692c574b8 100644
---
a/common/workflow-core/src/main/scala/org/apache/texera/service/util/S3StorageClient.scala
+++
b/common/workflow-core/src/main/scala/org/apache/texera/service/util/S3StorageClient.scala
@@ -38,7 +38,8 @@ import scala.jdk.CollectionConverters._
object S3StorageClient {
val MINIMUM_NUM_OF_MULTIPART_S3_PART: Long = 5L * 1024 * 1024 // 5 MiB
val MAXIMUM_NUM_OF_MULTIPART_S3_PARTS = 10_000
- val PHYSICAL_ADDRESS_EXPIRATION_TIME_HRS = 6
+ //Keep on sync with LakeFS https://github.com/treeverse/lakeFS/pull/10180
+ val PHYSICAL_ADDRESS_EXPIRATION_TIME_HRS = 24
// Initialize MinIO-compatible S3 Client
private lazy val s3Client: S3Client = {
diff --git
a/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala
b/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala
index 24253c3a92..a71861491a 100644
---
a/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala
+++
b/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala
@@ -670,6 +670,20 @@ class DatasetResourceSpec
sessionRecord.getUploadId
}
+ private def expireUploadSession(uploadId: String): Unit = {
+ val expiredHoursAgo = S3StorageClient.PHYSICAL_ADDRESS_EXPIRATION_TIME_HRS
+ 1
+ getDSLContext
+ .update(DATASET_UPLOAD_SESSION)
+ .set(
+ DATASET_UPLOAD_SESSION.CREATED_AT,
+ DSL
+ .field(s"current_timestamp - interval '${expiredHoursAgo} hours'")
+ .cast(classOf[java.time.OffsetDateTime])
+ )
+ .where(DATASET_UPLOAD_SESSION.UPLOAD_ID.eq(uploadId))
+ .execute()
+ }
+
private def assertPlaceholdersCreated(uploadId: String, expectedParts: Int):
Unit = {
val rows = fetchPartRows(uploadId).sortBy(_.getPartNumber)
rows.size shouldEqual expectedParts
@@ -717,17 +731,9 @@ class DatasetResourceSpec
initUpload(fpB, numParts = 2).getStatus shouldEqual 200
initUpload(fpA, numParts = 2).getStatus shouldEqual 200
- // Expire fpB by pushing created_at back > 6 hours.
+ // Expire fpB by pushing created_at back beyond the real session
expiration window.
val uploadIdB = fetchUploadIdOrFail(fpB)
- val tableName = DATASET_UPLOAD_SESSION.getName // typically
"dataset_upload_session"
- getDSLContext
- .update(DATASET_UPLOAD_SESSION)
- .set(
- DATASET_UPLOAD_SESSION.CREATED_AT,
- DSL.field("current_timestamp - interval '7
hours'").cast(classOf[java.time.OffsetDateTime])
- )
- .where(DATASET_UPLOAD_SESSION.UPLOAD_ID.eq(uploadIdB))
- .execute()
+ expireUploadSession(uploadIdB)
val listed = listUploads()
listed shouldEqual listed.sorted
@@ -905,19 +911,8 @@ class DatasetResourceSpec
uploadPart(filePath, 1, minPartBytes(1.toByte)).getStatus shouldEqual 200
fetchPartRows(oldUploadId).find(_.getPartNumber == 1).get.getEtag.trim
should not be ""
- // Age the session so it is definitely expired (>
PHYSICAL_ADDRESS_EXPIRATION_TIME_HRS = 6)
- val expireHrs = S3StorageClient.PHYSICAL_ADDRESS_EXPIRATION_TIME_HRS
-
- getDSLContext
- .update(DATASET_UPLOAD_SESSION)
- .set(
- DATASET_UPLOAD_SESSION.CREATED_AT,
- DSL
- .field(s"current_timestamp - interval '${expireHrs + 1} hours'")
- .cast(classOf[java.time.OffsetDateTime])
- )
- .where(DATASET_UPLOAD_SESSION.UPLOAD_ID.eq(oldUploadId))
- .execute()
+ // Age the session so it is definitely expired (>
PHYSICAL_ADDRESS_EXPIRATION_TIME_HRS)
+ expireUploadSession(oldUploadId)
// Same init config again -> should restart because it's expired
val r2 = initUpload(filePath, numParts = 2, lastPartBytes = 123)
