This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5432402f7dc6 [SPARK-52584][BUILD] Make build script to support preview
releases in finalize step
5432402f7dc6 is described below
commit 5432402f7dc6372a87b911778f15a4904afb2079
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Thu Jun 26 15:49:22 2025 +0900
[SPARK-52584][BUILD] Make build script to support preview releases in
finalize step
### What changes were proposed in this pull request?
This PR proposes to make release script to support preview releases as well.
### Why are the changes needed?
To make the release easier.
### Does this PR introduce _any_ user-facing change?
No, dev-only.
### How was this patch tested?
Manually tested against spark-website.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #51291 from HyukjinKwon/SPARK-52584.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
dev/create-release/release-build.sh | 210 +++++++++++++++++++++++++-----------
1 file changed, 147 insertions(+), 63 deletions(-)
diff --git a/dev/create-release/release-build.sh
b/dev/create-release/release-build.sh
index 45c359a66ae6..46e08637e54f 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -168,7 +168,12 @@ if [[ "$1" == "finalize" ]]; then
import re
release_version = "${RELEASE_VERSION}"
-newline = f' <li><a href="{{{{site.baseurl}}}}/docs/{release_version}/">Spark
{release_version}</a></li>'
+is_preview = bool(re.search(r'-preview\d*$', release_version))
+base_version = re.sub(r'-preview\d*$', '', release_version)
+
+stable_newline = f' <li><a
href="{{{{site.baseurl}}}}/docs/{release_version}/">Spark
{release_version}</a></li>'
+preview_newline = f' <li><a
href="{{{{site.baseurl}}}}/docs/{release_version}/">Spark {release_version}
preview</a></li>'
+
inserted = False
def parse_version(v):
@@ -183,29 +188,63 @@ with open("documentation.md") as f:
lines = f.readlines()
with open("documentation.md", "w") as f:
- for line in lines:
- match = re.search(r'docs/(\d+\.\d+\.\d+)/', line)
- if not inserted and match:
- existing_version = match.group(1)
- if vercmp(release_version, existing_version) >= 0:
- f.write(newline + "\n")
+ if is_preview:
+ in_preview_section = False
+ for i, line in enumerate(lines):
+ if '<p>Documentation for preview releases:</p>' in line:
+ in_preview_section = True
+ f.write(line)
+ continue
+
+ if in_preview_section and
re.search(r'docs/\d+\.\d+\.\d+-preview\d*/', line):
+ existing_version =
re.search(r'docs/(\d+\.\d+\.\d+-preview\d*)/', line).group(1)
+
+ if existing_version == release_version:
+ inserted = True # Already exists, don't add
+ elif not inserted:
+ base_existing = re.sub(r'-preview\d*$', '',
existing_version)
+ preview_num_existing = int(re.search(r'preview(\d*)',
existing_version).group(1) or "0")
+ preview_num_new = int(re.search(r'preview(\d*)',
release_version).group(1) or "0")
+
+ if (vercmp(base_version, base_existing) > 0) or \
+ (vercmp(base_version, base_existing) == 0 and
preview_num_new >= preview_num_existing):
+ f.write(preview_newline + "\n")
+ inserted = True
+
+ f.write(line)
+ continue
+
+ if in_preview_section and "</ul>" in line and not inserted:
+ f.write(preview_newline + "\n")
inserted = True
- f.write(line)
- if not inserted:
- f.write(newline + "\n")
+ f.write(line)
+ else:
+ for line in lines:
+ match = re.search(r'docs/(\d+\.\d+\.\d+)/', line)
+ if not inserted and match:
+ existing_version = match.group(1)
+ if vercmp(release_version, existing_version) >= 0:
+ f.write(stable_newline + "\n")
+ inserted = True
+ f.write(line)
+ if not inserted:
+ f.write(stable_newline + "\n")
EOF
echo "Edited documentation.md"
# 2. Add download link to js/downloads.js
- RELEASE_DATE=$(TZ=America/Los_Angeles date +"%m/%d/%Y")
- IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
- NEW_PACKAGES="packagesV14"
- if [[ "$rel_maj" -ge 4 ]]; then
- NEW_PACKAGES="packagesV15"
- fi
+ if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
+ echo "Skipping js/downloads.js for preview release: $RELEASE_VERSION"
+ else
+ RELEASE_DATE=$(TZ=America/Los_Angeles date +"%m/%d/%Y")
+ IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
+ NEW_PACKAGES="packagesV14"
+ if [[ "$rel_maj" -ge 4 ]]; then
+ NEW_PACKAGES="packagesV15"
+ fi
- python3 <<EOF
+ python3 <<EOF
import re
release_version = "${RELEASE_VERSION}"
@@ -254,13 +293,44 @@ with open("js/downloads.js", "w") as f:
f.write(newline + "\n")
EOF
- echo "Edited js/downloads.js"
+ echo "Edited js/downloads.js"
+ fi
# 3. Add news post
RELEASE_DATE=$(TZ=America/Los_Angeles date +"%Y-%m-%d")
FILENAME="news/_posts/${RELEASE_DATE}-spark-${RELEASE_VERSION//./-}-released.md"
mkdir -p news/_posts
- cat > "$FILENAME" <<EOF
+
+ if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
+ BASE_VERSION="${RELEASE_VERSION%%-preview*}"
+ cat > "$FILENAME" <<EOF
+---
+layout: post
+title: Preview release of Spark ${BASE_VERSION}
+categories:
+- News
+tags: []
+status: publish
+type: post
+published: true
+meta:
+ _edit_last: '4'
+ _wpas_done_all: '1'
+---
+To enable wide-scale community testing of the upcoming Spark ${BASE_VERSION}
release, the Apache Spark community has posted a
+<a
href="https://archive.apache.org/dist/spark/spark-${RELEASE_VERSION}/";>Spark
${RELEASE_VERSION} release</a>.
+This preview is not a stable release in terms of either API or functionality,
but it is meant to give the community early
+access to try the code that will become Spark ${BASE_VERSION}. If you would
like to test the release,
+please <a
href="https://archive.apache.org/dist/spark/spark-${RELEASE_VERSION}/";>download</a>
it, and send feedback using either
+<a href="https://spark.apache.org/community.html";>mailing lists</a> or
+<a
href="https://issues.apache.org/jira/browse/SPARK/?selectedTab=com.atlassian.jira.jira-projects-plugin:summary-panel";>JIRA</a>.
+The documentation is available at the <a
href="https://spark.apache.org/docs/${RELEASE_VERSION}/";>link</a>.
+
+We'd like to thank our contributors and users for their contributions and
early feedback to this release. This release would not have been possible
without you.
+EOF
+
+ else
+ cat > "$FILENAME" <<EOF
---
layout: post
title: Spark ${RELEASE_VERSION} released
@@ -276,16 +346,20 @@ meta:
---
We are happy to announce the availability of <a
href="{{site.baseurl}}/releases/spark-release-${RELEASE_VERSION}.html"
title="Spark Release ${RELEASE_VERSION}">Apache Spark ${RELEASE_VERSION}</a>!
Visit the <a
href="{{site.baseurl}}/releases/spark-release-${RELEASE_VERSION}.html"
title="Spark Release ${RELEASE_VERSION}">release notes</a> to read about the
new features, or <a href="{{site.baseurl}}/downloads.html">download</a> the
release today.
EOF
+ fi
echo "Created $FILENAME"
# 4. Add release notes with Python to extract JIRA version ID
- RELEASE_DATE=$(TZ=America/Los_Angeles date +"%Y-%m-%d")
- JIRA_PROJECT_ID=12315420
- JIRA_URL="https://issues.apache.org/jira/rest/api/2/project/SPARK/versions";
- JSON=$(curl -s "$JIRA_URL")
+ if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
+ echo "Skipping JIRA release notes for preview release: $RELEASE_VERSION"
+ else
+ RELEASE_DATE=$(TZ=America/Los_Angeles date +"%Y-%m-%d")
+ JIRA_PROJECT_ID=12315420
+ JIRA_URL="https://issues.apache.org/jira/rest/api/2/project/SPARK/versions";
+ JSON=$(curl -s "$JIRA_URL")
- VERSION_ID=$(python3 - <<EOF
+ VERSION_ID=$(python3 - <<EOF
import sys, json
release_version = "${RELEASE_VERSION}"
@@ -305,32 +379,32 @@ for v in versions:
print(version_id)
EOF
- )
+ )
- if [[ -z "$VERSION_ID" ]]; then
- echo "Error: Couldn't find JIRA version ID for $RELEASE_VERSION" >&2
- fi
+ if [[ -z "$VERSION_ID" ]]; then
+ echo "Error: Couldn't find JIRA version ID for $RELEASE_VERSION" >&2
+ fi
-
JIRA_LINK="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=${JIRA_PROJECT_ID}&version=${VERSION_ID}";
+
JIRA_LINK="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=${JIRA_PROJECT_ID}&version=${VERSION_ID}";
- IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
- if [[ "$rel_patch" -eq 0 ]]; then
- ACKNOWLEDGE="patches and features to this release."
- BODY="Apache Spark ${RELEASE_VERSION} is a new feature release. It
introduces new functionality and improvements. We encourage users to try it and
provide feedback."
- else
- ACKNOWLEDGE="patches to this release."
- BODY="Apache Spark ${RELEASE_VERSION} is a maintenance release containing
security and correctness fixes. This release is based on the
branch-${rel_maj}.${rel_min} maintenance branch of Spark. We strongly recommend
all ${rel_maj}.${rel_min} users to upgrade to this stable release."
- fi
+ IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
+ if [[ "$rel_patch" -eq 0 ]]; then
+ ACKNOWLEDGE="patches and features to this release."
+ BODY="Apache Spark ${RELEASE_VERSION} is a new feature release. It
introduces new functionality and improvements. We encourage users to try it and
provide feedback."
+ else
+ ACKNOWLEDGE="patches to this release."
+ BODY="Apache Spark ${RELEASE_VERSION} is a maintenance release
containing security and correctness fixes. This release is based on the
branch-${rel_maj}.${rel_min} maintenance branch of Spark. We strongly recommend
all ${rel_maj}.${rel_min} users to upgrade to this stable release."
+ fi
- BODY+="
+ BODY+="
You can find the list of resolved issues and detailed changes in the [JIRA
release notes](${JIRA_LINK}).
We would like to acknowledge all community members for contributing
${ACKNOWLEDGE}"
-
FILENAME="releases/_posts/${RELEASE_DATE}-spark-release-${RELEASE_VERSION}.md"
- mkdir -p releases/_posts
- cat > "$FILENAME" <<EOF
+
FILENAME="releases/_posts/${RELEASE_DATE}-spark-release-${RELEASE_VERSION}.md"
+ mkdir -p releases/_posts
+ cat > "$FILENAME" <<EOF
---
layout: post
title: Spark Release ${RELEASE_VERSION}
@@ -347,40 +421,50 @@ meta:
${BODY}
EOF
- echo "Created $FILENAME"
+ echo "Created $FILENAME"
+ fi
# 5. Build the website
bundle install
bundle exec jekyll build
- # 6. Update latest symlink if minor/major release
- LINK_PATH="site/docs/latest"
- TARGET_DIR="site/docs/$RELEASE_VERSION"
+ # 6. Update latest or preview symlink
IFS='.' read -r rel_maj rel_min rel_patch <<< "$RELEASE_VERSION"
- if [[ "$rel_patch" -eq 0 ]]; then
- if [[ -L "$LINK_PATH" ]]; then
- CURRENT_TARGET=$(readlink "$LINK_PATH")
- else
- CURRENT_TARGET=""
- fi
- if [[ "$CURRENT_TARGET" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
- IFS='.' read -r cur_maj cur_min cur_patch <<< "$CURRENT_TARGET"
+ if [[ "$RELEASE_VERSION" =~ -preview[0-9]*$ ]]; then
+ LINK_PATH="site/docs/preview"
- if [[ "$rel_maj" -gt "$cur_maj" ]]; then
- ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
- echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (major version
increased)"
- elif [[ "$rel_maj" -eq "$cur_maj" && "$rel_min" -gt "$cur_min" ]]; then
- ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
- echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (minor version
increased)"
+ ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
+ echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (preview release)"
+
+ else
+ LINK_PATH="site/docs/latest"
+
+ if [[ "$rel_patch" -eq 0 ]]; then
+ if [[ -L "$LINK_PATH" ]]; then
+ CURRENT_TARGET=$(readlink "$LINK_PATH")
else
- echo "Symlink $LINK_PATH points to $CURRENT_TARGET with equal or newer
major.minor, no change"
+ CURRENT_TARGET=""
+ fi
+
+ if [[ "$CURRENT_TARGET" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+ IFS='.' read -r cur_maj cur_min cur_patch <<< "$CURRENT_TARGET"
+
+ if [[ "$rel_maj" -gt "$cur_maj" ]]; then
+ ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
+ echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (major version
increased)"
+ elif [[ "$rel_maj" -eq "$cur_maj" && "$rel_min" -gt "$cur_min" ]]; then
+ ln -sfn "$RELEASE_VERSION" "$LINK_PATH"
+ echo "Updated symlink $LINK_PATH -> $RELEASE_VERSION (minor version
increased)"
+ else
+ echo "Symlink $LINK_PATH points to $CURRENT_TARGET with equal or
newer major.minor, no change"
+ fi
+ else
+ echo "No valid existing version target."
fi
else
- echo "No valid existing version target."
+ echo "Patch release detected ($RELEASE_VERSION), not updating symlink"
fi
- else
- echo "Patch release detected ($RELEASE_VERSION), not updating symlink"
fi
git add .
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
