This is an automated email from the ASF dual-hosted git repository. lhotari pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/pulsar-site.git
commit c02cd69da597969c4af38639548c0094b5f72be2 Author: Lari Hotari <[email protected]> AuthorDate: Fri Apr 24 11:55:27 2026 +0300 Handle processing of @pulsar:.*@ variables in docsify site /reference --- package.json | 1 + scripts/process-reference-markdown.js | 96 ++++++++++++++++++++++ .../markdownPreprocessors/pulsarVariables.ts | 19 +++-- tools/pytools/lib/execute/site_builder.py | 4 + 4 files changed, 113 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 7f5b3328fe8..9d033ac7d7c 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "docusaurus": "docusaurus", "start": "NODE_OPTIONS=--max_old_space_size=16000 docusaurus start", "build": "NODE_OPTIONS=--max_old_space_size=16000 docusaurus build", + "process-reference-markdown": "node scripts/process-reference-markdown.js", "preview": "docusaurus build --locale en && docker-compose up", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", diff --git a/scripts/process-reference-markdown.js b/scripts/process-reference-markdown.js new file mode 100644 index 00000000000..ee1b5d6ab44 --- /dev/null +++ b/scripts/process-reference-markdown.js @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Post-processes the Docsify reference site under build/reference/<version>/**/*.md: +// - expands @pulsar:...@ tokens using the same resolver the Docusaurus +// markdown preprocessor uses (src/server/markdownPreprocessors/pulsarVariables) +// - rewrites `pathname:///` (a Docusaurus-only link prefix that Docsify +// renders literally) to `/` +// +// The Docsify site is served straight from static/, so Docusaurus's markdown +// preprocessor pipeline does not touch it. This script fills that gap at +// build time, operating on the build output to keep source files clean. + +"use strict"; + +require("ts-node").register({ + transpileOnly: true, + compilerOptions: { + module: "commonjs", + moduleResolution: "node", + esModuleInterop: true, + resolveJsonModule: true, + target: "ES2022", + }, +}); + +const fs = require("fs"); +const path = require("path"); + +const {replacePulsarTokens} = require("../src/server/markdownPreprocessors/pulsarVariables"); + +const PATHNAME_PREFIX_RE = /pathname:\/\/\//g; + +function collectMarkdown(dir) { + const out = []; + for (const entry of fs.readdirSync(dir, {withFileTypes: true})) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + out.push(...collectMarkdown(full)); + } else if (entry.isFile() && full.endsWith(".md")) { + out.push(full); + } + } + return out; +} + +function processVersion(referenceDir, versionDirName) { + const versionKey = versionDirName === "next" ? "current" : versionDirName; + const absDir = path.join(referenceDir, versionDirName); + const files = collectMarkdown(absDir); + let changed = 0; + for (const file of files) { + const original = fs.readFileSync(file, "utf8"); + const next = replacePulsarTokens(original, versionKey, file).replace(PATHNAME_PREFIX_RE, "/"); + if (next !== original) { + fs.writeFileSync(file, next); + changed += 1; + } + } + return {total: files.length, changed}; +} + +function main() { + const referenceDir = path.resolve(__dirname, "..", "build", "reference"); + if (!fs.existsSync(referenceDir)) { + console.log(`[process-reference-markdown] ${referenceDir} does not exist, skipping`); + return; + } + let totalFiles = 0; + let totalChanged = 0; + for (const entry of fs.readdirSync(referenceDir, {withFileTypes: true})) { + if (!entry.isDirectory()) continue; + const {total, changed} = processVersion(referenceDir, entry.name); + totalFiles += total; + totalChanged += changed; + } + console.log( + `[process-reference-markdown] processed ${totalFiles} file(s) under build/reference (${totalChanged} rewritten)`, + ); +} + +main(); diff --git a/src/server/markdownPreprocessors/pulsarVariables.ts b/src/server/markdownPreprocessors/pulsarVariables.ts index e39b7c8d2ef..7c41d1233e2 100644 --- a/src/server/markdownPreprocessors/pulsarVariables.ts +++ b/src/server/markdownPreprocessors/pulsarVariables.ts @@ -12,12 +12,7 @@ const TOKEN_RE = /@pulsar:([^@\s]+)@/g; type Args = {filePath: string; fileContent: string}; -export default function pulsarVariablesPreprocessor({filePath, fileContent}: Args): string { - if (!SCOPE_RE.test(filePath)) { - return fileContent; - } - const versionMatch = filePath.match(VERSIONED_RE); - const versionKey = versionMatch ? versionMatch[1] : "current"; +export function replacePulsarTokens(fileContent: string, versionKey: string, filePath?: string): string { const tokens = resolveTokens(versionKey); const warned = new Set<string>(); return fileContent.replace(TOKEN_RE, (match, key: string) => { @@ -27,8 +22,18 @@ export default function pulsarVariablesPreprocessor({filePath, fileContent}: Arg } if (!warned.has(key)) { warned.add(key); - console.warn(`[pulsarVariables] unknown token ${match} in ${filePath}`); + const where = filePath ? ` in ${filePath}` : ""; + console.warn(`[pulsarVariables] unknown token ${match}${where}`); } return match; }); } + +export default function pulsarVariablesPreprocessor({filePath, fileContent}: Args): string { + if (!SCOPE_RE.test(filePath)) { + return fileContent; + } + const versionMatch = filePath.match(VERSIONED_RE); + const versionKey = versionMatch ? versionMatch[1] : "current"; + return replacePulsarTokens(fileContent, versionKey, filePath); +} diff --git a/tools/pytools/lib/execute/site_builder.py b/tools/pytools/lib/execute/site_builder.py index 95595ac2c89..14fd2a19633 100644 --- a/tools/pytools/lib/execute/site_builder.py +++ b/tools/pytools/lib/execute/site_builder.py @@ -38,6 +38,10 @@ def execute(asf_site: Path): bash = find_command('bash', msg="bash is required") run(yarn, 'install', cwd=site_path()) run(bash, 'scripts/split-version-build.sh', *modified_files, cwd=site_path()) + # Expand @pulsar:...@ tokens and rewrite `pathname:///` in the Docsify + # reference site (build/reference/), which Docusaurus copies verbatim + # from static/ and so isn't touched by the markdown preprocessor pipeline. + run(yarn, 'process-reference-markdown', cwd=site_path()) latest_content = site_path() / 'build' # 3. Publish content to asf-site-next branch
