This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-website.git
The following commit(s) were added to refs/heads/master by this push:
new 5e9ed7644189 [improvement](website) Avoid discarded Git dates and
repeated HTML parsing (#4538)
5e9ed7644189 is described below
commit 5e9ed76441890c1096e7c562746d00702e93f038
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Wed Oct 28 03:43:39 2026 +0000
[improvement](website) Avoid discarded Git dates and repeated HTML parsing
(#4538)
The complete five-language production build now has healthy memory
headroom, but run 40608604747 took 143:30 for the build job, exceeding
the existing 180-minute budget with 30% margin. Its managed docs
repeatedly ask Git for dates of generated, untracked translation views,
then discard those dates in favor of actual source dates; strict
verification also reparses pages already visited as counterparts.
Suppress native date/author lookup only while loading a managed
translation view, restore the original options before contentLoaded, and
retain the existing per-document source-date restoration. Real source
lookups and displayed date/author options remain intact. Reuse the same
invocation's compact immutable HTML metadata for each page's own
assertions as well as counterpart checks. Keep every language, status,
canonical, robots, heading, sitemap, search-exclusion and
reciprocal-alternate assertion, including duplicate canonicals without
href.
Validation:
- 408 local Node checks: 406 passed, 2 conditional skips; 27 workflow
checks passed.
- Same-source real five-language fixture: all 230 files / 3,294,801
bytes identical before and after the date change; generated-view Git
lookups decreased from 36 to zero.
- New native fixture verifies source date/author and visible footer
options on 40 source, translated and fallback pages; it runs in the
existing content-fixture CI path and its own changes select that path.
- Existing five-language content fixture returns identical strict
results in before/after/after/before comparisons; full HTML parses
decrease from 51 to 41. Negative cases still reject damaged headings,
duplicate alternates, wrong body language/status, duplicate canonicals
without href, and indexed-page noindex.
These are fixture results, not a claim that the complete production time
budget has passed. T22 remains open until a new complete production run
and public acceptance succeed. No local full website build was
performed.
| Repository review checkpoint | Conclusion |
| --- | --- |
| Goal | Removes measured discarded work without changing source-date
policy or reducing checks; final production budget remains to be
measured. |
| Focus | Only managed date loading, immutable verifier summaries and
their regression/CI coverage change. |
| Information architecture / locales / versions | No content, state,
ownership, versions, sidebars or coverage denominators change. |
| URLs / anchors / navigation | Routes and anchors are unchanged;
heading and reciprocal-link assertions remain. |
| Frontend / mobile / SSR / themes | No frontend or styling change;
date/author footers and same-source output bytes are verified. |
| Validation | Real five-language builds, negative cases, CI selection
and existing Node/workflow checks cover the changes; production timing
will follow. |
| Other quality | No dependency, credential, permission or
publication-boundary changes. Existing content link debt remains
separately registered in T23. |
Co-authored-by: morningman <[email protected]>
---
.github/workflows/build-check.yml | 1 +
scripts/i18n/__tests__/ci-sampling.test.cjs | 3 ++
scripts/i18n/__tests__/static-cache.test.cjs | 13 +++++++
scripts/i18n/ci-plan.cjs | 2 +-
scripts/i18n/docs-plugin.cjs | 10 ++++-
scripts/i18n/verify-doc-dates-fixture.cjs | 56 ++++++++++++++++++++++++++++
scripts/i18n/verify-static.cjs | 47 +++++++++++------------
7 files changed, 107 insertions(+), 25 deletions(-)
diff --git a/.github/workflows/build-check.yml
b/.github/workflows/build-check.yml
index a9873df6e599..3ade1b45f782 100644
--- a/.github/workflows/build-check.yml
+++ b/.github/workflows/build-check.yml
@@ -140,6 +140,7 @@ jobs:
if: steps.detect.outputs.content_fixture == 'true'
run: |
node scripts/i18n/verify-content-fixture.cjs
+ node scripts/i18n/verify-doc-dates-fixture.cjs
node scripts/i18n/verify-blog-fixture.cjs
node scripts/i18n/verify-release-policy-fixture.cjs
diff --git a/scripts/i18n/__tests__/ci-sampling.test.cjs
b/scripts/i18n/__tests__/ci-sampling.test.cjs
index 73695721b86e..e9476784c3b9 100644
--- a/scripts/i18n/__tests__/ci-sampling.test.cjs
+++ b/scripts/i18n/__tests__/ci-sampling.test.cjs
@@ -95,6 +95,9 @@ test('CI tooling only runs the fixtures for its own
subsystem', () => {
assert.equal(language.runContentFixture, true);
assert.equal(language.runSamplingFixture, false);
assert.equal(language.runPublishFixture, false);
+ const dates=policy(['scripts/i18n/verify-doc-dates-fixture.cjs']);
+
assert.equal(dates.mode,'fixtures');assert.equal(dates.runContentFixture,true);
+ assert.equal(dates.publicationAllowed,false);
const sampler = policy(['scripts/i18n/ci-sample.cjs']);
assert.equal(sampler.runSamplingFixture, true);
assert.equal(sampler.runPublishFixture, false);
diff --git a/scripts/i18n/__tests__/static-cache.test.cjs
b/scripts/i18n/__tests__/static-cache.test.cjs
index b0456e1068e5..fe41260f015c 100644
--- a/scripts/i18n/__tests__/static-cache.test.cjs
+++ b/scripts/i18n/__tests__/static-cache.test.cjs
@@ -39,3 +39,16 @@ test('each invocation rereads the artifact and rejects
damaged heading IDs and d
const
$=cheerio.load(original);$('head').append($('link[hreflang="ko"]').clone());fs.writeFileSync(file,$.html());
assert.throws(()=>verifyAll(data),/Duplicate hreflang/);
});
+
+test('cached counterpart metadata still rejects each page own language,
notice, canonical and robots defects',t=>{
+ const
data=fixture(t),file=path.join(data.site,'ko/topic/index.html'),original=fs.readFileSync(file,'utf8');
+ for(const [change,message]of [
+ [text=>text.replace('lang="ko"><h2','lang="en"><h2'),/Body language/],
+
[text=>text.replace('data-translation-status="current"','data-translation-status="wrong"'),/Notice/],
+ [text=>text.replace('</head>','<link
rel="canonical"></head>'),/Duplicate canonical/],
+ [text=>text.replace('</head>','<meta name="robots"
content="noindex"></head>'),/ko\/topic/],
+ ]){
+ fs.writeFileSync(file,change(original));
+ assert.throws(()=>verifyAll(data),message);
+ }
+});
diff --git a/scripts/i18n/ci-plan.cjs b/scripts/i18n/ci-plan.cjs
index b02eae80433c..ff99e7a6695c 100644
--- a/scripts/i18n/ci-plan.cjs
+++ b/scripts/i18n/ci-plan.cjs
@@ -65,7 +65,7 @@ function policy(files, {forceFull = false, forceSample =
false} = {}) {
runContentFixture:
(shared && checks.check_content_fixture) ||
files.some(file =>
-
/verify-(content-fixture|global-language)|__tests__\/fixtures\/content-manifest/.test(file),
+
/verify-(content-fixture|doc-dates-fixture|global-language)|__tests__\/fixtures\/content-manifest/.test(file),
),
runSamplingFixture: files.some(file =>
/^scripts\/(build-locales?\.cjs$|i18n\/(compiler-processes|ci-sample|ci-blog-sample|build-cache|verify-ci-sampl(?:e|ing-fixture))\.cjs$)/.test(file)),
diff --git a/scripts/i18n/docs-plugin.cjs b/scripts/i18n/docs-plugin.cjs
index 057a6751e15c..4b6dc94b03e1 100644
--- a/scripts/i18n/docs-plugin.cjs
+++ b/scripts/i18n/docs-plugin.cjs
@@ -208,7 +208,15 @@ module.exports = async function
translationDocsPlugin(context, options) {
// the view, just like a new static build. Do not watch our own
view.
if (loaded) return (await translationDocsPlugin(context,
options)).loadContent();
loaded = true;
- const content = await plugin.loadContent();
+ let content;
+ const dateOptions = {showLastUpdateTime:
options.showLastUpdateTime,
+ showLastUpdateAuthor: options.showLastUpdateAuthor};
+ // Native dates for generated views would query untracked files.
+ // Every managed doc receives its actual source dates below. Keep
+ // the original options for contentLoaded and the displayed footer.
+ if (activeLocale) Object.assign(options, {showLastUpdateTime:
false, showLastUpdateAuthor: false});
+ try {content = await plugin.loadContent();}
+ finally {if (activeLocale) Object.assign(options, dateOptions);}
if (activeLocale) {
for (const version of content.loadedVersions) {
for (const doc of [...version.docs, ...version.drafts]) {
diff --git a/scripts/i18n/verify-doc-dates-fixture.cjs
b/scripts/i18n/verify-doc-dates-fixture.cjs
new file mode 100644
index 000000000000..2da72bb4cb91
--- /dev/null
+++ b/scripts/i18n/verify-doc-dates-fixture.cjs
@@ -0,0 +1,56 @@
+// Real native docs dates and footer checks, including untracked translation
views.
+const
fs=require('node:fs'),path=require('node:path'),assert=require('node:assert/strict');
+const {spawnSync,execFileSync}=require('node:child_process');
+const cheerio=require('cheerio');
+const {createFixture}=require('./__tests__/fixtures/multi-locale.cjs');
+const {hash}=require('./state.cjs');
+const
root=path.resolve(__dirname,'../..'),fixture=path.join(root,'.i18n-build/date-fixture');
+async function main(){
+ fs.rmSync(fixture,{recursive:true,force:true});
+ await createFixture(fixture);
+ for(const locale of ['ja','ko','ru']){
+ const
file=path.join(fixture,'translations/state',locale,'docs.json'),state=JSON.parse(fs.readFileSync(file));
+ for(const record of state.pages){
+ const
target=path.join(fixture,record.targetPath),content=fs.readFileSync(target,'utf8').replace(/last_update:\n
date: 2026-01-01\n/,'');
+ fs.writeFileSync(target,content);record.targetHash=hash(content);
+ }
+ fs.writeFileSync(file,JSON.stringify(state));
+ }
+ fs.appendFileSync(path.join(fixture,'docusaurus.config.js'),`\nconst
original=module.exports;module.exports=()=>{const config=original();
+for(const item of
config.plugins)if(Array.isArray(item)&&item[1]?.path){item[1].showLastUpdateTime=true;item[1].showLastUpdateAuthor=true;}
+return config;};\n`);
+ execFileSync('git',['add','.'],{cwd:fixture});
+
execFileSync('git',['-c','user.name=Fixture','-c','[email protected]','commit','-qm','Source
date fixture'],{cwd:fixture});
+ const
directory=path.join(root,'.i18n-build/date-git-calls'),probe=path.join(root,'.i18n-build/date-git-probe.cjs');
+
fs.rmSync(directory,{recursive:true,force:true});fs.mkdirSync(directory,{recursive:true});
+ fs.writeFileSync(probe,`const
fs=require('node:fs'),path=require('node:path');
+const
utils=require(${JSON.stringify(require.resolve('@docusaurus/utils/lib/gitUtils.js'))}),original=utils.getFileCommitDate,paths=[];
+utils.getFileCommitDate=function(file,...args){paths.push(file);return
original(file,...args);};
+process.on('exit',()=>{fs.writeFileSync(path.join(process.env.DORIS_DATE_PROBE_DIRECTORY,process.pid+'.json'),JSON.stringify({paths}));});\n`);
+ const
result=spawnSync(process.execPath,[path.join(root,'scripts/build-locales.cjs'),'--site-dir',fixture],{
+
cwd:root,stdio:'inherit',env:{...process.env,NODE_OPTIONS:`--max-old-space-size=4096
--require ${JSON.stringify(probe)}`,
+ DORIS_DATE_PROBE_DIRECTORY:directory}});
+ assert.equal(result.status,0,'Source date five-language native build');
+ const
calls=fs.readdirSync(directory).flatMap(file=>JSON.parse(fs.readFileSync(path.join(directory,file))).paths);
+ const
viewCalls=calls.filter(file=>/[/\\]\.i18n-build[/\\](ja|ko|ru)[/\\]docusaurus-plugin-content-docs/.test(file));
+ assert.equal(viewCalls.length,0,'Generated translation views must never
query Git dates');
+ assert(calls.length>0,'Real source author dates must still use native Git
lookup when required');
+ const checked=[];
+ for(const locale of ['en','zh-CN','ja','ko','ru']){
+ const
manifest=JSON.parse(fs.readFileSync(path.join(fixture,'.i18n-build/content-manifests',locale+'.json')));
+ for(const page of manifest.pages.filter(page=>page.kind==='docs')){
+ const
$=cheerio.load(fs.readFileSync(path.join(fixture,'build',page.route,'index.html'),'utf8')),footer=$('.theme-last-updated');
+ assert.equal(footer.length,1,`Date footer options must remain
enabled: ${page.route}`);
+
assert.equal(footer.find('time').attr('datetime'),'2026-01-01T00:00:00.000Z',page.route);
+ assert(footer.text().includes('Fixture'),`Source author:
${page.route}`);
+
checked.push({locale,route:page.route,contentLocale:page.contentLocale,sourceDate:'2026-01-01',sourceAuthor:'Fixture'});
+ }
+ }
+ const
report={passed:true,implementationSha:execFileSync('git',['rev-parse','HEAD'],{cwd:root,encoding:'utf8'}).trim(),
+
fixtureSourceSha:execFileSync('git',['rev-parse','HEAD'],{cwd:fixture,encoding:'utf8'}).trim(),
+ temporaryViewGitCalls:0,sourceGitCalls:calls.length,checked,
+ scope:'Real five-language native docs build; source date/author and
visible footer options retained for source, translated and fallback pages. No
full website build.'};
+
fs.writeFileSync(path.join(root,'.i18n-build/doc-dates-fixture-verification.json'),JSON.stringify(report,null,2)+'\n');
+
console.log(JSON.stringify({passed:true,temporaryViewGitCalls:0,sourceGitCalls:calls.length,checked:checked.length}));
+}
+if(require.main===module)main().catch(error=>{console.error(error);process.exitCode=1;});
diff --git a/scripts/i18n/verify-static.cjs b/scripts/i18n/verify-static.cjs
index 68389918b403..d20c124b6d29 100644
--- a/scripts/i18n/verify-static.cjs
+++ b/scripts/i18n/verify-static.cjs
@@ -52,7 +52,12 @@ function verify({site, manifest, partial = false, headCache
= new Map()}) {
if (!alternates.has(lang)) alternates.set(lang, []);
alternates.get(lang).push($(node).attr('href'));
}
- headCache.set(file, {headingIds:headingIds($), alternates});
+ headCache.set(file, {headingIds:headingIds($), alternates,
+ htmlLang:$('html').attr('lang'),
bodyLang:$('.theme-doc-markdown,.course-doc-content').attr('lang'),
+
translationStatus:$('[data-translation-status]').attr('data-translation-status'),
+
canonicals:$('link[rel="canonical"]').toArray().map(node=>$(node).attr('href')),
+ robots:$('meta[name="robots"]').attr('content'),
+ ogUrl:$('meta[property="og:url"]').attr('content'),
ogLocale:$('meta[property="og:locale"]').attr('content')});
}
return headCache.get(file);
}
@@ -71,19 +76,15 @@ function verify({site, manifest, partial = false, headCache
= new Map()}) {
assert(page.draft, `Missing non-draft static page:
${page.permalink}`);
continue;
}
- const $ = cheerio.load(fs.readFileSync(filename, 'utf8'));
- head(filename, $);
- const body = $('.theme-doc-markdown,.course-doc-content');
- assert.equal(body.attr('lang'), page.contentLocale, `Body language:
${page.permalink}`);
-
assert.equal($('[data-translation-status]').attr('data-translation-status'),
page.status, `Notice: ${page.permalink}`);
- assert.equal($('link[rel="canonical"]').length, 1, `Duplicate
canonical: ${page.permalink}`);
+ const metadata = head(filename);
+ assert.equal(metadata.bodyLang, page.contentLocale, `Body language:
${page.permalink}`);
+ assert.equal(metadata.translationStatus, page.status, `Notice:
${page.permalink}`);
+ assert.equal(metadata.canonicals.length, 1, `Duplicate canonical:
${page.permalink}`);
const canonical = page.useTarget ? page.permalink : page.contentLocale
=== 'en' ? page.englishPermalink : page.sourcePermalink;
- assert.equal($('link[rel="canonical"]').attr('href'),
`https://doris.apache.org${canonical}`);
- const alternates = $('link[rel="alternate"][hreflang]');
- const localized = alternates.filter(`[hreflang="${htmlLang}"]`);
- assert.equal(localized.length, alternateGroup[locale] ? 1 : 0,
`Localized alternate: ${page.permalink}`);
+ assert.equal(metadata.canonicals[0],
`https://doris.apache.org${canonical}`);
+ assert.equal(metadata.alternates.get(htmlLang)?.length ?? 0,
alternateGroup[locale] ? 1 : 0, `Localized alternate: ${page.permalink}`);
if (!page.indexable) {
-
assert($('meta[name="robots"]').attr('content').includes('noindex'),
page.permalink);
+ assert(metadata.robots.includes('noindex'), page.permalink);
assert(!sitemapPaths.has(page.permalink), `Fallback/stale page in
sitemap: ${page.permalink}`);
assert(!indexedPaths.has(page.permalink), `Fallback/stale page in
local search index: ${page.permalink}`);
for (const permalink of Object.values(alternateGroup)) {
@@ -94,10 +95,10 @@ function verify({site, manifest, partial = false, headCache
= new Map()}) {
}
}
} else {
- assert(!($('meta[name="robots"]').attr('content') ??
'').includes('noindex'), page.permalink);
+ assert(!(metadata.robots ?? '').includes('noindex'),
page.permalink);
assert(sitemapPaths.has(page.permalink), `Current page missing
from sitemap: ${page.permalink}`);
const english = counterpart(page.englishPermalink, 'source heading
IDs');
- if (english) assert.deepEqual(headingIds($), english.headingIds,
`Source heading IDs: ${page.permalink}`);
+ if (english) assert.deepEqual(metadata.headingIds,
english.headingIds, `Source heading IDs: ${page.permalink}`);
for (const [lang, permalink] of Object.entries(alternateGroup)) {
const counterpart$ = counterpart(permalink, 'reciprocal
alternates');
if (!counterpart$) continue;
@@ -117,20 +118,20 @@ function verify({site, manifest, partial = false,
headCache = new Map()}) {
for (const permalink of data.pageRoutes ?? []) {
const filename = path.join(site, decodeURIComponent(permalink),
'index.html');
assert(fs.existsSync(filename), `Missing ordinary page: ${permalink}`);
- const $ = cheerio.load(fs.readFileSync(filename, 'utf8'));
+ const metadata = head(filename);
const source = permalink.startsWith(`/${locale}/`) ?
permalink.slice(locale.length + 1) : permalink;
const translatedPages = data.translatedPagePaths?.[locale] ??
record?.translatedPages;
const decision = contentByRoute.get(permalink);
const fallback = decision ? decision.fallback : record?.ordinaryPages
=== false && !translatedPages?.includes(source.replace(/\/+$/, ''));
if (fallback) {
- assert.equal($('html').attr('lang'), htmlLang, `Fallback
navigation language: ${permalink}`);
- assert.equal($('link[rel="canonical"]').length, 1, `Duplicate
ordinary canonical: ${permalink}`);
- assert.equal($('link[rel="canonical"]').attr('href'),
`https://doris.apache.org${source}`);
- assert.equal($('meta[property="og:url"]').attr('content'),
`https://doris.apache.org${source}`);
- assert.equal($('meta[property="og:locale"]').attr('content'),
'en_US');
- assert(($('meta[name="robots"]').attr('content') ??
'').includes('noindex'), `Indexed English fallback: ${permalink}`);
+ assert.equal(metadata.htmlLang, htmlLang, `Fallback navigation
language: ${permalink}`);
+ assert.equal(metadata.canonicals.length, 1, `Duplicate ordinary
canonical: ${permalink}`);
+ assert.equal(metadata.canonicals[0],
`https://doris.apache.org${source}`);
+ assert.equal(metadata.ogUrl, `https://doris.apache.org${source}`);
+ assert.equal(metadata.ogLocale, 'en_US');
+ assert((metadata.robots ?? '').includes('noindex'), `Indexed
English fallback: ${permalink}`);
assert(!sitemapPaths.has(permalink), `Ordinary fallback in
sitemap: ${permalink}`);
-
assert.equal($(`link[rel="alternate"][hreflang="${htmlLang}"]`).length, 0,
`Ordinary fallback alternate: ${permalink}`);
+ assert.equal(metadata.alternates.get(htmlLang)?.length ?? 0, 0,
`Ordinary fallback alternate: ${permalink}`);
for (const sourceRoute of (decision ?
Object.values(decision.navigationRoutes) : [source, `/zh-CN${source}`])) {
const source$ = counterpart(sourceRoute, 'ordinary fallback
alternates');
if (source$)
assert.equal(source$.alternates.get(htmlLang)?.length ?? 0, 0,
@@ -138,7 +139,7 @@ function verify({site, manifest, partial = false, headCache
= new Map()}) {
}
ordinaryPages.fallback++;
} else {
- assert.equal($('html').attr('lang'), htmlLang, `Translated
ordinary page language: ${permalink}`);
+ assert.equal(metadata.htmlLang, htmlLang, `Translated ordinary
page language: ${permalink}`);
}
ordinaryPages.verified++;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
