This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-website.git
The following commit(s) were added to refs/heads/master by this push:
new 12d50a851e3 feat(i18n): plan Japanese translation tasks from current
docs (#4227)
12d50a851e3 is described below
commit 12d50a851e3f80df09de5e04d6a4821883a02b65
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Tue Oct 13 15:46:25 2026 +0800
feat(i18n): plan Japanese translation tasks from current docs (#4227)
## Summary
- Add an offline Japanese translation task planner based on the complete
native Dev, 4.x, and community audit at the checked-out source commit.
- Plan new, changed, moved, and deleted documents with stable IDs. Keep
active PR candidates separate from merged coverage, protect edited
targets and ambiguous source states, and report Chinese-only counterpart
changes.
- Add a reviewed deletion ledger, command and candidate schema
documentation, and planner tests to Build Check.
## Validation
- `node --test scripts/i18n/__tests__/*.test.cjs`: 43 tests passed.
- Two full scans of commit `58dc485ae1f60fcb2168b65b405384c97485a561`
were byte-identical. The final head
`c08868902b36e5ac32d309108223d360e8035221` also completed a full scan:
3,286 required, 233 merged/current, 3,053 pending; 18 tasks selected
under the default 50,000 approximate source-token budget.
- `git diff --check` passed. No model service or GitHub write API is
called by the planner.
## Scope
This PR delivers T06 only. It does not generate translations or change
website routes and content. T07 and T08 remain separate tasks.
---------
Co-authored-by: morningman <[email protected]>
---
.github/workflows/build-check.yml | 1 +
scripts/i18n/__tests__/planner.test.cjs | 189 +++++++++++++++++++
scripts/i18n/__tests__/t04.test.cjs | 5 +
scripts/i18n/audit-coverage.cjs | 9 +-
scripts/i18n/plan-translations-baseline.cjs | 13 ++
scripts/i18n/plan-translations.cjs | 261 ++++++++++++++++++++++++++
scripts/i18n/structure.cjs | 4 +
translations/README.md | 70 +++++++
translations/state/ja/resolved-deletions.json | 4 +
9 files changed, 552 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/build-check.yml
b/.github/workflows/build-check.yml
index f3c6afaf408..06971519cfc 100644
--- a/.github/workflows/build-check.yml
+++ b/.github/workflows/build-check.yml
@@ -96,6 +96,7 @@ jobs:
yarn
yarn key-features:generate
node --test scripts/i18n/__tests__/t04.test.cjs
+ node --test scripts/i18n/__tests__/planner.test.cjs
# Dependencies are installed unconditionally above because
the
# governance steps below need node_modules; only the site
build
diff --git a/scripts/i18n/__tests__/planner.test.cjs
b/scripts/i18n/__tests__/planner.test.cjs
new file mode 100644
index 00000000000..e721af07a60
--- /dev/null
+++ b/scripts/i18n/__tests__/planner.test.cjs
@@ -0,0 +1,189 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const {hash} = require('../state.cjs');
+const {plan, formattingOnly, validateActive, validateResolved} =
require('../plan-translations.cjs');
+
+const sha = 'a'.repeat(40);
+const baselineSha = 'b'.repeat(40);
+const scope = {schemaVersion: 1, locale: 'ja', maxBatchPages: 20, groups: [
+ {id: 'DEV', pluginId: 'default', version: 'current', sourceRoot: 'docs'},
+ {id: '4X', pluginId: 'default', version: '4.x', sourceRoot:
'versioned_docs/version-4.x'},
+ {id: 'COMMUNITY', pluginId: 'community', version: 'current', sourceRoot:
'community'},
+]};
+const source = '# Example\n\nHello.\n';
+function page(docId, options = {}) {
+ const version = options.version ?? 'current';
+ const pluginId = options.pluginId ?? 'default';
+ const root = pluginId === 'community' ? 'community' : version === '4.x' ?
'versioned_docs/version-4.x' : 'docs';
+ const sourcePath = options.sourcePath ?? `${root}/${docId}.md`;
+ return {pluginId, version, docId, sourcePath, targetPath:
`i18n/ja/${version}/${docId}.md`,
+ permalink: `/ja/${version}/${docId}/`, sourceHash:
hash(options.content ?? source),
+ targetHash: options.targetHash ?? null, state: options.state ??
'untranslated',
+ reviewStatus: 'unverified', change: options.change ?? 'new'};
+}
+async function run(pages, {old = [], records = [], removed = [], active, files
= {}, historical = {}, reports = [],
+ chineseChangedPaths = [], limit = 20, tokenBudget = 50000} = {}) {
+ const all = Object.fromEntries(pages.map(item => [item.sourcePath,
source]));
+ const report = {schemaVersion: 1, sourceCommit: sha, baselineCommit:
baselineSha,
+ groups: scope.groups.map(group => ({...group, entries:
pages.filter(item => item.pluginId === group.pluginId && item.version ===
group.version),
+ unregistered: []})), removed, reports};
+ return plan({report, scope, records: new Map(records.map(item =>
[JSON.stringify([item.pluginId, item.version, item.docId, 'ja']), item])),
+ baseline: old, active, chineseChangedPaths, readCurrent: name =>
files[name] ?? all[name] ?? null,
+ readAt: (ref, name) => historical[`${ref}:${name}`] ?? null, limit,
tokenBudget});
+}
+
+test('stable full scan includes new pages after baseline, isolates versions,
and never requeues current pages', async () => {
+ const pages = [page('new'), page('new', {version: '4.x'}), page('done',
{state: 'current'})];
+ const first = await run(pages);
+ const second = await run(pages);
+ assert.deepEqual(first, second);
+ assert.equal(first.counts.required, 3);
+ assert.equal(first.counts.mergedCurrent, 1);
+ assert.equal(first.tasks.length, 2);
+ assert.notEqual(first.tasks[0].id, first.tasks[1].id);
+ assert(first.tasks.every(task => task.kind === 'translate_new'));
+});
+
+test('modified completed source is an update and valid active PR suppresses
selection without adding coverage', async () => {
+ const previous = {...page('edited'), locale: 'ja', sourceHash: hash('#
Old\n'), sourceCommit: baselineSha,
+ targetHash: hash('訳文'), disposition: 'translate'};
+ const changed = page('edited', {state: 'stale_technical', change:
'source_changed', targetHash: hash('訳文')});
+ const active = {schemaVersion: 1, candidates: [{pluginId: 'default',
version: 'current', docId: 'edited',
+ locale: 'ja', sourcePath: changed.sourcePath, sourceHash:
changed.sourceHash, pr: '#123'}]};
+ const result = await run([changed], {old: [previous], active});
+ assert.equal(result.tasks[0].kind, 'update');
+ assert.equal(result.tasks[0].activePr, '#123');
+ assert.equal(result.counts.mergedCurrent, 0);
+ assert.deepEqual(result.selected, []);
+});
+
+test('stale active PR is reported and a later source revision reenters
selection', async () => {
+ const changed = page('edited', {state: 'stale_unknown'});
+ const active = {schemaVersion: 1, candidates: [{pluginId: 'default',
version: 'current', docId: 'edited',
+ locale: 'ja', sourcePath: changed.sourcePath, sourceHash:
hash('older'), pr: '#123'}]};
+ const result = await run([changed], {old: [{...changed, sourceHash:
hash('older'), sourceCommit: baselineSha}], active});
+ assert.equal(result.exceptions.some(item => item.kind ===
'stale_active_candidate'), true);
+ assert.equal(result.selected.length, 1);
+});
+
+test('an active PR for an old path cannot suppress a moved page with the same
source hash', async () => {
+ const changed = page('new', {sourcePath: 'docs/new.md'});
+ const active = {schemaVersion: 1, candidates: [{pluginId: 'default',
version: 'current', docId: 'new',
+ locale: 'ja', sourcePath: 'docs/old.md', sourceHash:
changed.sourceHash, pr: '#123'}]};
+ const result = await run([changed], {active});
+ assert.equal(result.tasks[0].activePr, null);
+ assert.equal(result.selected.length, 1);
+});
+
+test('same identity path move includes cleanup and only reuses a verified
target', async () => {
+ const old = {...page('old'), locale: 'ja', docId: 'moved', sourceHash:
hash(source),
+ targetHash: hash('訳文'), disposition: 'translate'};
+ const newer = page('moved', {sourcePath: 'docs/new/moved.md', state:
'state_conflict',
+ targetHash: hash('訳文'), change: 'moved'});
+ const result = await run([newer], {old: [old], files: {[old.targetPath]:
'訳文'}});
+ assert.equal(result.tasks[0].kind, 'move');
+ assert.equal(result.tasks[0].reuseTarget, true);
+ assert.equal(result.tasks[0].referenceChecks.oldSourcePath,
old.sourcePath);
+});
+
+test('unique same-content rename pairs a removed doc and has a reference
plan', async () => {
+ const before = {...page('old'), locale: 'ja', disposition: 'translate',
targetHash: hash('訳文')};
+ const after = page('new', {state: 'untranslated'});
+ const result = await run([after], {old: [before], files:
{[before.targetPath]: '訳文'}});
+ assert.equal(result.tasks.length, 1);
+ assert.equal(result.tasks[0].kind, 'move');
+ assert.equal(result.tasks[0].reuseTarget, true);
+ assert.equal(result.tasks[0].referenceChecks.oldSourcePath,
before.sourcePath);
+});
+
+test('ambiguous same-content rename remains new plus deletions, and deleted
records are planned', async () => {
+ const before = {...page('old'), locale: 'ja', disposition: 'translate'};
+ const result = await run([page('new'), page('new2')], {old: [before]});
+ assert.equal(result.tasks.filter(item => item.kind === 'move').length, 0);
+ assert.equal(result.tasks.filter(item => item.kind === 'delete').length,
1);
+});
+
+test('target edits, source mismatch and unsupported routes require human
review', async () => {
+ const result = await run([page('edited', {state: 'target_modified'}),
page('missing', {state: 'source_missing'}),
+ page('route', {state: 'source_route_conflict'})]);
+ assert(result.tasks.every(item => item.requiresHumanReview));
+ assert.deepEqual(result.selected, []);
+});
+
+test('format-only changes are review tasks; SQL, MDX attributes, and front
matter changes remain updates', async () => {
+ const first = '# Example\n\nHello world.\n';
+ assert.equal(await formattingOnly(first, '# Example\n\n\nHello world.\n',
'test.md'), true);
+ assert.equal(await formattingOnly('```sql\nSELECT 1;\n```\n',
'```sql\nSELECT 2;\n```\n', 'test.md'), false);
+ assert.equal(await formattingOnly('<Box size={1}>Hi</Box>\n', '<Box
size={2}>Hi</Box>\n', 'test.mdx'), false);
+ assert.equal(await formattingOnly('---\ntitle: A\n---\nHi\n', '---\ntitle:
B\n---\nHi\n', 'test.md'), false);
+});
+
+test('historical formatting only and prose edits receive different work
kinds', async () => {
+ const current = page('formatted', {state: 'stale_unknown', change:
'source_changed'});
+ const before = {...current, sourceHash: hash('# Example\n\n\nHello.\n'),
sourceCommit: baselineSha};
+ const formatting = await run([current], {old: [before], historical:
{[`${baselineSha}:${before.sourcePath}`]: '# Example\n\n\nHello.\n'}});
+ assert.equal(formatting.tasks[0].kind, 'formatting_review');
+ assert.deepEqual(formatting.selected, []);
+ const prose = await run([current], {old: [{...before, sourceHash: hash('#
Example\n\nGoodbye.\n')}],
+ historical: {[`${baselineSha}:${before.sourcePath}`]: '#
Example\n\nGoodbye.\n'}});
+ assert.equal(prose.tasks[0].kind, 'update');
+});
+
+test('Chinese-only routes are exceptions and never become translation tasks',
async () => {
+ const result = await run([], {reports: [{kind: 'unsupported_source_route',
pluginId: 'default',
+ version: 'current', sourceLocale: 'zh-CN', sourcePath:
'i18n/zh-CN/only.md'}]});
+ assert.equal(result.tasks.length, 0);
+ assert.equal(result.exceptions.length, 1);
+});
+
+test('Chinese-only counterpart edits are reported without creating Japanese
work', async () => {
+ const current = page('shared', {state: 'current'});
+ const before = {...current, disposition: 'translate'};
+ const chinese =
'i18n/zh-CN/docusaurus-plugin-content-docs/current/shared.md';
+ const result = await run([current], {old: [before], chineseChangedPaths:
[chinese]});
+ assert.equal(result.tasks.length, 0);
+ assert.equal(result.exceptions.find(item => item.kind ===
'zh_only_change').chinesePath, chinese);
+ const changedEnglish = await run([current], {old: [{...before, sourceHash:
hash('older')}], chineseChangedPaths: [chinese]});
+ assert.equal(changedEnglish.exceptions.some(item => item.kind ===
'zh_only_change'), false);
+ const translatedAfterEnglishEdit = await run([current], {old: [{...before,
sourceHash: hash('older')}],
+ records: [{...before, sourceHash: current.sourceHash}],
chineseChangedPaths: [chinese]});
+ assert.equal(translatedAfterEnglishEdit.exceptions.some(item => item.kind
=== 'zh_only_change'), false);
+});
+
+test('limit, token budget and path filter apply after full counts without
renumbering', async () => {
+ const a = page('a'), b = page('b');
+ const result = await run([a, b], {tokenBudget: 5});
+ assert.equal(result.counts.pending, 2);
+ assert.equal(result.selected.length, 1);
+ const report = {schemaVersion: 1, sourceCommit: sha, baselineCommit:
baselineSha,
+ groups: scope.groups.map(group => ({...group, entries: [a,
b].filter(item => item.pluginId === group.pluginId && item.version ===
group.version),
+ unregistered: []})), reports: []};
+ const filtered = await plan({report, scope, records: new Map(),
readCurrent: () => source, readAt: () => null,
+ filters: {path: 'docs/b'}, limit: 1});
+ assert.equal(filtered.tasks.length, 1);
+ assert.equal(filtered.counts.pending, 2);
+ assert.equal(filtered.tasks[0].id, result.tasks.find(item => item.docId
=== 'b').id);
+});
+
+test('active input rejects duplicate, out-of-scope and malformed candidates',
() => {
+ const candidate = {pluginId: 'default', version: 'current', docId: 'a',
locale: 'ja', sourcePath: 'docs/a.md',
+ sourceHash: hash(source), pr: '#1'};
+ assert.throws(() => validateActive({schemaVersion: 1, candidates:
[candidate, candidate]}, scope), /Duplicate/);
+ assert.throws(() => validateActive({schemaVersion: 1, candidates:
[{...candidate, version: '3.x'}]}, scope), /outside/);
+ assert.throws(() => validateActive({schemaVersion: 1, candidates:
[{...candidate, sourceHash: 'bad'}]}, scope), /Invalid/);
+});
+
+test('resolved deletion requires review evidence and does not recur', async ()
=> {
+ const before = {...page('old'), locale: 'ja', disposition: 'translate'};
+ const first = await run([], {old: [before]});
+ const deletion = first.tasks[0];
+ const resolved = {schemaVersion: 1, deletions: [{id: deletion.id,
sourceHash: deletion.sourceHash,
+ sourcePath: deletion.sourcePath, reviewedBy: 'reviewer', resolvedAt:
'2026-10-13T00:00:00Z', evidence: 'PR #1'}]};
+ assert.equal(validateResolved(resolved).length, 1);
+ const report = {schemaVersion: 1, sourceCommit: sha, baselineCommit:
baselineSha,
+ groups: scope.groups.map(group => ({...group, entries: [],
unregistered: []})), reports: []};
+ const second = await plan({report, scope, records: new Map(), baseline:
[before], resolved,
+ readCurrent: () => null, readAt: () => null});
+ assert.equal(second.tasks.length, 0);
+ assert.throws(() => validateResolved({schemaVersion: 1, deletions:
[{...resolved.deletions[0], evidence: ''}]}), /Invalid/);
+});
diff --git a/scripts/i18n/__tests__/t04.test.cjs
b/scripts/i18n/__tests__/t04.test.cjs
index dd334250ec4..0e62bb9dd2e 100644
--- a/scripts/i18n/__tests__/t04.test.cjs
+++ b/scripts/i18n/__tests__/t04.test.cjs
@@ -83,6 +83,7 @@ test('full audit finds documents and sidebar keys added after
the T03 snapshot',
write('translations/scope/ja.json',JSON.stringify({schemaVersion:1,locale:'ja',maxBatchPages:20,groups,retainEnglish:[]}));
write('translations/config.json','{}');
write('translations/state/ja/docs.json',JSON.stringify({schemaVersion:1,pages:[]}));
+
write('translations/state/ja/resolved-deletions.json',JSON.stringify({schemaVersion:1,deletions:[]}));
for (const file of
['docusaurus.config.js','versions.json','sidebars.ts','sidebarsCommunity.json','versioned_sidebars/version-4.x-sidebars.json'])
write(file,file+'\n');
const pages = groups.map(group => {
@@ -91,6 +92,9 @@ test('full audit finds documents and sidebar keys added after
the T03 snapshot',
sourceHash:hash(content),targetPath:`i18n/ja/${group.id}/new.md`,targetHash:null,
status:'untranslated',reviewStatus:'unverified',useTarget:false,draft:false,permalink:`/ja/${group.id}/new/`};
});
+ pages[0].status = 'current';
+ pages[0].reviewStatus = 'unverified';
+ pages[0].useTarget = true;
execFileSync('git',['init','-q'],{cwd:root});
execFileSync('git',['add','.'],{cwd:root});
execFileSync('git',['-c','user.name=T04
Test','-c','[email protected]','commit','-qm','candidate
scope'],{cwd:root});
@@ -102,6 +106,7 @@ test('full audit finds documents and sidebar keys added
after the T03 snapshot',
const report = await audit({root,sourceRef:'HEAD',baseline,native});
assert.deepEqual(report.groups.map(group => group.counts.new),[1,1,1]);
assert.deepEqual(report.groups.map(group =>
group.counts.pending),[1,1,1]);
+ assert.equal(report.groups[0].entries[0].state,'unverified');
assert.equal(report.labels.missing,3);
assert.equal(report.removed.length,0);
} finally {fs.rmSync(root,{recursive:true,force:true});}
diff --git a/scripts/i18n/audit-coverage.cjs b/scripts/i18n/audit-coverage.cjs
index 6ffef07052c..cbf1af88419 100644
--- a/scripts/i18n/audit-coverage.cjs
+++ b/scripts/i18n/audit-coverage.cjs
@@ -72,11 +72,12 @@ async function audit({root,sourceRef,baseline =
baselineDir,native}) {
seen.add(page.englishPath); current.add(key(page));
const previous = old.get(key(page));
const disposition = page.draft ? 'draft' :
scope.retainEnglish.some(entry => key(entry) === key(page)) ? 'retain_english'
: 'translate';
- const state = disposition !== 'translate' ? disposition :
page.status === 'untranslated' ? 'untranslated'
- : page.status === 'current' && page.reviewStatus !==
'unverified' && page.useTarget
+ const mergedCurrent = page.status === 'current' &&
page.reviewStatus !== 'unverified' && page.useTarget
&& refContent(root,commit,page.targetPath) !== null
&& hash(refContent(root,commit,page.targetPath)) ===
page.targetHash
- &&
refContent(root,commit,'translations/state/ja/docs.json') !== null ? 'current'
: page.status;
+ &&
refContent(root,commit,'translations/state/ja/docs.json') !== null;
+ const state = disposition !== 'translate' ? disposition :
page.status === 'current'
+ ? mergedCurrent ? 'current' : 'unverified' : page.status;
entries.push({pluginId:page.pluginId,version:page.version,docId:page.docId,sourcePath:page.sourcePath,
targetPath:page.targetPath,sourceHash:page.sourceHash,targetHash:page.targetHash,
sourceCommit:commit,permalink:page.permalink,reviewStatus:page.reviewStatus,state,
@@ -101,7 +102,7 @@ async function audit({root,sourceRef,baseline =
baselineDir,native}) {
.map(state => [state,sidebar.filter(entry => entry.state ===
state).length]));
const routingInputs =
['docusaurus.config.js','versions.json','sidebars.ts','sidebarsCommunity.json','versioned_sidebars/version-4.x-sidebars.json',
'translations/config.json','translations/scope/ja.json'];
- routingInputs.push('translations/state/ja/docs.json');
+ routingInputs.push('translations/state/ja/docs.json',
'translations/state/ja/resolved-deletions.json');
for (const file of routingInputs)
assert.equal(refContent(root,commit,file),fs.readFileSync(path.join(root,file),'utf8'),
`Routing input differs from source ref: ${file}`);
return
{schemaVersion:1,sourceCommit:commit,baselineCommit:JSON.parse(fs.readFileSync(path.join(baseline,'summary.json'),'utf8')).sourceCommit,
diff --git a/scripts/i18n/plan-translations-baseline.cjs
b/scripts/i18n/plan-translations-baseline.cjs
new file mode 100644
index 00000000000..14cedf3b35d
--- /dev/null
+++ b/scripts/i18n/plan-translations-baseline.cjs
@@ -0,0 +1,13 @@
+const fs = require('node:fs');
+const path = require('node:path');
+
+// T03 inventory is a historical comparison only. Never use it to filter the
+// native current-page enumeration.
+function baselineEntries(directory = path.resolve(__dirname,
'../../translations/inventory/ja/2026-10-12')) {
+ return ['dev', '4x', 'community'].flatMap(group => {
+ const metadata = JSON.parse(fs.readFileSync(path.join(directory,
`${group}.json`), 'utf8'));
+ return metadata.entryFiles.flatMap(shard =>
JSON.parse(fs.readFileSync(path.join(directory, shard), 'utf8')).entries);
+ });
+}
+
+module.exports = {baselineEntries};
diff --git a/scripts/i18n/plan-translations.cjs
b/scripts/i18n/plan-translations.cjs
new file mode 100644
index 00000000000..dae3f445c02
--- /dev/null
+++ b/scripts/i18n/plan-translations.cjs
@@ -0,0 +1,261 @@
+// Offline, read-only planner. The native audit supplies the complete current
+// scope; the T03 inventory is historical evidence, never an allowlist.
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const {createHash} = require('node:crypto');
+const {execFileSync} = require('node:child_process');
+const {audit} = require('./audit-coverage.cjs');
+const {baselineEntries} = require('./plan-translations-baseline.cjs');
+const {hash, identity, loadConfig, loadRecords} = require('./state.cjs');
+const {structure} = require('./structure.cjs');
+
+const compare = (a, b) => a < b ? -1 : a > b ? 1 : 0;
+const key = page => identity({...page, locale: 'ja'});
+const sha = value => /^[a-f0-9]{64}$/.test(value ?? '');
+const priority = {delete: 0, move: 1, move_and_update: 1, update: 2,
formatting_review: 2,
+ target_modified: 2, state_conflict: 2, source_divergence: 2,
unsupported_route: 2,
+ translate_new: 3, translate_backlog: 4};
+const managed = (scope, item) => scope.groups.find(group => group.pluginId ===
item.pluginId && group.version === item.version);
+const taskId = item => {
+ const seed = JSON.stringify([item.pluginId, item.version, item.docId,
item.locale, item.kind,
+ item.sourceHash ?? '', item.previousDocId ?? '']);
+ const number =
BigInt(`0x${createHash('sha256').update(seed).digest('hex').slice(0,
12)}`).toString().padStart(15, '0');
+ return `T06-${item.groupId}-${number}`;
+};
+const estimate = source => Math.max(1, Math.ceil((source?.length ?? 0) / 4));
+const chinesePath = (group, page) => {
+ if (!group.sourceRoot ||
!page.sourcePath.startsWith(`${group.sourceRoot}/`)) return null;
+ const plugin = group.pluginId === 'default' ? '' : `-${group.pluginId}`;
+ const version = group.version === 'current' ? 'current' :
`version-${group.version}`;
+ return
`i18n/zh-CN/docusaurus-plugin-content-docs${plugin}/${version}/${page.sourcePath.slice(group.sourceRoot.length
+ 1)}`;
+};
+
+function validateActive(data, scope) {
+ if (!data) return [];
+ assert(data.schemaVersion === 1 && Array.isArray(data.candidates),
'Invalid active candidate schema');
+ const seen = new Set();
+ return data.candidates.map(candidate => {
+ assert(candidate.locale === scope.locale && managed(scope, candidate),
'Active candidate outside managed scope');
+ assert(typeof candidate.docId === 'string' && candidate.docId &&
sha(candidate.sourceHash)
+ && typeof candidate.sourcePath === 'string' &&
candidate.sourcePath, 'Invalid active candidate identity/path/hash');
+ assert(typeof candidate.pr === 'string' && candidate.pr.trim(),
'Active candidate needs a PR reference');
+ const id = key(candidate);
+ assert(!seen.has(id), `Duplicate active candidate: ${id}`);
+ seen.add(id);
+ return candidate;
+ });
+}
+
+function validateResolved(data) {
+ assert(data.schemaVersion === 1 && Array.isArray(data.deletions), 'Invalid
resolved deletion schema');
+ const ids = new Set();
+ for (const item of data.deletions) {
+ assert(/^T06-[A-Z0-9]+-[0-9]{15}$/.test(item.id ?? '') &&
sha(item.sourceHash)
+ && typeof item.sourcePath === 'string' && item.sourcePath
+ && typeof item.reviewedBy === 'string' && item.reviewedBy.trim()
+ && typeof item.evidence === 'string' && item.evidence.trim()
+ && Number.isFinite(Date.parse(item.resolvedAt)), 'Invalid resolved
deletion');
+ assert(!ids.has(item.id), `Duplicate resolved deletion: ${item.id}`);
+ ids.add(item.id);
+ }
+ return data.deletions;
+}
+
+async function formattingOnly(before, after, filename) {
+ if (before == null || after == null) return false;
+ try {
+ const extension = path.extname(filename);
+ const [old, current] = await Promise.all([structure(before,
extension), structure(after, extension)]);
+ return old.semanticHash === current.semanticHash;
+ } catch { return false; }
+}
+
+function referenceChecks(oldPage, newPage) {
+ return {
+ oldSourcePath: oldPage?.sourcePath ?? null,
+ oldTargetPath: oldPage?.targetPath ?? null,
+ oldPermalink: oldPage?.permalink ?? null,
+ newSourcePath: newPage?.sourcePath ?? null,
+ newTargetPath: newPage?.targetPath ?? null,
+ newPermalink: newPage?.permalink ?? null,
+ actions: ['check inbound Markdown/MDX links and heading anchors',
'check sidebar references and redirects',
+ 'remove or relocate obsolete Japanese target and state record'],
+ };
+}
+
+async function plan({report, scope, records, baseline = [], active,
readCurrent, readAt,
+ resolved = {schemaVersion: 1, deletions: []}, chineseChangedPaths = [],
+ limit = scope.maxBatchPages, tokenBudget = 50000, filters = {}}) {
+ assert(report.schemaVersion === 1 &&
/^[a-f0-9]{40}$/.test(report.sourceCommit), 'Invalid source audit');
+ assert(scope.schemaVersion === 1 && scope.locale === 'ja' &&
Number.isInteger(scope.maxBatchPages), 'Invalid managed scope');
+ assert(Number.isInteger(limit) && limit >= 0 && limit <=
scope.maxBatchPages, 'Limit exceeds managed batch size');
+ assert(Number.isInteger(tokenBudget) && tokenBudget >= 0, 'Invalid token
budget');
+ if (filters.locale && filters.locale !== scope.locale) throw new
Error(`Unsupported locale: ${filters.locale}`);
+ const candidates = validateActive(active, scope);
+ const resolvedDeletions = validateResolved(resolved);
+ const activeByKey = new Map(candidates.map(candidate => [key(candidate),
candidate]));
+ const baselineByKey = new Map(baseline.filter(page => managed(scope,
page)).map(page => [key(page), page]));
+ const old = new Map(baselineByKey);
+ for (const record of records.values()) if (managed(scope, record))
old.set(key(record), {...old.get(key(record)), ...record});
+ const pages = report.groups.filter(group => managed(scope,
group)).flatMap(group =>
+ group.entries.filter(page => page.state !== 'draft' && page.state !==
'retain_english')
+ .map(page => ({...page, groupId: group.id})));
+ const current = new Set(pages.map(key));
+ assert.equal(current.size, pages.length, 'Duplicate current identity');
+ const removed = [...old.values()].filter(page => !current.has(key(page))
&& page.disposition !== 'draft'
+ && page.disposition !== 'retain_english').map(page => ({...page,
groupId: managed(scope, page).id}));
+ const matchedRemoval = new Set();
+ const tasks = [];
+ const exceptions = report.groups.flatMap(group =>
group.unregistered.map(sourcePath => ({kind: 'unregistered_file',
+ groupId: group.id, sourcePath}))).concat(report.reports.filter(item =>
managed(scope, item)));
+ const changedChinese = new Set(chineseChangedPaths);
+ for (const page of pages) {
+ const previous = old.get(key(page));
+ const group = managed(scope, page);
+ const chinese = chinesePath(group, page);
+ if (chinese && changedChinese.has(chinese) &&
baselineByKey.get(key(page))?.sourceHash === page.sourceHash) {
+ exceptions.push({kind: 'zh_only_change', groupId: page.groupId,
docId: page.docId,
+ sourcePath: page.sourcePath, chinesePath: chinese,
+ reason: 'English source is unchanged since the inventory
baseline; check terminology and behavior consistency.'});
+ }
+ if (page.state === 'current') continue;
+ const source = readCurrent(page.sourcePath);
+ assert(source !== null && hash(source) === page.sourceHash, `Current
source/hash mismatch: ${page.sourcePath}`);
+ let moveFrom = previous?.sourcePath !== page.sourcePath ? previous :
null;
+ if (!previous || !moveFrom) {
+ // A new docId can still be a move, but only if one removed page in
+ // the same plugin/version has the exact source hash and path
differs.
+ if (!previous) {
+ const matches = removed.filter(item =>
!matchedRemoval.has(key(item)) && item.pluginId === page.pluginId
+ && item.version === page.version && item.sourceHash ===
page.sourceHash && item.sourcePath !== page.sourcePath);
+ const destinations = pages.filter(item => item.pluginId ===
page.pluginId && item.version === page.version
+ && item.sourceHash === page.sourceHash &&
!old.has(key(item)));
+ if (matches.length === 1 && destinations.length === 1)
moveFrom = matches[0];
+ }
+ }
+ if (moveFrom && !current.has(key(moveFrom)))
matchedRemoval.add(key(moveFrom));
+ const candidate = activeByKey.get(key(page));
+ const activeValid = candidate?.sourceHash === page.sourceHash &&
candidate.sourcePath === page.sourcePath;
+ if (candidate && !activeValid) exceptions.push({kind:
'stale_active_candidate', groupId: page.groupId,
+ docId: page.docId, pr: candidate.pr, candidateHash:
candidate.sourceHash, sourceHash: page.sourceHash});
+ let kind;
+ if (['source_missing', 'source_route_conflict'].includes(page.state))
kind = page.state === 'source_missing' ? 'source_divergence' :
'unsupported_route';
+ else if (page.state === 'target_modified') kind = 'target_modified';
+ else if (moveFrom) kind = moveFrom.sourceHash === page.sourceHash ?
'move' : 'move_and_update';
+ else if (page.state === 'state_conflict' || page.state ===
'unverified') kind = 'state_conflict';
+ else if (page.state.startsWith('stale_')) {
+ const historical = previous?.sourcePath
+ ? readAt(previous.sourceCommit ?? report.baselineCommit,
previous.sourcePath) : null;
+ kind = historical && hash(historical) === previous.sourceHash
+ && await formattingOnly(historical, source, page.sourcePath) ?
'formatting_review' : 'update';
+ } else kind = previous ? 'translate_backlog' : 'translate_new';
+ const protectedTask = ['target_modified', 'state_conflict',
'source_divergence', 'unsupported_route', 'formatting_review'].includes(kind);
+ const task = {groupId: page.groupId, pluginId: page.pluginId, version:
page.version, docId: page.docId,
+ locale: scope.locale, kind, sourcePath: page.sourcePath,
targetPath: page.targetPath,
+ permalink: page.permalink, sourceHash: page.sourceHash,
sourceCommit: report.sourceCommit,
+ targetHash: page.targetHash, state: page.state, reviewStatus:
page.reviewStatus,
+ estimatedSourceTokens: estimate(source), requiresHumanReview:
protectedTask,
+ sourceChange: page.change,
+ activePr: activeValid ? candidate.pr : null,
+ ...(moveFrom ? {previousDocId: moveFrom.docId, reuseTarget:
moveFrom.sourceHash === page.sourceHash
+ && !!moveFrom.targetHash && !!moveFrom.targetPath
+ && readCurrent(moveFrom.targetPath) !== null &&
hash(readCurrent(moveFrom.targetPath)) === moveFrom.targetHash,
+ referenceChecks: referenceChecks(moveFrom, page)} : {})};
+ task.id = taskId(task);
+ tasks.push(task);
+ }
+ for (const page of removed.filter(item => !matchedRemoval.has(key(item))))
{
+ const task = {groupId: page.groupId, pluginId: page.pluginId, version:
page.version,
+ docId: page.docId, locale: scope.locale, kind: 'delete',
sourcePath: page.sourcePath,
+ targetPath: page.targetPath ?? null, sourceHash: page.sourceHash
?? null, sourceCommit: report.sourceCommit,
+ estimatedSourceTokens: 0, requiresHumanReview: true, activePr:
null,
+ referenceChecks: referenceChecks(page, null)};
+ task.id = taskId(task);
+ if (!resolvedDeletions.some(item => item.id === task.id &&
item.sourceHash === task.sourceHash
+ && item.sourcePath === task.sourcePath)) tasks.push(task);
+ }
+ const groupOrder = new Map(scope.groups.map((group, index) => [group.id,
index]));
+ tasks.sort((a, b) => priority[a.kind] - priority[b.kind]
+ || groupOrder.get(a.groupId) - groupOrder.get(b.groupId)
+ || compare(a.sourcePath, b.sourcePath) || compare(a.id, b.id));
+ const allCounts = {
+ required: pages.length,
+ mergedCurrent: pages.filter(page => page.state === 'current').length,
+ pending: pages.filter(page => page.state !== 'current').length,
+ untranslated: pages.filter(page => page.state ===
'untranslated').length,
+ needsUpdate: pages.filter(page => page.state !== 'current' &&
page.state !== 'untranslated').length,
+ deletions: tasks.filter(task => task.kind === 'delete').length,
+ moves: tasks.filter(task => task.kind === 'move' || task.kind ===
'move_and_update').length,
+ activeCandidates: tasks.filter(task => task.activePr).length,
+ };
+ assert.equal(allCounts.required, allCounts.mergedCurrent +
allCounts.pending);
+ const selectedTasks = tasks.filter(task => (!filters.plugin ||
task.pluginId === filters.plugin)
+ && (!filters.version || task.version === filters.version)
+ && (!filters.path || task.sourcePath.startsWith(filters.path) ||
task.targetPath?.startsWith(filters.path)));
+ const selected = [];
+ let tokens = 0;
+ for (const task of selectedTasks) {
+ if (selected.length >= limit) break;
+ if (task.activePr || task.requiresHumanReview) continue;
+ if (tokens + task.estimatedSourceTokens > tokenBudget) continue;
+ selected.push(task.id);
+ tokens += task.estimatedSourceTokens;
+ }
+ return {schemaVersion: 1, locale: scope.locale, sourceCommit:
report.sourceCommit,
+ baselineCommit: report.baselineCommit, counts: allCounts,
selectedCount: selected.length,
+ selectedEstimatedSourceTokens: tokens, selected, tasks: selectedTasks,
+ exceptions: exceptions.sort((a, b) => compare(JSON.stringify(a),
JSON.stringify(b)))};
+}
+
+function argumentsFrom(argv) {
+ const options = {}, known = new Set(['--source-ref', '--output',
'--active-candidates', '--locale', '--plugin',
+ '--version', '--path', '--limit', '--token-budget']);
+ for (let i = 0; i < argv.length; i += 2) {
+ assert(known.has(argv[i]) && argv[i + 1] && !options[argv[i]],
`Unknown or incomplete option: ${argv[i]}`);
+ options[argv[i]] = argv[i + 1];
+ }
+ return options;
+}
+
+async function main() {
+ const options = argumentsFrom(process.argv.slice(2));
+ const root = path.resolve(__dirname, '../..');
+ const scope = JSON.parse(fs.readFileSync(path.join(root,
'translations/scope/ja.json'), 'utf8'));
+ const config = loadConfig(root);
+ assert(config.locales[scope.locale]?.sourceLocale === 'en', 'Planner
requires an English primary source');
+ const report = await audit({root, sourceRef: options['--source-ref'] ??
'HEAD'});
+ const chineseRoots = ['i18n/zh-CN/docusaurus-plugin-content-docs',
'i18n/zh-CN/docusaurus-plugin-content-docs-community'];
+ execFileSync('git', ['diff', '--quiet', report.sourceCommit, '--',
...chineseRoots], {cwd: root});
+ const untrackedChinese = execFileSync('git', ['ls-files', '--others',
'--exclude-standard', '--', ...chineseRoots],
+ {cwd: root, encoding: 'utf8'}).trim();
+ assert(!untrackedChinese, `Uncommitted Chinese counterpart:
${untrackedChinese}`);
+ const chineseChangedPaths = execFileSync('git', ['diff', '--name-only',
report.baselineCommit, report.sourceCommit, '--', ...chineseRoots],
+ {cwd: root, encoding: 'utf8'}).trim().split('\n').filter(Boolean);
+ const records = loadRecords(root, scope.locale);
+ const active = options['--active-candidates'] ?
JSON.parse(fs.readFileSync(options['--active-candidates'], 'utf8')) : null;
+ const resolved = JSON.parse(fs.readFileSync(path.join(root,
`translations/state/${scope.locale}/resolved-deletions.json`), 'utf8'));
+ const readCurrent = file => {
+ try { return fs.readFileSync(path.join(root, file), 'utf8'); } catch
(error) {
+ if (error.code === 'ENOENT') return null;
+ throw error;
+ }
+ };
+ const readAt = (ref, file) => {
+ try { return execFileSync('git', ['show', `${ref}:${file}`],
+ {cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'],
maxBuffer: 16 * 1024 * 1024}); }
+ catch { return null; }
+ };
+ const result = await plan({report, scope, records, baseline:
baselineEntries(), active, resolved,
+ chineseChangedPaths, readCurrent, readAt,
+ limit: options['--limit'] === undefined ? scope.maxBatchPages :
Number(options['--limit']),
+ tokenBudget: options['--token-budget'] === undefined ? 50000 :
Number(options['--token-budget']),
+ filters: {locale: options['--locale'], plugin: options['--plugin'],
version: options['--version'], path: options['--path']}});
+ const output = JSON.stringify(result, null, 2) + '\n';
+ if (options['--output']) fs.writeFileSync(options['--output'], output);
+ else process.stdout.write(output);
+ process.stderr.write(`Japanese plan ${result.sourceCommit}:
${JSON.stringify(result.counts)}, selected ${result.selectedCount}\n`);
+}
+
+if (require.main === module) main().catch(error => {console.error(error);
process.exitCode = 1;});
+module.exports = {plan, validateActive, validateResolved, formattingOnly,
argumentsFrom};
diff --git a/scripts/i18n/structure.cjs b/scripts/i18n/structure.cjs
index 25f0a549dc0..27fa8d37636 100644
--- a/scripts/i18n/structure.cjs
+++ b/scripts/i18n/structure.cjs
@@ -63,6 +63,10 @@ async function structure(raw, extension, anchorsMaintainCase
= false) {
return {
content, headings, format, protectedNodes, technicalMetadata,
technicalHash: hash(JSON.stringify([technicalMetadata, protectedNodes,
headings.map(({depth, id}) => ({depth, id}))])),
+ // A secondary comparison for planning only. The state sourceHash stays
+ // byte-sensitive, so a formatting edit still needs an explicit review.
+ semanticHash:
hash(JSON.stringify([Object.fromEntries(Object.entries(data).sort(([a], [b]) =>
a.localeCompare(b))),
+ withoutPositions(tree)])),
};
}
diff --git a/translations/README.md b/translations/README.md
index 9724949bc9a..fdf1266e245 100644
--- a/translations/README.md
+++ b/translations/README.md
@@ -114,6 +114,76 @@ The candidate scope covers all formal Dev, 4.x and
community documents. See
[initial complete batch ledger](inventory/ja/2026-10-12/batches.md). Scope
limits
translation candidates, not the native website build or existing fallback
routes.
+## Offline Japanese task planner
+
+The planner recounts the native Dev, 4.x and community pages at the checked-out
+commit. The 2026-10-12 inventory is only a comparison baseline. New registered
+pages enter the plan even when they have never appeared in that inventory.
+
+```bash
+node scripts/i18n/plan-translations.cjs --source-ref HEAD \
+ --limit 20 --token-budget 50000 --output /tmp/japanese-plan.json
+```
+
+The source ref must match the checked-out source, routing files, Japanese state
+and targets. The scanner fails on mismatches instead of counting a working-tree
+candidate as merged coverage. It needs no model credential, network or GitHub
+write permission. Without `--output`, JSON goes to stdout; a compact count goes
+to stderr. `--locale ja`, `--plugin default`, `--version current` and
+`--path docs/install/` filter visible tasks after the full recount. The limit
+cannot exceed `translations/scope/ja.json`'s `maxBatchPages` (20). The token
+budget uses an approximate source length of four characters per token, so a
+generation tool must check its actual model budget again.
+
+`counts` cover the full managed scope. `tasks` list pending work, while
+`selected` holds the next eligible task IDs within the page and token limits.
+Stable numeric IDs derive from the page identity, task kind and source hash;
+they do not renumber existing T09 batches. Updates, deletion and moves precede
+historical backlog. Formatting-only changes still require review of the exact
+new source hash. Target edits, invalid state, missing sources and unsupported
+routes are reported as protected tasks and never enter `selected`. Move tasks
+include old/new paths and a reference checklist; `reuseTarget` is true only
+when the old target's content hash matches its recorded hash. No plan changes
+translation state or declares a page translated.
+
+To avoid assigning work already in an open human or bot PR, pass a local JSON
+file with the reviewed PR candidates:
+
+```json
+{
+ "schemaVersion": 1,
+ "candidates": [
+ {
+ "pluginId": "default",
+ "version": "current",
+ "docId": "getting-started/what-is-apache-doris",
+ "locale": "ja",
+ "sourcePath": "docs/getting-started/what-is-apache-doris.md",
+ "sourceHash": "<64 lowercase hex digits>",
+ "pr": "#1234"
+ }
+ ]
+}
+```
+
+Use `--active-candidates /path/to/open-prs.json`. A matching source hash marks
+the task as active and removes it from `selected`, but does not add coverage.
+A source change makes the PR candidate stale and requeues the page. The planner
+does not query GitHub: a caller that wants complete PR deduplication must
supply
+all active candidate PRs, including manually opened T09 PRs.
+
+The plan also reports `zh_only_change` exceptions when a committed Chinese
+counterpart changed since the T03 baseline while its English fact source did
+not. These are consistency checks, never Japanese translation candidates.
+
+After a deletion cleanup PR has removed or checked the Japanese target, state,
+links, sidebars and redirects, record the completed task in
+`state/ja/resolved-deletions.json` with its ID, source hash, source path,
+`reviewedBy`, ISO `resolvedAt` and PR `evidence`. This separate schema 1 ledger
+prevents the fixed T03 baseline from reopening an already reviewed deletion.
+The live docs adapter continues to read `state/ja/docs.json` unchanged. A
+reintroduced page returns to normal native enumeration.
+
## Validation
```bash
diff --git a/translations/state/ja/resolved-deletions.json
b/translations/state/ja/resolved-deletions.json
new file mode 100644
index 00000000000..22890804fd1
--- /dev/null
+++ b/translations/state/ja/resolved-deletions.json
@@ -0,0 +1,4 @@
+{
+ "schemaVersion": 1,
+ "deletions": []
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
