This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-website.git
The following commit(s) were added to refs/heads/master by this push:
new bf484b3db93 feat(i18n): shard Japanese state and reconcile open PRs
(#4248)
bf484b3db93 is described below
commit bf484b3db93d65bfd6a66e4ad8044be8dc0936e0
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Wed Oct 14 08:36:21 2026 +0800
feat(i18n): shard Japanese state and reconcile open PRs (#4248)
## Summary
Store Japanese translation state in one file per document at
`translations/state/ja/pages/<pluginId>/<version>/<docId>.json`. The
state loader combines these files with the existing `docs.json` ledger,
and a page file supersedes its legacy entry. Independent translation PRs
can therefore add disjoint state paths.
Update the planner, coverage and inventory audits, PR validation,
open-PR reconciliation, and build-scope detection to recognize page
files. Keep `docs.json` readable during migration.
Also include the fix from #4246: compare each open PR's state changes
with its own Git merge base. Older PRs no longer appear to remove
translations that merged into master later. Reconciliation also handles
an existing page file when a PR removes its superseded legacy record.
## Validation
- `node --test scripts/i18n/__tests__/*.test.cjs`: 73 passed.
- `node scripts/i18n/audit-coverage.cjs --source-ref HEAD`: passed;
Japanese coverage counts remain unchanged by this compatibility layer.
- Read-only reconciliation of current open PRs: 107 inspected, 161
active candidates, 0 blocked targets, 0 exceptions.
- Japanese locale build passed for the shard foundation before the
reconciliation-only additions.
- `git diff --check upstream-apache/master...HEAD`: passed.
## Migration note
Already-open translation PRs that still edit `docs.json` can still
conflict with each other. They need separate conversion to page files or
a state merge after this PR lands. New page-file PRs are independent at
the state-file level.
---------
Co-authored-by: morningman <[email protected]>
---
scripts/i18n/__tests__/state.test.cjs | 55 +++++++++++++++++++++++++++-
scripts/i18n/__tests__/t04.test.cjs | 5 +++
scripts/i18n/__tests__/workflow.test.cjs | 51 +++++++++++++++++++++++++-
scripts/i18n/audit-coverage.cjs | 9 ++++-
scripts/i18n/build-scope.cjs | 6 ++-
scripts/i18n/check-translations.cjs | 14 +++----
scripts/i18n/inventory.cjs | 8 ++--
scripts/i18n/state.cjs | 57 +++++++++++++++++++++++++++--
scripts/i18n/workflow-prs.cjs | 63 ++++++++++++++++++++++++++------
translations/README.md | 11 ++++--
translations/state/ja/shards.schema.json | 4 ++
11 files changed, 247 insertions(+), 36 deletions(-)
diff --git a/scripts/i18n/__tests__/state.test.cjs
b/scripts/i18n/__tests__/state.test.cjs
index 66fec3d4572..502ba009e5a 100644
--- a/scripts/i18n/__tests__/state.test.cjs
+++ b/scripts/i18n/__tests__/state.test.cjs
@@ -1,6 +1,10 @@
const test = require('node:test');
const assert = require('node:assert/strict');
-const {hash, identity, decide} = require('../state.cjs');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const {execFileSync} = require('node:child_process');
+const {hash, identity, decide, loadRecords, loadRecordsAt, stateFileFor} =
require('../state.cjs');
const {structure, alignHeadings} = require('../structure.cjs');
test('Japanese version messages match the custom banner interpolation
contract', () => {
@@ -102,6 +106,55 @@ test('source hashing includes SQL, parameters, MDX,
metadata and whitespace, nor
assert.notEqual(hash(original), hash(original + '\n'));
});
+test('page shards override legacy state without changing other identities',
async () => {
+ const data = await setup();
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ja-state-shards-'));
+ try {
+ const stateDir = path.join(root, 'translations/state/ja');
+ fs.mkdirSync(stateDir, {recursive: true});
+ fs.writeFileSync(path.join(stateDir, 'docs.json'),
JSON.stringify({schemaVersion: 1,
+ pages: [data.record, {...data.record, docId: 'other'}]}));
+ const file = stateFileFor(data.record);
+ fs.mkdirSync(path.dirname(path.join(root, file)), {recursive: true});
+ fs.writeFileSync(path.join(root, file), JSON.stringify({schemaVersion:
1,
+ page: {...data.record, translatedAt: '2026-10-13T00:00:00Z'}}));
+ const records = loadRecords(root, 'ja');
+ assert.equal(records.size, 2);
+ assert.equal(records.get(identity(data.record)).translatedAt,
'2026-10-13T00:00:00Z');
+ assert(records.has(identity({...data.record, docId: 'other'})));
+ fs.writeFileSync(path.join(root, file), JSON.stringify({schemaVersion:
1,
+ page: {...data.record, docId: '../outside'}}));
+ assert.throws(() => loadRecords(root, 'ja'), /Invalid translation
state path/);
+ } finally {
+ fs.rmSync(root, {recursive: true, force: true});
+ }
+});
+
+test('committed page shards are read from the selected Git ref', async () => {
+ const data = await setup();
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ja-state-ref-'));
+ const git = (...args) => execFileSync('git', args, {cwd: root, stdio:
'ignore'});
+ try {
+ git('init', '-q');
+ git('config', 'user.name', 'State Test');
+ git('config', 'user.email', '[email protected]');
+ const stateDir = path.join(root, 'translations/state/ja');
+ fs.mkdirSync(stateDir, {recursive: true});
+ fs.writeFileSync(path.join(stateDir, 'docs.json'),
JSON.stringify({schemaVersion: 1, pages: []}));
+ const file = stateFileFor(data.record);
+ fs.mkdirSync(path.dirname(path.join(root, file)), {recursive: true});
+ fs.writeFileSync(path.join(root, file), JSON.stringify({schemaVersion:
1, page: data.record}));
+ git('add', '.');
+ git('commit', '-qm', 'state');
+ assert.equal(loadRecordsAt(root, 'ja',
'HEAD').get(identity(data.record)).targetHash, data.record.targetHash);
+ fs.writeFileSync(path.join(root, file), JSON.stringify({schemaVersion:
1,
+ page: {...data.record, targetHash: 'b'.repeat(64)}}));
+ assert.equal(loadRecordsAt(root, 'ja',
'HEAD').get(identity(data.record)).targetHash, data.record.targetHash);
+ } finally {
+ fs.rmSync(root, {recursive: true, force: true});
+ }
+});
+
test('heading alignment follows native duplicate IDs, explicit IDs, setext and
fenced code', async () => {
const source = '---\ntitle: Heading\n---\n## Usage\n## Usage\n## Fixed
{#CaseID}\n\nSetext\n------\n\n```md\n## Not a heading\n```\n';
const japanese = '---\ntitle: 見出し\n---\n## 使用方法\n## 使用方法\n##
固定\n\nセクション\n------\n\n```md\n## Not a heading\n```\n';
diff --git a/scripts/i18n/__tests__/t04.test.cjs
b/scripts/i18n/__tests__/t04.test.cjs
index 0e62bb9dd2e..1ecb37e35b5 100644
--- a/scripts/i18n/__tests__/t04.test.cjs
+++ b/scripts/i18n/__tests__/t04.test.cjs
@@ -17,6 +17,11 @@ test('PR build scope separates content locales, shared
inputs, and retired versi
assert.deepEqual(japanese.locales,['ja']);
assert.equal(japanese.docs_versions,'4.x');
assert.equal(japanese.enable_ja,true);
+ const shard = 'translations/state/ja/pages/default/4.x/guide.json';
+ const stateOnly = buildScope([shard],versions);
+ assert.deepEqual(stateOnly.locales,['ja']);
+ assert.equal(stateOnly.docs_versions,'4.x');
+ assert.deepEqual(stateOnly.json,[shard]);
assert.deepEqual(buildScope(['sidebars.ts'],versions).locales,['en','zh-CN','ja']);
assert.deepEqual(buildScope(['.github/workflows/cron-deploy-website.yml'],versions).locales,['en','zh-CN','ja']);
assert.deepEqual(buildScope(['.github/workflows/manual-deploy-website.yml'],versions).locales,['en','zh-CN','ja']);
diff --git a/scripts/i18n/__tests__/workflow.test.cjs
b/scripts/i18n/__tests__/workflow.test.cjs
index 5d9bbaa4f03..7b31fd4ba77 100644
--- a/scripts/i18n/__tests__/workflow.test.cjs
+++ b/scripts/i18n/__tests__/workflow.test.cjs
@@ -4,7 +4,7 @@ const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const {execFileSync} = require('node:child_process');
-const {hash} = require('../state.cjs');
+const {hash, stateFileFor} = require('../state.cjs');
const {reconcileOpenPrs} = require('../workflow-prs.cjs');
const {relevantPush, replaceableBotPr} = require('../workflow-scan.cjs');
const {pairFor, machineOwned, generate, allowGeneration} =
require('../workflow-generate.cjs');
@@ -33,6 +33,55 @@ test('all open PRs are inspected by files, including unnamed
human PRs', async (
assert.equal(result.blockedTargets.length, 0);
});
+test('an open PR can register its target with a page shard instead of
docs.json', async () => {
+ const shardPath = stateFileFor(record);
+ const result = await reconcileOpenPrs({scope, repository:
'apache/doris-website', pulls: [pull(13)],
+ filesFor: async () => [{filename: record.targetPath}, {filename:
shardPath}],
+ readHead: async (_item, file) => file === shardPath
+ ? JSON.stringify({schemaVersion: 1, page: record})
+ : file.endsWith('docs.json') ? JSON.stringify({schemaVersion: 1,
pages: []}) : japanese,
+ readBase: async (_item, file) => file === shardPath ? null
+ : JSON.stringify({schemaVersion: 1, pages: []})});
+ assert.deepEqual(result.relevantPrs, [13]);
+ assert.equal(result.active.candidates[0].pr, '#13');
+ assert.deepEqual(result.blockedTargets, []);
+});
+
+test('state changes are compared with the PR merge base, not current master',
async () => {
+ const previous = {...record, docId: 'previous', sourcePath:
'docs/previous.md',
+ targetPath:
'i18n/ja/docusaurus-plugin-content-docs/current/previous.md'};
+ let baseReads = 0;
+ const result = await reconcileOpenPrs({scope, repository:
'apache/doris-website',
+ pulls: [pull(13)],
+ filesFor: async () => [{filename: record.targetPath}, {filename:
'translations/state/ja/docs.json'}],
+ readHead: async (_item, file) => file.includes('/pages/') ? null
+ : file.endsWith('docs.json') ? JSON.stringify({schemaVersion: 1,
pages: [previous, record]}) : japanese,
+ readBase: async (_item, file) => {
+ if (file.includes('/pages/')) return null;
+ baseReads++;
+ return JSON.stringify({schemaVersion: 1, pages: [previous]});
+ }});
+ assert.equal(baseReads, 1);
+ assert.deepEqual(result.relevantPrs, [13]);
+ assert.deepEqual(result.active.candidates.map(candidate =>
candidate.docId), ['guide']);
+ assert.deepEqual(result.blockedTargets, []);
+ assert.deepEqual(result.exceptions, []);
+});
+
+test('removing a legacy record already covered by a shard creates no
candidate', async () => {
+ const shardPath = stateFileFor(record);
+ const shard = JSON.stringify({schemaVersion: 1, page: record});
+ const result = await reconcileOpenPrs({scope, repository:
'apache/doris-website', pulls: [pull(14)],
+ filesFor: async () => [{filename: 'translations/state/ja/docs.json'}],
+ readHead: async (_item, file) => file === shardPath ? shard
+ : JSON.stringify({schemaVersion: 1, pages: []}),
+ readBase: async (_item, file) => file === shardPath ? shard
+ : JSON.stringify({schemaVersion: 1, pages: [record]})});
+ assert.deepEqual(result.active.candidates, []);
+ assert.deepEqual(result.blockedTargets, []);
+ assert.deepEqual(result.exceptions, []);
+});
+
test('edited or unparseable PR candidate blocks its target without claiming
coverage', async () => {
const result = await reconcileOpenPrs({scope, repository:
'apache/doris-website', pulls: [pull(12)],
filesFor: async () => [{filename: record.targetPath}],
diff --git a/scripts/i18n/audit-coverage.cjs b/scripts/i18n/audit-coverage.cjs
index cbf1af88419..f4203f15196 100644
--- a/scripts/i18n/audit-coverage.cjs
+++ b/scripts/i18n/audit-coverage.cjs
@@ -4,7 +4,7 @@ const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const {execFileSync} = require('node:child_process');
-const {hash, identity} = require('./state.cjs');
+const {hash, identity, loadRecordsAt} = require('./state.cjs');
const {collect, validateScope} = require('./inventory.cjs');
const baselineDir =
path.resolve(__dirname,'../../translations/inventory/ja/2026-10-12');
@@ -54,6 +54,7 @@ async function audit({root,sourceRef,baseline =
baselineDir,native}) {
assert(!untracked,`Uncommitted Japanese targets/state cannot count at
${commit}: ${untracked}`);
const scope =
JSON.parse(fs.readFileSync(path.join(root,'translations/scope/ja.json'),'utf8'));
validateScope(scope);
+ const committedRecords = loadRecordsAt(root, 'ja', commit);
native ||= await collect(root);
const old = new Map(baselineEntries(baseline).map(page =>
[key(page),page]));
const current = new Set(), groups = [];
@@ -75,7 +76,7 @@ async function audit({root,sourceRef,baseline =
baselineDir,native}) {
const mergedCurrent = page.status === 'current' &&
page.reviewStatus !== 'unverified' && page.useTarget
&& refContent(root,commit,page.targetPath) !== null
&& hash(refContent(root,commit,page.targetPath)) ===
page.targetHash
- &&
refContent(root,commit,'translations/state/ja/docs.json') !== null;
+ && committedRecords.get(key(page))?.targetHash ===
page.targetHash;
const state = disposition !== 'translate' ? disposition :
page.status === 'current'
? mergedCurrent ? 'current' : 'unverified' : page.status;
entries.push({pluginId:page.pluginId,version:page.version,docId:page.docId,sourcePath:page.sourcePath,
@@ -103,6 +104,10 @@ async function audit({root,sourceRef,baseline =
baselineDir,native}) {
const routingInputs =
['docusaurus.config.js','versions.json','sidebars.ts','sidebarsCommunity.json','versioned_sidebars/version-4.x-sidebars.json',
'translations/config.json','translations/scope/ja.json'];
routingInputs.push('translations/state/ja/docs.json',
'translations/state/ja/resolved-deletions.json');
+ if (refContent(root, commit, 'translations/state/ja/shards.schema.json')
!== null)
+ routingInputs.push('translations/state/ja/shards.schema.json');
+ routingInputs.push(...execFileSync('git', ['ls-tree', '-r', '--name-only',
commit, '--', 'translations/state/ja/pages'],
+ {cwd:root,encoding:'utf8'}).split('\n').filter(Boolean));
for (const file of routingInputs)
assert.equal(refContent(root,commit,file),fs.readFileSync(path.join(root,file),'utf8'),
`Routing input differs from source ref: ${file}`);
return
{schemaVersion:1,sourceCommit:commit,baselineCommit:JSON.parse(fs.readFileSync(path.join(baseline,'summary.json'),'utf8')).sourceCommit,
diff --git a/scripts/i18n/build-scope.cjs b/scripts/i18n/build-scope.cjs
index 5ea26f492b6..a13277b9de7 100644
--- a/scripts/i18n/build-scope.cjs
+++ b/scripts/i18n/build-scope.cjs
@@ -34,7 +34,11 @@ function buildScope(files, versions) {
else if (/^(src|config)\//.test(file) ||
['docusaurus.config.js','package.json','yarn.lock','tailwind.config.js','tsconfig.json'].includes(file)
|| /^scripts\/(key-features|last-update)\//.test(file) ||
/^scripts\/(patch-search-tokenize.js|build-locales.cjs|build-locale.cjs)$/.test(file)
||
/^scripts\/i18n\/(docs-plugin|status-plugin|state|structure)\.cjs$/.test(file))
shared();
- else if (['translations/config.json',
'translations/state/ja/docs.json'].includes(file)) {languages.add('ja');
build();}
+ else if (['translations/config.json',
'translations/state/ja/docs.json',
+ 'translations/state/ja/shards.schema.json'].includes(file))
{languages.add('ja'); build();}
+ else if ((match =
file.match(/^translations\/state\/ja\/pages\/[^/]+\/([^/]+)\//))) {
+ languages.add('ja'); wanted.add(match[1]); json.add(file); build();
+ }
else if (file === 'static/.htaccess') shared();
else if
(/^(static|\.github|\.claude|\.agents|\.vscode|\.idea|scripts|website-quality-governance|translations|plan-doc)\//.test(file)
|| /\.(py|sh)$/.test(file)
diff --git a/scripts/i18n/check-translations.cjs
b/scripts/i18n/check-translations.cjs
index f1fdc72b6d9..9536e102fa3 100644
--- a/scripts/i18n/check-translations.cjs
+++ b/scripts/i18n/check-translations.cjs
@@ -2,15 +2,17 @@
const fs = require('node:fs');
const path = require('node:path');
const {execFileSync} = require('node:child_process');
-const {identity, hash, loadRecords} = require('./state.cjs');
+const {identity, hash, loadRecords, loadRecordsAt} = require('./state.cjs');
const {structure, alignHeadings} = require('./structure.cjs');
const {collect} = require('./inventory.cjs');
const {protectionSignature} = require('./protection.cjs');
async function checkTranslations({root, pages, reports = [], files, records,
previousRecords = new Map()}) {
const changed = new Set(files);
+ const stateChanged = files.some(file => file ===
'translations/state/ja/docs.json'
+ || file.startsWith('translations/state/ja/pages/'));
const selected = pages.filter(page => changed.has(page.targetPath)
- || (changed.has('translations/state/ja/docs.json')
+ || (stateChanged
&& JSON.stringify(records.get(identity(page))) !==
JSON.stringify(previousRecords.get(identity(page)))));
const errors = [];
const exceptionFile =
path.join(root,'translations/protection-exceptions/ja.json');
@@ -54,7 +56,7 @@ async function checkTranslations({root, pages, reports = [],
files, records, pre
for (const file of files.filter(file =>
/^i18n\/ja\/docusaurus-plugin-content-docs[^/]*\/(current|version-[^/]+)\/.+\.(md|mdx)$/.test(file)))
{
if (fs.existsSync(path.join(root,file)) && !known.has(file))
errors.push({path:file, message:'Target has no registered source in the same
plugin/version'});
}
- if (changed.has('translations/state/ja/docs.json')) for (const [key,
record] of records) {
+ if (stateChanged) for (const [key, record] of records) {
if (JSON.stringify(record) !==
JSON.stringify(previousRecords.get(key)) && !pages.some(page => identity(page)
=== key)) {
errors.push({path:record.targetPath, message:'Edited state has no
native source identity'});
}
@@ -68,11 +70,7 @@ async function main() {
if (!args.includes('--base')) throw new Error('Usage:
check-translations.cjs --base <PR merge base>');
const files =
execFileSync('git',['diff','--name-only',base,'HEAD'],{cwd:root,encoding:'utf8'}).trim().split('\n').filter(Boolean);
const records = loadRecords(root,'ja');
- let previousRecords = new Map();
- try {
- const previous =
JSON.parse(execFileSync('git',['show',`${base}:translations/state/ja/docs.json`],{cwd:root,encoding:'utf8'}));
- previousRecords = new Map(previous.pages.map(record =>
[identity(record),record]));
- } catch (error) {if (!String(error.message).includes('does not exist'))
throw error;}
+ const previousRecords = loadRecordsAt(root, 'ja', base);
const native = await collect(root);
const result = await
checkTranslations({root,...native,files,records,previousRecords});
console.log(JSON.stringify(result,null,2));
diff --git a/scripts/i18n/inventory.cjs b/scripts/i18n/inventory.cjs
index ffe4a74a011..a83bcc47a33 100644
--- a/scripts/i18n/inventory.cjs
+++ b/scripts/i18n/inventory.cjs
@@ -4,7 +4,7 @@ const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const {execFileSync} = require('node:child_process');
-const {hash, identity, decide} = require('./state.cjs');
+const {hash, identity, decide, loadRecordsAt} = require('./state.cjs');
const compare = (a, b) => a < b ? -1 : a > b ? 1 : 0;
const key = page => identity({...page, locale: 'ja'});
@@ -49,8 +49,7 @@ function sidebarOrder(sidebars) {
// as merged coverage. The caller supplies the reviewed, merged source commit.
function mergedCurrent(page, committedHash) {
return page.status === 'current' && page.useTarget && page.reviewStatus
!== 'unverified'
- && committedHash(page.targetPath) === page.targetHash
- && committedHash('translations/state/ja/docs.json') !== null;
+ && committedHash(page.targetPath) === page.targetHash;
}
function buildGroup({group, pages, files, order, scope, committedHash,
committedRecords}) {
@@ -205,8 +204,7 @@ async function main() {
catch {return null;}
};
execFileSync('git', ['cat-file', '-e', `${sourceCommit}^{commit}`], {cwd:
root});
- const committedText = gitRead('translations/state/ja/docs.json');
- const committedRecords = new Map((committedText ?
JSON.parse(committedText).pages : []).map(record => [key(record), record]));
+ const committedRecords = loadRecordsAt(root, 'ja', sourceCommit);
const committedCache = new Map();
const committedHash = file => {
if (!committedCache.has(file)) {
diff --git a/scripts/i18n/state.cjs b/scripts/i18n/state.cjs
index c0bff70fa27..b126528d9b6 100644
--- a/scripts/i18n/state.cjs
+++ b/scripts/i18n/state.cjs
@@ -1,6 +1,7 @@
const fs = require('node:fs');
const path = require('node:path');
const {createHash} = require('node:crypto');
+const {execFileSync} = require('node:child_process');
const hash = value => createHash('sha256').update(value.replace(/\r\n/g,
'\n')).digest('hex');
const identity = ({pluginId, version, docId, locale}) =>
JSON.stringify([pluginId, version, docId, locale]);
@@ -30,8 +31,29 @@ function loadConfig(root) {
return config;
}
-function loadRecords(root, locale) {
- const data = readJson(path.join(root,
`translations/state/${locale}/docs.json`), {schemaVersion: 1, pages: []});
+const stateRoot = locale => `translations/state/${locale}`;
+const shardRoot = locale => `${stateRoot(locale)}/pages`;
+function stateFileFor(record) {
+ const parts = [record.pluginId, record.version,
...record.docId.split('/')];
+ if (parts.some(part => !part || part === '.' || part === '..' ||
/[\\/]/.test(part)))
+ throw new Error(`Invalid translation state path: ${identity(record)}`);
+ return
`${shardRoot(record.locale)}/${record.pluginId}/${record.version}/${record.docId}.json`;
+}
+function listShardFiles(root, locale) {
+ const base = path.join(root, shardRoot(locale));
+ const files = [];
+ function walk(dir) {
+ if (!fs.existsSync(dir)) return;
+ for (const entry of fs.readdirSync(dir, {withFileTypes: true})) {
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) walk(full);
+ else if (entry.isFile() && entry.name.endsWith('.json'))
files.push(relative(root, full));
+ }
+ }
+ walk(base);
+ return files.sort();
+}
+function collectRecords(locale, data, shards) {
if (data.schemaVersion !== 1 || !Array.isArray(data.pages)) throw new
Error('Invalid translation state schema');
const records = new Map();
for (const record of data.pages) {
@@ -39,8 +61,36 @@ function loadRecords(root, locale) {
if (record.locale !== locale || records.has(key)) throw new
Error(`Duplicate or invalid translation identity: ${key}`);
records.set(key, record);
}
+ const seenShards = new Set();
+ for (const [file, raw] of shards) {
+ const shard = JSON.parse(raw);
+ const record = shard.page;
+ if (shard.schemaVersion !== 1 || !record || record.locale !== locale
+ || stateFileFor(record) !== file ||
seenShards.has(identity(record)))
+ throw new Error(`Invalid or duplicate translation shard: ${file}`);
+ seenShards.add(identity(record));
+ records.set(identity(record), record); // A shard supersedes a legacy
docs.json entry.
+ }
return records;
}
+function loadRecords(root, locale) {
+ const data = readJson(path.join(root, `${stateRoot(locale)}/docs.json`),
{schemaVersion: 1, pages: []});
+ const shards = listShardFiles(root, locale).map(file => [file,
fs.readFileSync(path.join(root, file), 'utf8')]);
+ return collectRecords(locale, data, shards);
+}
+function loadRecordsAt(root, locale, ref) {
+ const read = file => {
+ try {return execFileSync('git', ['show', `${ref}:${file}`],
+ {cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'],
maxBuffer: 16 * 1024 * 1024});}
+ catch {return null;}
+ };
+ const base = read(`${stateRoot(locale)}/docs.json`);
+ const paths = execFileSync('git', ['ls-tree', '-r', '--name-only', ref,
'--', shardRoot(locale)],
+ {cwd: root, encoding: 'utf8'}).split('\n').filter(Boolean).sort();
+ const shards = paths.map(file => [file, read(file)]);
+ if (shards.some(([, raw]) => raw === null)) throw new Error('Committed
translation shard is unreadable');
+ return collectRecords(locale, base === null ? {schemaVersion: 1, pages:
[]} : JSON.parse(base), shards);
+}
function sourceLocaleFor(settings, page) {
return settings.sourceOverrides.find(override => identity({...override,
locale: page.locale}) === identity(page))?.sourceLocale
@@ -82,4 +132,5 @@ function decide({page, source, target, record,
technicalHash}) {
return result('stale_unknown');
}
-module.exports = {hash, identity, relative, loadConfig, loadRecords,
sourceLocaleFor, decide};
+module.exports = {hash, identity, relative, loadConfig, loadRecords,
loadRecordsAt,
+ stateFileFor, listShardFiles, sourceLocaleFor, decide};
diff --git a/scripts/i18n/workflow-prs.cjs b/scripts/i18n/workflow-prs.cjs
index 1cb2bac92be..2b764a5fd9b 100644
--- a/scripts/i18n/workflow-prs.cjs
+++ b/scripts/i18n/workflow-prs.cjs
@@ -4,7 +4,7 @@ const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const {execFileSync} = require('node:child_process');
-const {hash, identity} = require('./state.cjs');
+const {hash, identity, stateFileFor} = require('./state.cjs');
const {managedPath} = require('./translation-engine.cjs');
const statePath = 'translations/state/ja/docs.json';
@@ -27,9 +27,8 @@ function parseState(raw) {
return data.pages;
}
-async function reconcileOpenPrs({scope, pulls, filesFor, readHead, basePages =
[], repository}) {
+async function reconcileOpenPrs({scope, pulls, filesFor, readHead, readBase,
repository}) {
const active = new Map(), blocked = new Set(), exceptions = [], relevant =
[];
- const base = new Map(basePages.map(page => [identity(page), page]));
const botPrs = [], botDetails = [];
for (const pull of pulls) {
if (pull.base?.ref !== 'master') continue;
@@ -43,22 +42,50 @@ async function reconcileOpenPrs({scope, pulls, filesFor,
readHead, basePages = [
}
const files = await filesFor(pull);
const changed = new Set(files.map(file => file.filename).filter(file
=> managedTarget(scope, file)));
- const stateChanged = files.some(file => file.filename === statePath);
+ const shards = files.map(file => file.filename)
+ .filter(file =>
/^translations\/state\/ja\/pages\/.+\.json$/.test(file));
+ const stateChanged = files.some(file => file.filename === statePath)
|| shards.length > 0;
if (!changed.size && !stateChanged) continue;
relevant.push(pull.number);
let pages;
try {
const raw = await readHead(pull, statePath);
- pages = parseState(raw);
+ const byIdentity = new Map(parseState(raw).map(page =>
[identity(page), page]));
+ let base;
+ const shardFiles = new Set(shards);
+ if (stateChanged) {
+ // Current master may contain translations merged after this
PR branched.
+ assert(typeof readBase === 'function', 'PR merge-base reader
is required');
+ base = new Map(parseState(await readBase(pull, statePath))
+ .map(page => [identity(page), page]));
+ if (files.some(file => file.filename === statePath)) {
+ for (const page of [...byIdentity.values(),
...base.values()]) {
+ if (JSON.stringify(byIdentity.get(identity(page))) !==
JSON.stringify(base.get(identity(page))))
+ shardFiles.add(stateFileFor(page));
+ }
+ }
+ }
+ for (const file of shardFiles) {
+ for (const [records, rawShard] of [[byIdentity, await
readHead(pull, file)],
+ [base, await readBase(pull, file)]]) {
+ if (rawShard === null) continue; // Added or deleted shard.
+ const shard = json(rawShard);
+ assert(shard.schemaVersion === 1 && shard.page &&
stateFileFor(shard.page) === file,
+ `Invalid open PR state shard ${file}`);
+ records.set(identity(shard.page), shard.page);
+ if (shards.includes(file))
changed.add(shard.page.targetPath);
+ }
+ }
+ pages = [...byIdentity.values()];
if (stateChanged) {
for (const page of pages) {
if (!managedTarget(scope, page.targetPath)) continue;
const old = base.get(identity(page));
if (JSON.stringify(old) !== JSON.stringify(page))
changed.add(page.targetPath);
}
- for (const page of basePages) {
+ for (const page of base.values()) {
if (managedTarget(scope, page.targetPath)
- && !pages.some(item => identity(item) ===
identity(page))) changed.add(page.targetPath);
+ && !byIdentity.has(identity(page)))
changed.add(page.targetPath);
}
}
} catch (error) {
@@ -123,11 +150,10 @@ async function main() {
const repository = process.env.GITHUB_REPOSITORY || 'apache/doris-website';
assert(repository === 'apache/doris-website', 'Unexpected repository');
const scope = json(fs.readFileSync(path.join(root,
'translations/scope/ja.json'), 'utf8'));
- const basePages = parseState(fs.readFileSync(path.join(root, statePath),
'utf8'));
const pulls =
ghPages(`repos/${repository}/pulls?state=open&base=master&per_page=100`);
const remote = process.env.I18N_PR_REMOTE || 'origin';
- const fetched = new Map();
- const readHead = (pull, file) => {
+ const fetched = new Map(), bases = new Map();
+ const fetchHead = pull => {
if (!fetched.has(pull.number)) {
execFileSync('git', ['fetch', '--no-tags', remote,
`refs/pull/${pull.number}/head`],
{cwd: root, stdio: ['ignore', 'ignore', 'pipe']});
@@ -135,13 +161,26 @@ async function main() {
assert.equal(fetchedSha, pull.head.sha, `Open PR #${pull.number}
changed during reconciliation`);
fetched.set(pull.number, fetchedSha);
}
- try {return execFileSync('git', ['show',
`${fetched.get(pull.number)}:${file}`],
+ return fetched.get(pull.number);
+ };
+ const readFile = (commit, file) => {
+ try {return execFileSync('git', ['show', `${commit}:${file}`],
{cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'],
maxBuffer: 16 * 1024 * 1024});}
catch {return null;}
};
+ const readHead = (pull, file) => readFile(fetchHead(pull), file);
+ const readBase = (pull, file) => {
+ if (!bases.has(pull.number)) {
+ const commit = execFileSync('git', ['merge-base', 'HEAD',
fetchHead(pull)],
+ {cwd: root, encoding: 'utf8'}).trim();
+ assert(/^[a-f0-9]{40}$/.test(commit), `Missing merge base for PR
#${pull.number}`);
+ bases.set(pull.number, commit);
+ }
+ return readFile(bases.get(pull.number), file);
+ };
const result = await reconcileOpenPrs({scope, pulls,
filesFor: pull =>
ghPages(`repos/${repository}/pulls/${pull.number}/files?per_page=100`),
- readHead, basePages, repository});
+ readHead, readBase, repository});
fs.writeFileSync(path.join(output, 'active-candidates.json'),
JSON.stringify(result.active, null, 2) + '\n');
fs.writeFileSync(path.join(output, 'open-pr-reconciliation.json'),
JSON.stringify(result, null, 2) + '\n');
process.stdout.write(JSON.stringify({openPrCount: result.openPrCount,
diff --git a/translations/README.md b/translations/README.md
index 85ce9068e1d..b1c3e9cd20b 100644
--- a/translations/README.md
+++ b/translations/README.md
@@ -12,8 +12,13 @@ English docs plugin. A Chinese-only file is reported as
## State schema, version 1
-`state/ja/docs.json` stores a `schemaVersion` and a `pages` array. Each record
-contains these fields:
+`state/ja/docs.json` is the legacy shared ledger. New Japanese records can live
+in independent files at `state/ja/pages/<pluginId>/<version>/<docId>.json`.
+Each file contains `{"schemaVersion":1,"page":<record>}`. The loader combines
+both locations; a page file supersedes a legacy entry with the same identity.
+The path must match the record's native identity, so independent translation
PRs
+modify distinct state files. `state/ja/shards.schema.json` marks the repository
+as ready for page files. Each record contains these fields:
| Field | Meaning |
| --- | --- |
@@ -181,7 +186,7 @@ links, sidebars and redirects, record the completed task in
`state/ja/resolved-deletions.json` with its ID, source hash, source path,
`reviewedBy`, ISO `resolvedAt` and PR `evidence`. This separate schema 1 ledger
prevents the fixed T03 baseline from reopening an already reviewed deletion.
-The live docs adapter continues to read `state/ja/docs.json` unchanged. A
+The live docs adapter reads the combined legacy and page-file state. A
reintroduced page returns to normal native enumeration.
## Manual Japanese translation candidates
diff --git a/translations/state/ja/shards.schema.json
b/translations/state/ja/shards.schema.json
new file mode 100644
index 00000000000..95abdd99357
--- /dev/null
+++ b/translations/state/ja/shards.schema.json
@@ -0,0 +1,4 @@
+{
+ "schemaVersion": 1,
+ "format": "page-record-v1"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
