feat: start crowdin integration

.gitignore

@@ -4,4 +4,6 @@ node_modules
 .env
 .vscode/settings.json
 build/
-content/
+content/
+i18n/
+!i18n/en/

crowdin.yml

@@ -0,0 +1,21 @@
+project_id: '273870'
+api_token_env: 'CROWDIN_PERSONAL_TOKEN'
+preserve_hierarchy: true
+files: [
+    # JSON translation files
+    {
+      source: '/i18n/en/**/*',
+      translation: '/i18n/%two_letters_code%/**/%original_file_name%',
+    },
+    # Docs Markdown files
+    {
+      source: '/docs/**/*',
+      translation: '/i18n/%two_letters_code%/docusaurus-plugin-content-docs/current/**/%original_file_name%',
+      ignore: ['/docs/**/fiddles', '/docs/**/images'],
+    },
+    # Blog Markdown files
+    {
+      source: '/blog/**/*',
+      translation: '/i18n/%two_letters_code%/docusaurus-plugin-content-blog/**/%original_file_name%',
+    },
+  ]

docusaurus.config.js

@@ -13,6 +13,19 @@ module.exports = {
   favicon: 'assets/img/favicon.ico',
   organizationName: 'electron',
   projectName: 'electron',
+  i18n: {
+    defaultLocale: 'en',
+    locales: [
+      'en',
+      'de',
+      'es',
+      'fr',
+      'ja',
+      'pt',
+      'ru',
+      'zh',
+    ],
+  },
   themeConfig: {
     announcementBar: {
       id: 'to_old_docs',
@@ -143,13 +156,17 @@ module.exports = {
         docs: {
           sidebarPath: require.resolve('./sidebars.js'),
           routeBasePath: '/docs/',
-          editUrl: ({docPath}) => {
+          editUrl: ({ docPath }) => {
             // TODO: remove when `latest/` is no longer hardcoded
             const fixedPath = docPath.replace('latest/', '');
             // TODO: versioning?
-            return `https://github.com/electron/electron/edit/main/docs/${fixedPath}`
+            return `https://github.com/electron/electron/edit/main/docs/${fixedPath}`;
           },
-          remarkPlugins: [fiddleEmbedder, apiLabels, [npm2yarn, { sync: true }]],
+          remarkPlugins: [
+            fiddleEmbedder,
+            apiLabels,
+            [npm2yarn, { sync: true }],
+          ],
         },
         blog: {
           // See `node_modules/@docusaurus/plugin-content-blog/src/pluginOptionSchema.ts` for full undocumented options

i18n/en-US/code.json → i18n/en/code.json

@@ -206,5 +206,13 @@
   "theme.tags.tagsPageTitle": {
     "message": "Tags",
     "description": "The title of the tag list page"
+  },
+  "theme.blog.archive.title": {
+    "message": "Archive",
+    "description": "The page & hero title of the blog archive page"
+  },
+  "theme.blog.archive.description": {
+    "message": "Archive",
+    "description": "The page & hero description of the blog archive page"
   }
 }

package.json

@@ -4,9 +4,13 @@
   "private": true,
   "license": "Apache-2.0",
   "scripts": {
+    "crowdin": "crowdin",
+    "i18n:upload": "crowdin upload sources",
+    "i18n:download": "crowdin download && node scripts/prepare-i18n-content.js",
+    "i18n:build": "node scripts/i18n-build.js",
     "docusaurus": "docusaurus",
     "start": "docusaurus start",
-    "build": "docusaurus build",
+    "build": "yarn pre-build && docusaurus build --locale en",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
     "clear": "docusaurus clear",
@@ -16,7 +20,7 @@
     "update-l10n-sources": "node scripts/update-l10n-sources.js",
     "lint": "prettier -c ./scripts/**/*.js",
     "test": "yarn lint && jest",
-    "prebuild": "node ./scripts/pre-build.js",
+    "pre-build": "node ./scripts/pre-build.js",
     "process-docs-changes": "node ./scripts/process-docs-changes.js",
     "update-pinned-version": "node ./scripts/update-pinned-version.js",
     "prepare": "husky install"
@@ -50,6 +54,7 @@
   "devDependencies": {
     "@actions/core": "^1.2.7",
     "@actions/github": "^4.0.0",
+    "@crowdin/cli": "3",
     "@types/jest": "^26.0.23",
     "@types/unist": "^2.0.3",
     "del": "^6.0.0",

scripts/i18n-build.js

@@ -0,0 +1,58 @@
+//@ts-check
+const fs = require('fs').promises;
+const { join } = require('path');
+const { execute } = require('./utils/execute');
+const {
+  i18n: { locales, defaultLocale },
+} = require('../docusaurus.config');
+
+const updateConfig = async (locale) => {
+  const baseUrl = locale !== defaultLocale ? `/${locale}/` : '/';
+  // Translations might not be completely in sync and we need to keep publishing
+  const onBrokenLinks = locale !== defaultLocale ? `warn` : `throw`;
+  const configPath = join(__dirname, '../docusaurus.config.js');
+
+  let docusaurusConfig = await fs.readFile(configPath, 'utf-8');
+
+  docusaurusConfig = docusaurusConfig
+    .replace(/baseUrl: '.*?',/, `baseUrl: '${baseUrl}',`)
+    .replace(/onBrokenLinks: '.*?',/, `onBrokenLinks: '${onBrokenLinks}',`);
+
+  await fs.writeFile(configPath, docusaurusConfig, 'utf-8');
+};
+
+const processLocale = async (locale) => {
+  const start = Date.now();
+  const outdir = locale !== defaultLocale ? `--out-dir build/${locale}` : '';
+  await execute(`yarn docusaurus build --locale ${locale} ${outdir}`);
+  console.log(`Locale ${locale} finished in ${(Date.now() - start) / 1000}s`);
+};
+
+/**
+ *
+ * @param {string} [locale]
+ */
+const start = async (locale) => {
+  const start = Date.now();
+
+  const localesToBuild = locale ? [locale] : locales;
+
+  console.log('Building the following locales:');
+  console.log(localesToBuild);
+
+  try {
+    for (const locale of localesToBuild) {
+      await updateConfig(locale);
+      await processLocale(locale);
+    }
+  } catch (e) {
+    // We catch instead of just stopping the process because we want to restore docusaurus.config.js
+    console.error(`e`);
+  }
+  // Restore `docusaurus.config.js` to the default values
+  await updateConfig(defaultLocale);
+
+  console.log(`Process finished in ${(Date.now() - start) / 1000}s`);
+};
+
+start(process.argv[2]);

scripts/prepare-i18n-content.js

@@ -0,0 +1,47 @@
+//@ts-check
+
+/**
+ * Takes care of downloading the documentation from the
+ * right places, and transform it to make it ready to
+ * be used by docusaurus.
+ */
+const path = require('path');
+const fs = require('fs-extra');
+
+const { addFrontmatter } = require('./tasks/add-frontmatter');
+const { fixContent } = require('./tasks/md-fixers');
+
+const DOCS_FOLDER = path.join('docs', 'latest');
+const {
+  i18n: { locales: configuredLocales },
+} = require('../docusaurus.config');
+
+const start = async () => {
+  const locales = new Set(configuredLocales);
+  locales.delete('en');
+  for (const locale of locales) {
+    const localeDocs = path.join(
+      'i18n',
+      locale,
+      'docusaurus-plugin-content-docs',
+      'current'
+    );
+    const staticResources = ['fiddles', 'images'];
+
+    console.log(`Copying static assets to ${locale}`);
+    for (const staticResource of staticResources) {
+      await fs.copy(
+        path.join(DOCS_FOLDER, staticResource),
+        path.join(localeDocs, 'latest', staticResource)
+      );
+    }
+
+    console.log(`Fixing markdown (${locale})`);
+    await fixContent(localeDocs, 'latest');
+
+    console.log(`Adding automatic frontmatter (${locale})`);
+    await addFrontmatter(path.join(localeDocs, 'latest'));
+  }
+};
+
+start();

scripts/process-docs-changes.js

@@ -11,6 +11,7 @@ if (!(process.env.CI || process.env.NODE_ENV === 'test') && !GITHUB_TOKEN) {
   process.exit(1);
 }
 
+const { execute } = require('./utils/execute');
 const { createPR, getChanges, pushChanges } = require('./utils/git-commands');
 
 const HEAD = 'main';
@@ -42,9 +43,12 @@ const processDocsChanges = async () => {
     console.log('package.json is not modified, skipping');
     return;
   } else {
-    const lines = output.split('\n');
-    if (lines.length > 1) {
-      console.log(`New documents available, creating PR.`);
+    console.log(`Uploading changes to Crowdin`);
+    await execute(`yarn crowdin:upload`);
+
+    // If there are new docs, `sidebars.js` gets updated so we check if there are changes
+    if (output.includes('M sidebars.js')) {
+      console.log(`New documents available.`);
       await createPR(PR_BRANCH, HEAD, EMAIL, NAME, COMMIT_MESSAGE);
     } else {
       console.log(

scripts/tasks/add-frontmatter.js

@@ -87,7 +87,7 @@ const descriptionFromContent = (content) => {
 
     // The content of structures is often only bullet lists and no general description
     if (trimmedLine.startsWith('#') || trimmedLine.startsWith('*')) {
-      if (subHeader) {
+      if (subHeader && description.length > 0) {
         return cleanUpMarkdown(description.trim());
       } else {
         subHeader = true;
@@ -122,7 +122,10 @@ const addFrontMatter = (content, filepath) => {
     ? titleMatches[1].trim()
     : titleFromPath(filepath).trim();
 
-  const description = descriptionFromContent(content);
+  // The description of the files under `api/structures` is not meaningful so we ignore it
+  const description = filepath.includes('structures')
+    ? ''
+    : descriptionFromContent(content);
   const defaultSlug = path.basename(filepath, '.md');
 
   let slug;

scripts/tasks/md-fixers.js

@@ -66,12 +66,31 @@ const fiddleTransformer = (line) => {
   if (matches) {
     return `\`\`\`fiddle docs/latest/${matches[1]}`;
   } else if (hasNewPath) {
-    return line.replace(fiddlePathFixRegex, '```fiddle docs/latest/');
+    return (
+      line
+        .replace(fiddlePathFixRegex, '```fiddle docs/latest/')
+        // we could have a double transformation if the path is already the good one
+        // this happens especially with the i18n content
+        .replace('latest/latest', 'latest')
+    );
   } else {
     return line;
   }
 };
 
+/**
+ * Crowdin translations put markdown content right
+ * after HTML comments and thus breaking Docusaurus
+ * parse engine. We need to add a new EOL after `-->`
+ * is found.
+ */
+const newLineOnHTMLComment = (line) => {
+  if (line.includes('-->')) {
+    return line.replace('-->', '-->\n');
+  }
+  return line;
+};
+
 /**
  * Applies any transformation that can be executed line by line on
  * the document to make sure it is ready to be consumed by
@@ -83,7 +102,11 @@ const fiddleTransformer = (line) => {
 const transform = (doc) => {
   const lines = doc.split('\n');
   const newDoc = [];
-  const transformers = [apiTransformer, fiddleTransformer];
+  const transformers = [
+    apiTransformer,
+    fiddleTransformer,
+    newLineOnHTMLComment,
+  ];
 
   for (const line of lines) {
     const newLine = transformers.reduce((newLine, transformer) => {

scripts/update-l10n-sources.js

@@ -26,14 +26,13 @@ ${files.join('\n')}`);
     return;
   }
 
-  await del('i18n/en-US');
-  await execute('yarn write-translations --locale en-US');
+  await execute('yarn write-translations --locale en');
 
   const localeModified = (await getChanges()) !== output;
 
   if (localeModified) {
     const pleaseCommit =
-      'Contents in "/i18n/en-US/" have been modified. Please add the changes to your commit';
+      'Contents in "/i18n/en/" have been modified. Please add the changes to your commit';
     console.error('\x1b[31m%s\x1b', pleaseCommit);
     process.exit(1);
   }

yarn.lock

@@ -1310,6 +1310,13 @@
     exec-sh "^0.3.2"
     minimist "^1.2.0"
 
+"@crowdin/cli@3":
+  version "3.7.0"
+  resolved "https://registry.yarnpkg.com/@crowdin/cli/-/cli-3.7.0.tgz#d35b69e90b6a737a9de017423ac34c02291cdebb"
+  integrity sha512-7eje7V6BGMeW23ywbrYdvpdIIxG5O1WP2wit4MVP9EtuZMOfr1M0l9BnObbkSYK86UiZuoJFHs1Q1KoCWg1rlA==
+  dependencies:
+    shelljs "^0.8.4"
+
 "@docsearch/[email protected]":
   version "3.0.0-alpha.39"
   resolved "https://registry.yarnpkg.com/@docsearch/css/-/css-3.0.0-alpha.39.tgz#1ebd390d93e06aad830492f5ffdc8e05d058813f"