Extract locales automatically in CI

.github/workflows/localization.yml

@@ -0,0 +1,49 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - master
+      - localization
+  pull_request:
+
+jobs:
+  locales:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+            node-version: 18
+            cache: 'yarn'
+
+      - name: Install gettext
+        run: sudo apt-get install gettext
+
+      - name: Yarn install
+        run: yarn install --frozen-lockfile --prefer-offline
+
+      - name: Extract locales
+        run: yarn extract-locales
+
+      - name: Push Locales
+        if: github.event_name == 'push'
+        env:
+          # The default branch of the repository, in this case "master"
+          default_branch: ${{ github.event.repository.default_branch }}
+        run: |
+          # Stable check for if the workflow is running on the default branch
+          # https://stackoverflow.com/questions/64781462/github-actions-default-branch-variable
+          is_default_branch="${{ format('refs/heads/{0}', env.default_branch) == github.ref }}"
+
+          ARGS=""
+
+          if [[ "$is_default_branch" == 'false' ]]; then
+            ARGS="--dry-run"
+          fi
+
+          ./bin/push-locales $ARGS
+
+
+

.prettierignore

@@ -2,6 +2,7 @@
 *.*
 # exclude these files
 Dockerfile
+src/fonts/LICENSE
 # exclude these directories
 /assets/
 /bin/

babel.config.locales.js

@@ -0,0 +1,41 @@
+// Create UTC creation date in the correct format.
+const potCreationDate = new Date()
+  .toISOString()
+  .replace('T', ' ')
+  .replace(/:\d{2}.\d{3}Z/, '+0000');
+
+module.exports = {
+  extends: './babel.config.js',
+  plugins: [
+    [
+      'module:babel-gettext-extractor',
+      {
+        headers: {
+          'Project-Id-Version': 'amo',
+          'Report-Msgid-Bugs-To': 'EMAIL@ADDRESS',
+          'POT-Creation-Date': potCreationDate,
+          'PO-Revision-Date': 'YEAR-MO-DA HO:MI+ZONE',
+          'Last-Translator': 'FULL NAME <EMAIL@ADDRESS>',
+          'Language-Team': 'LANGUAGE <[email protected]>',
+          'MIME-Version': '1.0',
+          'Content-Type': 'text/plain; charset=utf-8',
+          'Content-Transfer-Encoding': '8bit',
+          'plural-forms': 'nplurals=2; plural=(n!=1);',
+        },
+        functionNames: {
+          gettext: ['msgid'],
+          dgettext: ['domain', 'msgid'],
+          ngettext: ['msgid', 'msgid_plural', 'count'],
+          dngettext: ['domain', 'msgid', 'msgid_plural', 'count'],
+          pgettext: ['msgctxt', 'msgid'],
+          dpgettext: ['domain', 'msgctxt', 'msgid'],
+          npgettext: ['msgctxt', 'msgid', 'msgid_plural', 'count'],
+          dnpgettext: ['domain', 'msgctxt', 'msgid', 'msgid_plural', 'count'],
+        },
+        fileName: './locale/templates/LC_MESSAGES/amo.pot',
+        baseDirectory: process.cwd(),
+        stripTemplateLiteralIndent: true,
+      },
+    ],
+  ],
+};

bin/locales.mjs

@@ -0,0 +1,59 @@
+#!/usr/bin/env zx
+
+import {$, path, echo, within, glob} from 'zx';
+
+const root = path.join(__dirname, '..');
+const localeDir = path.join(root, 'locale');
+const templateFile = path.join(localeDir, '/templates/LC_MESSAGES/amo.pot');
+
+within(async () => {
+  echo('Extracting locales...');
+
+  const sourceDir = path.join(root, 'src', 'amo');
+  const outputDir = path.join(root, 'dist', 'locales');
+  const localesConfig = path.join(root, 'babel.config.locales.js');
+
+  await $`babel ${sourceDir} \
+    --out-dir ${outputDir} \
+    --config-file ${localesConfig} \
+    --verbose \
+  `;
+
+  const {stdout: output} = await $`git diff --numstat -- ${templateFile}`;
+
+  // git diff --numstat returns the number of insertions and deletions for each file
+  // this regex extracts the numbers from the output
+  const regex = /([0-9]+).*([0-9]+)/;
+
+  const [, insertions = 0, deletions = 0] = output.match(regex) || [];
+
+  const isLocaleClean = insertions < 2 && deletions < 2;
+
+  if (isLocaleClean) {
+    return echo('No locale changes, nothing to update, ending process');
+  }
+
+  echo(`Found ${insertions} insertions and ${deletions} deletions in ${templateFile}.`);
+
+  const poFiles = await glob(`${localeDir}/**/amo.po`);
+
+  echo(`Merging ${poFiles.length} translation files.`);
+
+  for await (const poFile of poFiles) {
+    const dir = path.dirname(poFile);
+    const stem = path.basename(poFile, '.po');
+    const tempFile = path.join(dir, `${stem}.po.tmp`);
+    echo(`merging: ${poFile}`);
+
+    try {
+      await $`msgmerge --no-fuzzy-matching -q -o ${tempFile} ${poFile} ${templateFile}`
+      await $`mv ${tempFile} ${poFile}`
+    } catch (error) {
+      await $`rm ${tempFile}`;
+      throw new Error(`Error merging ${poFile}`);
+    }
+  }
+
+  return true;
+});
+

bin/push-locales

@@ -0,0 +1,58 @@
+#! /bin/bash
+
+# Exit immediately when a command fails.
+set -e
+
+# Make sure exit code are respected in a pipeline.
+set -o pipefail
+
+# Treat unset variables as an error an exit immediately.
+set -u
+
+info() {
+  local message="$1"
+
+  echo ""
+  echo "INFO: $message"
+  echo ""
+}
+
+ROBOT_EMAIL="[email protected]"
+ROBOT_NAME="Mozilla Add-ons Robot"
+
+# Set git committer/author to the robot.
+export GIT_AUTHOR_NAME="$ROBOT_NAME"
+export GIT_AUTHOR_EMAIL="$ROBOT_EMAIL"
+export GIT_COMMITTER_NAME="$ROBOT_NAME"
+export GIT_COMMITTER_EMAIL="$ROBOT_EMAIL"
+
+DATE=$(date -u +%Y-%m-%d)
+REV=$(git rev-parse --short HEAD)
+MESSAGE="Extracted l10n messages from $DATE at $REV"
+DIFF_WITH_ONE_LINE_CHANGE="2 files changed, 2 insertions(+), 2 deletions(-)"
+
+git_diff_stat=$(git diff --shortstat locale/templates/LC_MESSAGES)
+
+info "git_diff_stat: $git_diff_stat"
+
+# IF there are no uncommitted local changes, exit early.
+if [[ -z "$git_diff_stat" ]] || [[ "$git_diff_stat" == *"$DIFF_WITH_ONE_LINE_CHANGE"* ]]; then
+  info """
+    No substantial changes to l10n strings found. Exiting the process.
+  """
+  exit 0
+fi
+
+info """
+GIT_AUTHOR_NAME: $GIT_AUTHOR_NAME
+GIT_AUTHOR_EMAIL: $GIT_AUTHOR_EMAIL
+GIT_COMMITTER_NAME: $GIT_COMMITTER_NAME
+GIT_COMMITTER_EMAIL: $GIT_COMMITTER_EMAIL
+
+This script passes arguments directly to Git commands. We can pass --dry-mode to test this script
+Without actually committing or pushing. Make sure to only pass arguments supported on both commit and push..
+ARGS: $@
+"""
+
+git commit -am "$MESSAGE" "$@"
+git push "$@"