Automate L10n extraction locally and in CI

.github/actions/context/action.yml

@@ -0,0 +1,114 @@
+name: 'Dump Context'
+description: 'Display context for action run'
+
+outputs:
+  # All github action outputs are strings, even if set to "true"
+  # so when using these values always assert against strings or convert from json
+  # \$\{{ needs.context.outputs.is_fork == 'true' }} // true
+  # \$\{{ fromJson(needs.context.outputs.is_fork) == false }} // true
+  # \$\{{ needs.context.outputs.is_fork == true }} // false
+  # \$\{{ needs.context.outputs.is_fork }} // false
+  is_fork:
+    description: ""
+    value: ${{ steps.context.outputs.is_fork }}
+  is_default_branch:
+    description: ""
+    value: ${{ steps.context.outputs.is_default_branch }}
+  is_release_master:
+    description: ""
+    value: ${{ steps.context.outputs.is_release_master }}
+  is_release_tag:
+    description: ""
+    value: ${{ steps.context.outputs.is_release_tag }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Dump GitHub context
+      shell: bash
+      env:
+        GITHUB_CONTEXT: ${{ toJson(github) }}
+      run: echo "$GITHUB_CONTEXT"
+    - name: Dump job context
+      shell: bash
+      env:
+        JOB_CONTEXT: ${{ toJson(job) }}
+      run: echo "$JOB_CONTEXT"
+    - name: Dump steps context
+      shell: bash
+      env:
+        STEPS_CONTEXT: ${{ toJson(steps) }}
+      run: echo "$STEPS_CONTEXT"
+    - name: Dump runner context
+      shell: bash
+      env:
+        RUNNER_CONTEXT: ${{ toJson(runner) }}
+      run: echo "$RUNNER_CONTEXT"
+    - name: Dump env context
+      shell: bash
+      env:
+        ENV_CONTEXT: ${{ toJson(env) }}
+      run: |
+        echo "$ENV_CONTEXT"
+    - name: Dump inputs context
+      shell: bash
+      env:
+        INPUTS_CONTEXT: ${{ toJson(inputs) }}
+      run: |
+        echo "$INPUTS_CONTEXT"
+
+    - name: Set context
+      id: context
+      env:
+        # The default branch of the repository, in this case "master"
+        default_branch: ${{ github.event.repository.default_branch }}
+      shell: bash
+      run: |
+        event_name="${{ github.event_name }}"
+        event_action="${{ github.event.action }}"
+
+        # Stable check for if the workflow is running on the default branch
+        # https://stackoverflow.com/questions/64781462/github-actions-default-branch-variable
+        is_default_branch="${{ format('refs/heads/{0}', env.default_branch) == github.ref }}"
+
+        # In most events, the epository refers to the head which would be the fork
+        is_fork="${{ github.event.repository.fork }}"
+
+        # This is different in a pull_request where we need to check the head explicitly
+        if [[ "${{ github.event_name }}" == 'pull_request' ]]; then
+          # repository on a pull request refers to the base which is always mozilla/addons-server
+          is_head_fork="${{ github.event.pull_request.head.repo.fork }}"
+          # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions
+          is_dependabot="${{ github.actor == 'dependabot[bot]' }}"
+
+          # If the head repository is a fork or if the PR is opened by dependabot
+          # we consider the run to be a fork. Dependabot and proper forks are treated
+          # the same in terms of limited read only github token scope
+          if [[ "$is_head_fork" == 'true' || "$is_dependabot" == 'true' ]]; then
+            is_fork="true"
+          fi
+        fi
+
+        is_release_master="false"
+        is_release_tag="false"
+
+        # Releases can only happen if we are NOT on a fork
+        if [[ "$is_fork" == 'false' ]]; then
+          # A master release occurs on a push to the default branch of the origin repository
+          if [[ "$event_name" == 'push' && "$is_default_branch" == 'true' ]]; then
+            is_release_master="true"
+          fi
+
+          # A tag release occurs when a release is published
+          if [[ "$event_name" == 'release' && "$event_action" == 'publish' ]]; then
+            is_release_tag="true"
+          fi
+        fi
+
+        echo "is_default_branch=$is_default_branch" >> $GITHUB_OUTPUT
+        echo "is_fork=$is_fork" >> $GITHUB_OUTPUT
+        echo "is_release_master=$is_release_master" >> $GITHUB_OUTPUT
+        echo "is_release_tag=$is_release_tag" >> $GITHUB_OUTPUT
+
+        echo "event_name: $event_name"
+        cat $GITHUB_OUTPUT

.github/workflows/ci.yml

@@ -0,0 +1,68 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  context:
+    runs-on: ubuntu-latest
+
+    outputs:
+      is_fork: ${{ steps.context.outputs.is_fork }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - id: context
+        uses: ./.github/actions/context
+
+  locales:
+    needs: context
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+
+      - uses: actions/setup-node@v4
+        with:
+            node-version: 18
+            cache: 'yarn'
+
+      - name: Install gettext
+        run: sudo apt-get install gettext
+
+      - name: Yarn install
+        run: yarn install --frozen-lockfile --prefer-offline
+
+      - name: Extract locales
+        run: yarn extract-locales
+
+      - name: Push Locales
+        run: |
+          event_name="${{ github.event_name }}"
+          is_fork="${{ needs.context.outputs.is_fork }}"
+
+          if [[ "$is_fork" == 'true' ]]; then
+            cat <<'EOF'
+              Github actions are not authorized to push from workflows triggered by forks.
+              We cannot verify if the l10n extraction push will work or not.
+              Please submit a PR from the base repository if you are modifying l10n extraction scripts.
+          EOF
+            exit 0
+          fi
+
+          ARGS=""
+
+          if [[ "$event_name" == 'pull_request' ]]; then
+            ARGS="--dry-run"
+          fi
+
+          ./bin/push-locales $ARGS
+
+
+

.prettierignore

@@ -2,6 +2,7 @@
 *.*
 # exclude these files
 Dockerfile
+src/fonts/LICENSE
 # exclude these directories
 /assets/
 /bin/

babel.config.locales.js

@@ -0,0 +1,41 @@
+// Create UTC creation date in the correct format.
+const potCreationDate = new Date()
+  .toISOString()
+  .replace('T', ' ')
+  .replace(/:\d{2}.\d{3}Z/, '+0000');
+
+module.exports = {
+  extends: './babel.config.js',
+  plugins: [
+    [
+      'module:babel-gettext-extractor',
+      {
+        headers: {
+          'Project-Id-Version': 'amo',
+          'Report-Msgid-Bugs-To': 'EMAIL@ADDRESS',
+          'POT-Creation-Date': potCreationDate,
+          'PO-Revision-Date': 'YEAR-MO-DA HO:MI+ZONE',
+          'Last-Translator': 'FULL NAME <EMAIL@ADDRESS>',
+          'Language-Team': 'LANGUAGE <[email protected]>',
+          'MIME-Version': '1.0',
+          'Content-Type': 'text/plain; charset=utf-8',
+          'Content-Transfer-Encoding': '8bit',
+          'plural-forms': 'nplurals=2; plural=(n!=1);',
+        },
+        functionNames: {
+          gettext: ['msgid'],
+          dgettext: ['domain', 'msgid'],
+          ngettext: ['msgid', 'msgid_plural', 'count'],
+          dngettext: ['domain', 'msgid', 'msgid_plural', 'count'],
+          pgettext: ['msgctxt', 'msgid'],
+          dpgettext: ['domain', 'msgctxt', 'msgid'],
+          npgettext: ['msgctxt', 'msgid', 'msgid_plural', 'count'],
+          dnpgettext: ['domain', 'msgctxt', 'msgid', 'msgid_plural', 'count'],
+        },
+        fileName: './locale/templates/LC_MESSAGES/amo.pot',
+        baseDirectory: process.cwd(),
+        stripTemplateLiteralIndent: true,
+      },
+    ],
+  ],
+};

bin/extract-locales

@@ -1,3 +1,59 @@
-#!/usr/bin/env sh
+#!/usr/bin/env zx
+
+import {$, path, echo, within, glob} from 'zx';
+
+const root = path.join(__dirname, '..');
+const localeDir = path.join(root, 'locale');
+const templateFile = path.join(localeDir, '/templates/LC_MESSAGES/amo.pot');
+
+within(async () => {
+  echo('Extracting locales...');
+
+  const sourceDir = path.join(root, 'src', 'amo');
+  const outputDir = path.join(root, 'dist', 'locales');
+  const localesConfig = path.join(root, 'babel.config.locales.js');
+
+  await $`babel ${sourceDir} \
+    --out-dir ${outputDir} \
+    --config-file ${localesConfig} \
+    --verbose \
+  `;
+
+  const {stdout: output} = await $`git diff --numstat -- ${templateFile}`;
+
+  // git diff --numstat returns the number of insertions and deletions for each file
+  // this regex extracts the numbers from the output
+  const regex = /([0-9]+).*([0-9]+)/;
+
+  const [, insertions = 0, deletions = 0] = output.match(regex) || [];
+
+  const isLocaleClean = insertions < 2 && deletions < 2;
+
+  if (isLocaleClean) {
+    return echo('No locale changes, nothing to update, ending process');
+  }
+
+  echo(`Found ${insertions} insertions and ${deletions} deletions in ${templateFile}.`);
+
+  const poFiles = await glob(`${localeDir}/**/amo.po`);
+
+  echo(`Merging ${poFiles.length} translation files.`);
+
+  for await (const poFile of poFiles) {
+    const dir = path.dirname(poFile);
+    const stem = path.basename(poFile, '.po');
+    const tempFile = path.join(dir, `${stem}.po.tmp`);
+    echo(`merging: ${poFile}`);
+
+    try {
+      await $`msgmerge --no-fuzzy-matching -q -o ${tempFile} ${poFile} ${templateFile}`
+      await $`mv ${tempFile} ${poFile}`
+    } catch (error) {
+      await $`rm ${tempFile}`;
+      throw new Error(`Error merging ${poFile}`);
+    }
+  }
+
+  return true;
+});
 
-yarn extract-locales

bin/push-locales

@@ -0,0 +1,58 @@
+#! /bin/bash
+
+# Exit immediately when a command fails.
+set -e
+
+# Make sure exit code are respected in a pipeline.
+set -o pipefail
+
+# Treat unset variables as an error an exit immediately.
+set -u
+
+info() {
+  local message="$1"
+
+  echo ""
+  echo "INFO: $message"
+  echo ""
+}
+
+ROBOT_EMAIL="[email protected]"
+ROBOT_NAME="Mozilla Add-ons Robot"
+
+# Set git committer/author to the robot.
+export GIT_AUTHOR_NAME="$ROBOT_NAME"
+export GIT_AUTHOR_EMAIL="$ROBOT_EMAIL"
+export GIT_COMMITTER_NAME="$ROBOT_NAME"
+export GIT_COMMITTER_EMAIL="$ROBOT_EMAIL"
+
+DATE=$(date -u +%Y-%m-%d)
+REV=$(git rev-parse --short HEAD)
+MESSAGE="Extracted l10n messages from $DATE at $REV"
+DIFF_WITH_ONE_LINE_CHANGE="2 files changed, 2 insertions(+), 2 deletions(-)"
+
+git_diff_stat=$(git diff --shortstat locale/templates/LC_MESSAGES)
+
+info "git_diff_stat: $git_diff_stat"
+
+# IF there are no uncommitted local changes, exit early.
+if [[ -z "$git_diff_stat" ]] || [[ "$git_diff_stat" == *"$DIFF_WITH_ONE_LINE_CHANGE"* ]]; then
+  info """
+    No substantial changes to l10n strings found. Exiting the process.
+  """
+  exit 0
+fi
+
+info """
+GIT_AUTHOR_NAME: $GIT_AUTHOR_NAME
+GIT_AUTHOR_EMAIL: $GIT_AUTHOR_EMAIL
+GIT_COMMITTER_NAME: $GIT_COMMITTER_NAME
+GIT_COMMITTER_EMAIL: $GIT_COMMITTER_EMAIL
+
+This script passes arguments directly to Git commands. We can pass --dry-mode to test this script
+Without actually committing or pushing. Make sure to only pass arguments supported on both commit and push..
+ARGS: $@
+"""
+
+git commit -am "$MESSAGE" "$@"
+git push "$@"