Skip to content

Commit 98f538d

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #640 from jgbernalp/add-memsource-upload
OU-356: feat: add scripts for translations upload and download from memsource
2 parents afc4510 + 07f089f commit 98f538d

15 files changed

Lines changed: 533 additions & 72 deletions

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@ web/node_modules/
99
plugin-backend
1010
# adding these while the PR is in review. This should be removed after
1111
node_modules/
12-
dist/
12+
dist/
13+
web/po-files/

‎web/i18n-scripts/README.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Translation process
2+
3+
Translations for this project are managed using [Memsource](https://cloud.memsource.com/). The general process for handling translations is as follows:
4+
5+
## Pre-requisites
6+
7+
- Install the [unofficial Memsource CLI client](https://github.com/unofficial-memsource/memsource-cli-client#pip-install).
8+
- [Configure it with your Memsource login info](https://github.com/unofficial-memsource/memsource-cli-client#configuration-red-hat-enterprise-linux-derivatives).
9+
10+
Once your login info is configured, you should be able to log in by running `source ~/.memsourcerc`.
11+
12+
## Translation workflow
13+
14+
1. **Navigate to the `web` folder**: `cd web`
15+
1. **Generate english translations**: run `npm run i18n` to extract translatable strings from the codebase and generate/update the English translation files.
16+
1. **Commit english translations**: commit and push the updated English translation files to the repository.
17+
1. **Login to Memsource**: run `source ~/.memsourcerc` to authenticate the Memsource CLI client.
18+
1. **Upload to Memsource**: run `./i18n-scripts/memsource-upload.sh -v <version> -s <sprint>`, in which `<version>` is the OCP version and `<sprint>` is the current sprint, for example `./memsource-upload.sh -v 4.21 -s 258`. This will generate `.po` files required by Memsource for every language listed in `i18n-scripts/languages.sh` and upload them to Memsource. Every time translations are required a new project should be created in Memsource, this is handled automatically by the script.
19+
1. **Note the Memsource project link**: After the upload is complete, the script will output the Memsource project link. Make sure to copy it for the next step.
20+
1. **Translate in Memsource**: Send an email to the globalization team `localization-requests@redhat.com` with CC to `team-observability-ui@redhat.com` with the Memsource project link and request translation for the required languages.
21+
1. **Wait for translation completion**: Accessing memsource you can monitor the progress of the translations.
22+
1. **Download translations**: once the translations are completed, run `./i18n-scripts/memsource-download.sh -p <project_id>` to download the translated `.po` files from Memsource and convert them back to the i18n JSON format used by the application.

‎web/i18n-scripts/common.js‎

Lines changed: 0 additions & 34 deletions
This file was deleted.

‎web/i18n-scripts/export-pos.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
source ./i18n-scripts/languages.sh
6+
7+
for f in locales/en/* ; do
8+
for i in "${LANGUAGES[@]}"
9+
do
10+
npm run i18n-to-po -- -f "$(basename "$f" .json)" -l "$i"
11+
done
12+
done

‎web/i18n-scripts/i18n-to-po.js‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/* eslint-disable no-console */
2+
/* eslint-disable no-undef */
3+
/* eslint-disable @typescript-eslint/no-require-imports */
4+
5+
const fs = require('fs');
6+
const path = require('path');
7+
const { i18nextToPo } = require('i18next-conv');
8+
const minimist = require('minimist');
9+
10+
function save(target) {
11+
return (result) => {
12+
fs.writeFileSync(target, result);
13+
};
14+
}
15+
16+
function removeValues(i18nFile, filePath) {
17+
const file = require(i18nFile);
18+
19+
const updatedFile = {};
20+
21+
const keys = Object.keys(file);
22+
23+
for (let i = 0; i < keys.length; i++) {
24+
updatedFile[keys[i]] = '';
25+
}
26+
27+
const tmpFile = fs.openSync(filePath, 'w');
28+
29+
fs.writeFileSync(tmpFile, JSON.stringify(updatedFile, null, 2));
30+
}
31+
32+
function consolidateWithExistingTranslations(filePath, fileName, language) {
33+
const englishFile = require(filePath);
34+
const englishKeys = Object.keys(englishFile);
35+
const existingTranslationsPath = `./../locales/${language}/${fileName}.json`;
36+
37+
if (fs.existsSync(path.join(__dirname, existingTranslationsPath))) {
38+
const existingTranslationsFile = require(path.join(__dirname, existingTranslationsPath));
39+
const existingKeys = Object.keys(existingTranslationsFile);
40+
const matchingKeys = englishKeys.filter((k) => existingKeys.indexOf(k) > -1);
41+
42+
for (let i = 0; i < matchingKeys.length; i++) {
43+
englishFile[matchingKeys[i]] = existingTranslationsFile[matchingKeys[i]];
44+
}
45+
46+
fs.writeFileSync(filePath, JSON.stringify(englishFile, null, 2));
47+
}
48+
}
49+
50+
function processFile(fileName, language) {
51+
let tmpFile;
52+
53+
const i18nFile = path.join(__dirname, `./../locales/en/${fileName}.json`);
54+
55+
try {
56+
if (fs.existsSync(i18nFile)) {
57+
fs.mkdirSync(path.join(__dirname, './../locales/tmp'), { recursive: true });
58+
59+
tmpFile = path.join(__dirname, `./../locales/tmp/${fileName}.json`);
60+
61+
removeValues(i18nFile, tmpFile);
62+
consolidateWithExistingTranslations(tmpFile, fileName, language);
63+
64+
fs.mkdirSync(path.join(__dirname, `./../po-files/${language}`), { recursive: true });
65+
i18nextToPo(language, fs.readFileSync(tmpFile), {
66+
language,
67+
foldLength: 0,
68+
ctxSeparator: '~',
69+
})
70+
.then(
71+
save(
72+
path.join(__dirname, `./../po-files/${language}/${path.basename(fileName)}.po`),
73+
language,
74+
),
75+
)
76+
.catch((e) => console.error(fileName, e));
77+
} else {
78+
console.warn(`i18n file does not exist: ${i18nFile}`);
79+
}
80+
} catch (err) {
81+
console.error(`Failed to processFile ${fileName}:`, err);
82+
} finally {
83+
if (tmpFile && fs.existsSync(tmpFile)) {
84+
fs.unlinkSync(tmpFile);
85+
}
86+
}
87+
console.log(`Created po-files/${language}/${fileName}.po`);
88+
}
89+
90+
const options = {
91+
string: ['language', 'files'],
92+
boolean: ['help'],
93+
array: ['files'],
94+
alias: {
95+
h: 'help',
96+
f: 'files',
97+
l: 'language',
98+
},
99+
default: {
100+
files: [],
101+
},
102+
};
103+
104+
const args = minimist(process.argv.slice(2), options);
105+
106+
if (args.help) {
107+
console.log("-h: help\n-l: language (i.e. 'ja')\n-f: file name to convert (i.e. 'nav')");
108+
} else if (args.files && args.language) {
109+
if (Array.isArray(args.files)) {
110+
for (let i = 0; i < args.files.length; i++) {
111+
processFile(args.files[i], args.language);
112+
}
113+
} else {
114+
processFile(args.files, args.language);
115+
}
116+
} else {
117+
console.log('Invalid arguments. Use -h for help.');
118+
}

‎web/i18n-scripts/languages.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
export LANGUAGES=( 'ja' 'zh-cn' 'ko' 'fr' 'es')

‎web/i18n-scripts/lexers.js‎

Lines changed: 0 additions & 31 deletions
This file was deleted.

‎web/i18n-scripts/memsource-download.sh‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/usr/bin/env bash
2+
3+
set -exuo pipefail
4+
5+
source ./i18n-scripts/languages.sh
6+
7+
while getopts p: flag
8+
do
9+
case "${flag}" in
10+
p) PROJECT_ID=${OPTARG};;
11+
*) echo "usage: $0 [-p]" >&2
12+
exit 1;;
13+
esac
14+
done
15+
16+
echo "Checking if git workspace is clean"
17+
GIT_STATUS="$(git status --short --untracked-files -- locales)"
18+
if [ -n "$GIT_STATUS" ]; then
19+
echo "There are uncommitted files in locales folders. Remove or commit the files, then run this script again."
20+
git diff
21+
exit 1
22+
fi
23+
24+
echo "Downloading PO files from Project ID \"$PROJECT_ID\""
25+
26+
DOWNLOAD_PATH="$(mktemp -d)" || { echo "Failed to create temp folder"; exit 1; }
27+
28+
# Memsource job listing is limited to 50 jobs per page
29+
# We need to pull all the files down by page and stop when we reach a page with no data
30+
for i in "${LANGUAGES[@]}"
31+
do
32+
COUNTER=0
33+
CURRENT_PAGE=( $(memsource job list --project-id "$PROJECT_ID" --target-lang "$i" -f value --page-number 0 -c uid) )
34+
until [ -z "$CURRENT_PAGE" ]
35+
do
36+
((COUNTER++))
37+
echo Downloading page "$COUNTER"
38+
memsource job download --project-id "$PROJECT_ID" --output-dir "$DOWNLOAD_PATH/$i" --job-id "${CURRENT_PAGE[@]}"
39+
CURRENT_PAGE=$(memsource job list --project-id "$PROJECT_ID" --target-lang "$i" -f value --page-number "$COUNTER" -c uid | tr '\n' ' ')
40+
done
41+
done
42+
43+
echo Importing downloaded PO files
44+
for i in "${LANGUAGES[@]}"
45+
do
46+
# We don't treat zh-cn as a dialect in i18next right now, so we need to alter it to zh
47+
if [ "$i" == 'zh-cn' ]
48+
then
49+
npm run po-to-i18n -- -d "$DOWNLOAD_PATH/$i" -l 'zh'
50+
else
51+
npm run po-to-i18n -- -d "$DOWNLOAD_PATH/$i" -l "$i"
52+
fi
53+
done
54+
55+
echo Creating commit
56+
git add locales
57+
git commit -m "chore(i18n): update translations
58+
59+
Adding latest translations from Memsource project https://cloud.memsource.com/web/project2/show/$PROJECT_ID"

‎web/i18n-scripts/memsource-upload.sh‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env bash
2+
3+
set -exuo pipefail
4+
5+
source ./i18n-scripts/languages.sh
6+
7+
PLUGIN_NAME="monitoring-plugin"
8+
9+
while getopts v:s: flag
10+
do
11+
case "${flag}" in
12+
v) VERSION=${OPTARG};;
13+
s) SPRINT=${OPTARG};;
14+
*) echo "usage: $0 [-v] [-s]" >&2
15+
exit 1;;
16+
esac
17+
done
18+
19+
BRANCH=$(git branch --show-current)
20+
21+
echo "Creating project with title \"[OCP $VERSION] UI Localization $PLUGIN_NAME - Sprint $SPRINT/Branch $BRANCH\""
22+
23+
PROJECT_INFO=$(memsource project create --name "[OCP $VERSION] UI Localization $PLUGIN_NAME - Sprint $SPRINT/Branch $BRANCH" --template-id 169304 -f json)
24+
PROJECT_ID=$(echo "$PROJECT_INFO" | jq -r '.uid')
25+
26+
echo "Exporting PO files"
27+
./i18n-scripts/export-pos.sh
28+
echo "Exported all PO files"
29+
30+
echo "Creating jobs for exported PO files"
31+
for i in "${LANGUAGES[@]}"
32+
do
33+
memsource job create --filenames po-files/"$i"/*.po --target-langs "$i" --project-id "${PROJECT_ID}"
34+
done
35+
36+
echo "Uploaded PO files to Memsource, the project ID is: ${PROJECT_ID}"
37+
38+
# Clean up PO file directory
39+
rm -rf po-files

0 commit comments

Comments
 (0)