From ea2f28b1dbb9e31251dc65694a1b86e12dc57f14 Mon Sep 17 00:00:00 2001 From: Francesco Lodolo Date: Mon, 13 Jun 2022 13:31:48 +0200 Subject: [PATCH] Extract strings from all XLIFF files (#1019) --- app/classes/Transvision/Project.php | 6 ----- app/classes/Transvision/Xliff.php | 5 +++-- app/config/config.ini-dev | 2 +- app/config/config.ini-dist | 2 +- app/config/config.ini-ghactions | 2 +- app/scripts/tmx/tmx_xliff | 34 +++++++++++++++++++++-------- requirements.txt | 2 +- tests/units/Transvision/Xliff.php | 4 ++-- 8 files changed, 34 insertions(+), 23 deletions(-) diff --git a/app/classes/Transvision/Project.php b/app/classes/Transvision/Project.php index 966f9032..b061a551 100644 --- a/app/classes/Transvision/Project.php +++ b/app/classes/Transvision/Project.php @@ -53,9 +53,6 @@ class Project */ public static $repos_info = [ 'firefox_ios' => [ - 'files' => [ - 'firefox-ios.xliff', - ], 'git_repository' => 'firefoxios-l10n', 'locale_mapping' => [ 'bn-IN' => 'bn', @@ -86,9 +83,6 @@ class Project 'variable_patterns' => ['xml_android'], ], 'vpn_client' => [ - 'files' => [ - 'mozillavpn.xliff', - ], 'git_repository' => 'mozilla-vpn-client-l10n', 'underscore_locales' => true, 'pontoon_project' => 'mozilla-vpn-client', diff --git a/app/classes/Transvision/Xliff.php b/app/classes/Transvision/Xliff.php index 77bc921d..59df0530 100644 --- a/app/classes/Transvision/Xliff.php +++ b/app/classes/Transvision/Xliff.php @@ -15,16 +15,17 @@ class Xliff * Loads strings from a .xliff file * * @param string $xliff_path Path to the .xliff to load + * @param string $relative_file Relative path of the file within the locale * @param string $project_name The project this string belongs to * @param boolean $reference_locale If the current file belongs to the reference locale * * @return array Array of strings as [string_id => translation] */ - public static function getStrings($xliff_path, $project_name, $reference_locale = false) + public static function getStrings($xliff_path, $relative_file, $project_name, $reference_locale = false) { $strings = []; if ($xml = simplexml_load_file($xliff_path)) { - $file_name = basename($xliff_path); + $file_name = $relative_file; $namespaces = $xml->getDocNamespaces(); $xml->registerXPathNamespace('x', $namespaces['']); /* diff --git a/app/config/config.ini-dev b/app/config/config.ini-dev index 44221d7b..96160f15 100644 --- a/app/config/config.ini-dev +++ b/app/config/config.ini-dev @@ -10,7 +10,7 @@ install=${TRANSVISIONDIR}/ config=${TRANSVISIONDIR}/app/config ; URL to l10n web service. Wrap the value between quotes if the URL contains special characters like ~ -l10nwebservice='https://flod.org/mozilla-l10n-query/mozilla-l10n-query/' +l10nwebservice='https://flod.org/mozilla-l10n-query/' ; Only needed for production setup local_hg=/temp diff --git a/app/config/config.ini-dist b/app/config/config.ini-dist index 1b0abb78..69d91d0c 100644 --- a/app/config/config.ini-dist +++ b/app/config/config.ini-dist @@ -23,7 +23,7 @@ install=/home/pascalc/github/transvision config=/home/pascalc/repos/github/transvision/app/config ; URL to l10n web service. Wrap the value between quotes if the URL contains special characters like ~ -l10nwebservice = "https://flod.org/mozilla-l10n-query/mozilla-l10n-query/" +l10nwebservice = "https://flod.org/mozilla-l10n-query/" ; Flag to know if we are working in production mode or development mode dev=false diff --git a/app/config/config.ini-ghactions b/app/config/config.ini-ghactions index 492b9ce4..50c1379c 100644 --- a/app/config/config.ini-ghactions +++ b/app/config/config.ini-ghactions @@ -12,7 +12,7 @@ install=./ config=app/config ; URL to l10n web service. Wrap the value between quotes if the URL contains special characters like ~ -l10nwebservice = "https://flod.org/mozilla-l10n-query/mozilla-l10n-query/" +l10nwebservice = "https://flod.org/mozilla-l10n-query/" ; Path to the local Mercurial clones, both en-US and l10n. Could be external ; to the Git checkout of Transvision, as long as scripts have access to it. diff --git a/app/scripts/tmx/tmx_xliff b/app/scripts/tmx/tmx_xliff index 578a8e5f..8c308223 100755 --- a/app/scripts/tmx/tmx_xliff +++ b/app/scripts/tmx/tmx_xliff @@ -36,6 +36,25 @@ error_log('Extraction of strings from XLIFF file'); $base_path = isset($repo_data['git_subfolder']) ? GIT . "{$project_name}/{$repo_data['git_subfolder']}" : GIT . $project_name; + +$reference_locale = isset($repo_data['reference_locale']) + ? $repo_data['reference_locale'] + : 'en-US'; + +// Extract a list of reference files, with relative path to the locale folder +$ref_path = "{$base_path}/{$reference_locale}"; +$iterator = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($ref_path)); +$ref_files = array(); +foreach ($iterator as $file) { + $filename = $file->getFilename(); + if (! $file->isDir()) { + if (! Strings::startsWith($filename, '.') && $file->getExtension() == 'xliff') { + $relative_filename = substr($file->getPathname(), strlen($ref_path) + 1); + $ref_files[] = $relative_filename; + } + } +} + foreach (Files::getFoldersInPath($base_path, ['templates']) as $locale_folder) { $out_translation = ''; $total_strings = 0; @@ -44,17 +63,14 @@ foreach (Files::getFoldersInPath($base_path, ['templates']) as $locale_folder) { $locale = isset($repo_data['underscore_locales']) && $repo_data['underscore_locales'] ? str_replace('_', '-', $locale_folder) : $locale_folder; + $is_reference_locale = $locale == $reference_locale; - foreach ($repo_data['files'] as $file_name) { - $xliff_path = "{$base_path}/{$locale_folder}/{$file_name}"; + $total_strings = 0; + foreach ($ref_files as $filename) { + $xliff_path = "{$base_path}/{$locale_folder}/{$filename}"; if (file_exists($xliff_path)) { - // Make sure there is a reference locale defined - $reference_locale = isset($repo_data['reference_locale']) - ? $repo_data['reference_locale'] - : 'en-US'; - $is_reference_locale = $locale == $reference_locale; - $strings = Xliff::getStrings($xliff_path, $project_name, $is_reference_locale); - $total_strings = count($strings); + $strings = Xliff::getStrings($xliff_path, $filename, $project_name, $is_reference_locale); + $total_strings += count($strings); foreach ($strings as $string_id => $translation) { $out_translation .= "'{$string_id}' => '{$translation}', \n"; } diff --git a/requirements.txt b/requirements.txt index 04afd06c..bdd01ae1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -compare-locales==8.1.0 +compare-locales==8.2.1 diff --git a/tests/units/Transvision/Xliff.php b/tests/units/Transvision/Xliff.php index c9513898..c86e16cf 100644 --- a/tests/units/Transvision/Xliff.php +++ b/tests/units/Transvision/Xliff.php @@ -11,7 +11,7 @@ class Xliff extends atoum\test public function testGetStrings() { $obj = new _Xliff(); - $strings = $obj->getStrings(TEST_FILES . 'xliff/firefox-ios.xliff', 'firefox_ios'); + $strings = $obj->getStrings(TEST_FILES . 'xliff/firefox-ios.xliff', 'firefox-ios.xliff', 'firefox_ios'); // Check total number of strings $this @@ -40,7 +40,7 @@ public function testGetStrings() public function testGetStringsReference() { $obj = new _Xliff(); - $strings = $obj->getStrings(TEST_FILES . 'xliff/firefox-ios.xliff', 'firefox_ios', true); + $strings = $obj->getStrings(TEST_FILES . 'xliff/firefox-ios.xliff', 'firefox-ios.xliff', 'firefox_ios', true); // Check total number of strings $this