Extract strings from all XLIFF files (#1019)

app/classes/Transvision/Project.php

@@ -53,9 +53,6 @@ class Project
      */
     public static $repos_info = [
         'firefox_ios' => [
-            'files'             => [
-                'firefox-ios.xliff',
-            ],
             'git_repository'    => 'firefoxios-l10n',
             'locale_mapping'    => [
                 'bn-IN' => 'bn',
@@ -86,9 +83,6 @@ class Project
             'variable_patterns' => ['xml_android'],
         ],
         'vpn_client' => [
-            'files'              => [
-                'mozillavpn.xliff',
-            ],
             'git_repository'     => 'mozilla-vpn-client-l10n',
             'underscore_locales' => true,
             'pontoon_project'    => 'mozilla-vpn-client',

app/classes/Transvision/Xliff.php

@@ -15,16 +15,17 @@ class Xliff
      * Loads strings from a .xliff file
      *
      * @param string  $xliff_path       Path to the .xliff to load
+     * @param string  $relative_file    Relative path of the file within the locale
      * @param string  $project_name     The project this string belongs to
      * @param boolean $reference_locale If the current file belongs to the reference locale
      *
      * @return array Array of strings as [string_id => translation]
      */
-    public static function getStrings($xliff_path, $project_name, $reference_locale = false)
+    public static function getStrings($xliff_path, $relative_file, $project_name, $reference_locale = false)
     {
         $strings = [];
         if ($xml = simplexml_load_file($xliff_path)) {
-            $file_name = basename($xliff_path);
+            $file_name = $relative_file;
             $namespaces = $xml->getDocNamespaces();
             $xml->registerXPathNamespace('x', $namespaces['']);
             /*

app/config/config.ini-dev

@@ -10,7 +10,7 @@ install=${TRANSVISIONDIR}/
 config=${TRANSVISIONDIR}/app/config
 
 ; URL to l10n web service. Wrap the value between quotes if the URL contains special characters like ~
-l10nwebservice='https://flod.org/mozilla-l10n-query/mozilla-l10n-query/'
+l10nwebservice='https://flod.org/mozilla-l10n-query/'
 
 ; Only needed for production setup
 local_hg=/temp

app/config/config.ini-dist

@@ -23,7 +23,7 @@ install=/home/pascalc/github/transvision
 config=/home/pascalc/repos/github/transvision/app/config
 
 ; URL to l10n web service. Wrap the value between quotes if the URL contains special characters like ~
-l10nwebservice = "https://flod.org/mozilla-l10n-query/mozilla-l10n-query/"
+l10nwebservice = "https://flod.org/mozilla-l10n-query/"
 
 ; Flag to know if we are working in production mode or development mode
 dev=false

app/config/config.ini-ghactions

@@ -12,7 +12,7 @@ install=./
 config=app/config
 
 ; URL to l10n web service. Wrap the value between quotes if the URL contains special characters like ~
-l10nwebservice = "https://flod.org/mozilla-l10n-query/mozilla-l10n-query/"
+l10nwebservice = "https://flod.org/mozilla-l10n-query/"
 
 ; Path to the local Mercurial clones, both en-US and l10n. Could be external
 ; to the Git checkout of Transvision, as long as scripts have access to it.

app/scripts/tmx/tmx_xliff

@@ -36,6 +36,25 @@ error_log('Extraction of strings from XLIFF file');
 $base_path = isset($repo_data['git_subfolder'])
     ? GIT . "{$project_name}/{$repo_data['git_subfolder']}"
     : GIT . $project_name;
+
+$reference_locale = isset($repo_data['reference_locale'])
+    ? $repo_data['reference_locale']
+    : 'en-US';
+
+// Extract a list of reference files, with relative path to the locale folder
+$ref_path = "{$base_path}/{$reference_locale}";
+$iterator = new \RecursiveIteratorIterator(new \RecursiveDirectoryIterator($ref_path));
+$ref_files = array();
+foreach ($iterator as $file) {
+    $filename = $file->getFilename();
+    if (! $file->isDir()) {
+        if (! Strings::startsWith($filename, '.') && $file->getExtension() == 'xliff') {
+            $relative_filename = substr($file->getPathname(), strlen($ref_path) + 1);
+            $ref_files[] = $relative_filename;
+        }
+    }
+}
+
 foreach (Files::getFoldersInPath($base_path, ['templates']) as $locale_folder) {
     $out_translation = '';
     $total_strings = 0;
@@ -44,17 +63,14 @@ foreach (Files::getFoldersInPath($base_path, ['templates']) as $locale_folder) {
     $locale = isset($repo_data['underscore_locales']) && $repo_data['underscore_locales']
         ? str_replace('_', '-', $locale_folder)
         : $locale_folder;
+    $is_reference_locale = $locale == $reference_locale;
 
-    foreach ($repo_data['files'] as $file_name) {
-        $xliff_path = "{$base_path}/{$locale_folder}/{$file_name}";
+    $total_strings = 0;
+    foreach ($ref_files as $filename) {
+        $xliff_path = "{$base_path}/{$locale_folder}/{$filename}";
         if (file_exists($xliff_path)) {
-            // Make sure there is a reference locale defined
-            $reference_locale = isset($repo_data['reference_locale'])
-                ? $repo_data['reference_locale']
-                : 'en-US';
-            $is_reference_locale = $locale == $reference_locale;
-            $strings = Xliff::getStrings($xliff_path, $project_name, $is_reference_locale);
-            $total_strings = count($strings);
+            $strings = Xliff::getStrings($xliff_path, $filename, $project_name, $is_reference_locale);
+            $total_strings += count($strings);
             foreach ($strings as $string_id => $translation) {
                 $out_translation .= "'{$string_id}' => '{$translation}', \n";
             }

requirements.txt

@@ -1 +1 @@
-compare-locales==8.1.0
+compare-locales==8.2.1

tests/units/Transvision/Xliff.php

@@ -11,7 +11,7 @@ class Xliff extends atoum\test
     public function testGetStrings()
     {
         $obj = new _Xliff();
-        $strings = $obj->getStrings(TEST_FILES . 'xliff/firefox-ios.xliff', 'firefox_ios');
+        $strings = $obj->getStrings(TEST_FILES . 'xliff/firefox-ios.xliff', 'firefox-ios.xliff', 'firefox_ios');
 
         // Check total number of strings
         $this
@@ -40,7 +40,7 @@ public function testGetStrings()
     public function testGetStringsReference()
     {
         $obj = new _Xliff();
-        $strings = $obj->getStrings(TEST_FILES . 'xliff/firefox-ios.xliff', 'firefox_ios', true);
+        $strings = $obj->getStrings(TEST_FILES . 'xliff/firefox-ios.xliff', 'firefox-ios.xliff', 'firefox_ios', true);
 
         // Check total number of strings
         $this