Skip to content

Commit

Permalink
Use pathlib in babel_runner.py (#12666)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AA-Turner
AA-Turner authored Jul 23, 2024
1 parent 1c682bc commit a69ca67
Showing 1 changed file with 83 additions and 75 deletions.
158 changes: 83 additions & 75 deletions utils/babel_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@

import json
import logging
import os
import sys
import tempfile
from os import environ
from pathlib import Path

from babel.messages.catalog import Catalog
from babel.messages.extract import (
Expand All @@ -31,8 +32,8 @@
from babel.util import pathmatch
from jinja2.ext import babel_extract as extract_jinja2

IS_CI = 'CI' in os.environ
ROOT = os.path.realpath(os.path.join(os.path.abspath(__file__), '..', '..'))
IS_CI = 'CI' in environ
ROOT = Path(__file__).parent.parent.resolve()
TEX_DELIMITERS = {
'variable_start_string': '<%=',
'variable_end_string': '%>',
Expand Down Expand Up @@ -87,42 +88,37 @@ def run_extract() -> None:
sphinx_version = line[14:].strip()[1:-1]
break

input_path = 'sphinx'
catalogue = Catalog(project='Sphinx', version=sphinx_version, charset='utf-8')

base = os.path.abspath(input_path)
for root, dirnames, filenames in os.walk(base):
relative_root = os.path.relpath(root, base) if root != base else ''
dirnames.sort()
for filename in sorted(filenames):
relative_name = os.path.join(relative_root, filename)
for pattern, method in METHOD_MAP:
if not pathmatch(pattern, relative_name):
continue

options = {}
for opt_pattern, opt_dict in OPTIONS_MAP.items():
if pathmatch(opt_pattern, relative_name):
options = opt_dict
with open(os.path.join(root, filename), 'rb') as fileobj:
for lineno, message, comments, context in extract(
method, # type: ignore[arg-type]
fileobj,
KEYWORDS,
options=options,
):
filepath = os.path.join(input_path, relative_name)
catalogue.add(
message,
None,
[(filepath, lineno)],
auto_comments=comments,
context=context,
)
break

output_file = os.path.join('sphinx', 'locale', 'sphinx.pot')
log.info('writing PO template file to %s', output_file)
base = ROOT / 'sphinx'
for filename in base.rglob('*'):
relative_name = filename.relative_to(base)
for pattern, method in METHOD_MAP:
if not pathmatch(pattern, str(relative_name)):
continue

options = {}
for opt_pattern, opt_dict in OPTIONS_MAP.items():
if pathmatch(opt_pattern, str(relative_name)):
options = opt_dict
with open(filename, 'rb') as fileobj:
for lineno, message, comments, context in extract(
method, # type: ignore[arg-type]
fileobj,
KEYWORDS,
options=options,
):
catalogue.add(
message,
None,
[(str(filename), lineno)],
auto_comments=comments,
context=context,
)
break

output_file = ROOT / 'sphinx' / 'locale' / 'sphinx.pot'
log.info('writing PO template file to %s', output_file.relative_to(ROOT))
with open(output_file, 'wb') as outfile:
write_po(outfile, catalogue)

Expand All @@ -132,34 +128,35 @@ def run_update() -> None:
log = _get_logger()

domain = 'sphinx'
locale_dir = os.path.join('sphinx', 'locale')
template_file = os.path.join(locale_dir, 'sphinx.pot')
locale_dir = ROOT / 'sphinx' / 'locale'
template_file = locale_dir / 'sphinx.pot'

with open(template_file, encoding='utf-8') as infile:
template = read_po(infile)

for locale in os.listdir(locale_dir):
filename = os.path.join(locale_dir, locale, 'LC_MESSAGES', f'{domain}.po')
if not os.path.exists(filename):
for locale in locale_dir.iterdir():
filename = locale / 'LC_MESSAGES' / f'{domain}.po'
if not filename.exists():
continue

log.info('updating catalog %s based on %s', filename, template_file)
log.info(
'updating catalogue %s based on %s',
filename.relative_to(ROOT),
template_file.relative_to(ROOT),
)
with open(filename, encoding='utf-8') as infile:
catalog = read_po(infile, locale=locale, domain=domain)
catalogue = read_po(infile, locale=locale.name, domain=domain)

catalog.update(template)
tmp_name = os.path.join(
os.path.dirname(filename),
tempfile.gettempprefix() + os.path.basename(filename),
)
catalogue.update(template)
tmp_name = filename.parent / (tempfile.gettempprefix() + filename.name)
try:
with open(tmp_name, 'wb') as tmpfile:
write_po(tmpfile, catalog)
write_po(tmpfile, catalogue)
except Exception:
os.remove(tmp_name)
tmp_name.unlink()
raise

os.replace(tmp_name, filename)
tmp_name.replace(filename)


def run_compile() -> None:
Expand All @@ -174,47 +171,59 @@ def run_compile() -> None:
"""
log = _get_logger()

directory = os.path.join('sphinx', 'locale')
directory = ROOT / 'sphinx' / 'locale'
total_errors = {}

for locale in os.listdir(directory):
po_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.po')
if not os.path.exists(po_file):
for locale in directory.iterdir():
po_file = locale / 'LC_MESSAGES' / 'sphinx.po'
if not po_file.exists():
continue

with open(po_file, encoding='utf-8') as infile:
catalog = read_po(infile, locale)
catalogue = read_po(infile, locale=locale.name)

if catalog.fuzzy:
log.info('catalog %s is marked as fuzzy, skipping', po_file)
if catalogue.fuzzy:
log.info('catalogue %s is marked as fuzzy, skipping', po_file.relative_to(ROOT))
continue

locale_errors = 0
for message, errors in catalog.check():
for message, errors in catalogue.check():
for error in errors:
locale_errors += 1
log.error(
'error: %s:%d: %s\nerror: in message string: %r',
po_file,
po_file.relative_to(ROOT),
message.lineno,
error,
message.string,
)

if locale_errors:
total_errors[locale] = locale_errors
log.info('%d errors encountered in %r locale, skipping', locale_errors, locale)
total_errors[locale.name] = locale_errors
log.info(
'%d errors encountered in %r locale, skipping',
locale_errors,
locale.name,
)
continue

mo_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.mo')
log.info('compiling catalog %s to %s', po_file, mo_file)
mo_file = locale / 'LC_MESSAGES' / 'sphinx.mo'
log.info(
'compiling catalogue %s to %s',
po_file.relative_to(ROOT),
mo_file.relative_to(ROOT),
)
with open(mo_file, 'wb') as outfile:
write_mo(outfile, catalog, use_fuzzy=False)
write_mo(outfile, catalogue, use_fuzzy=False)

js_file = os.path.join(directory, locale, 'LC_MESSAGES', 'sphinx.js')
log.info('writing JavaScript strings in catalog %s to %s', po_file, js_file)
js_file = locale / 'LC_MESSAGES' / 'sphinx.js'
log.info(
'writing JavaScript strings in catalogue %s to %s',
po_file.relative_to(ROOT),
js_file.relative_to(ROOT),
)
js_catalogue = {}
for message in catalog:
for message in catalogue:
if any(
x[0].endswith(('.js', '.js.jinja', '.js_t', '.html'))
for x in message.locations
Expand All @@ -227,8 +236,8 @@ def run_compile() -> None:
obj = json.dumps(
{
'messages': js_catalogue,
'plural_expr': catalog.plural_expr,
'locale': str(catalog.locale),
'plural_expr': catalogue.plural_expr,
'locale': str(catalogue.locale),
},
sort_keys=True,
indent=4,
Expand All @@ -240,10 +249,10 @@ def run_compile() -> None:
if total_errors:
_write_pr_body_line('## Babel catalogue errors')
_write_pr_body_line('')
for locale, err_count in total_errors.items():
log.error('error: %d errors encountered in %r locale.', err_count, locale)
for locale_name, err_count in total_errors.items():
log.error('error: %d errors encountered in %r locale.', err_count, locale_name)
s = 's' if err_count != 1 else ''
_write_pr_body_line(f'* {locale}: {err_count} error{s}')
_write_pr_body_line(f'* {locale_name}: {err_count} error{s}')


def _get_logger() -> logging.Logger:
Expand All @@ -269,7 +278,6 @@ def _write_pr_body_line(message: str) -> None:
print(__doc__, file=sys.stderr)
raise SystemExit(2) from None

os.chdir(ROOT)
if action == 'extract':
run_extract()
elif action == 'update':
Expand Down

0 comments on commit a69ca67

Please sign in to comment.