diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9e66eeaf0..ecbb9a155 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,7 +42,7 @@ repos: ] - repo: https://github.com/asottile/pyupgrade - rev: v2.31.0 + rev: v2.31.1 hooks: - id: pyupgrade args: [ diff --git a/CHANGELOG.md b/CHANGELOG.md index 67318a80b..b0fd277c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## v.0.9.1 +[full changelog](https://github.com/JuDFTteam/masci-tools/compare/v0.9.0...v0.9.1) + +### Added +- Standalone function `masci_tools.tools.fleur_inpxml_converter.convert_inpxml` to allow conversions of `inp.xml` files within a python runtime without needing to go via the commandline + +### Bugfixes +- Fixed bug in bokeh testing fixtures using the wrong folder for fallback versions +- Fixed bug not correctly converting complex numbers from the Fleur xml files if they have whitespace at beginning/end + ## v.0.9.0 [full changelog](https://github.com/JuDFTteam/masci-tools/compare/v0.8.0...v0.9.0) diff --git a/masci_tools/__init__.py b/masci_tools/__init__.py index e46a20022..cbdaa84b0 100644 --- a/masci_tools/__init__.py +++ b/masci_tools/__init__.py @@ -21,7 +21,7 @@ __copyright__ = ('Copyright (c), Forschungszentrum Jülich GmbH, IAS-1/PGI-1, Germany. ' 'All rights reserved.') __license__ = 'MIT license, see LICENSE.txt file.' -__version__ = '0.9.0' +__version__ = '0.9.1' __authors__ = 'The JuDFT team. Also see AUTHORS.txt file.' logging.getLogger(__name__).addHandler(logging.NullHandler()) diff --git a/masci_tools/io/parsers/fleur_schema/fleur_schema_parser_functions.py b/masci_tools/io/parsers/fleur_schema/fleur_schema_parser_functions.py index 334311300..cf8d7dbfd 100644 --- a/masci_tools/io/parsers/fleur_schema/fleur_schema_parser_functions.py +++ b/masci_tools/io/parsers/fleur_schema/fleur_schema_parser_functions.py @@ -14,14 +14,19 @@ """ from __future__ import annotations +import sys from masci_tools.util.case_insensitive_dict import CaseInsensitiveDict, CaseInsensitiveFrozenSet from functools import wraps -from typing import Callable, NamedTuple, Any +from typing import Callable, NamedTuple, Any, overload from lxml import etree try: from typing import Literal, TypedDict except ImportError: from typing_extensions import Literal, TypedDict #type: ignore[misc] +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias import warnings import math @@ -35,7 +40,9 @@ # The types defined here should not be reduced further and are associated with one clear base type # AngularMomentumNumberType and MainQuantumNumberType are here because they are integers # but are implemented as xsd:string with a regex -BASE_TYPES = { +BaseType: TypeAlias = Literal['int', 'switch', 'string', 'float', 'float_expression', 'complex'] + +BASE_TYPES: dict[BaseType, set[str]] = { 'switch': {'FleurBool'}, 'int': { 'xsd:nonNegativeInteger', 'xsd:positiveInteger', 'xsd:integer', 'AngularMomentumNumberType', @@ -46,6 +53,7 @@ 'string': {'xsd:string'}, 'complex': {'FortranComplex'} } + NAMESPACES = {'xsd': 'http://www.w3.org/2001/XMLSchema'} @@ -66,12 +74,17 @@ def convert_str_version_number(version_str: str) -> tuple[int, int]: return tuple(int(part) for part in version_numbers) #type: ignore[return-value] -class AttributeType(NamedTuple): - """Type for describing the types of attributes/text""" +class _XSDAttributeType(NamedTuple): + """Type for describing the types of attributes/text. Can contain unfinished conversions""" base_type: str length: int | Literal['unbounded'] | None +class AttributeType(_XSDAttributeType): + """Type for describing the types of attributes/text""" + base_type: Literal['int', 'switch', 'string', 'float', 'float_expression', 'complex'] + + class TagInfo(TypedDict): """Dict representing the entries for the tag information. """ @@ -247,10 +260,32 @@ def _get_parent_fleur_type(elem: etree._Element, return parent, parent_type -def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, - type_elem: etree._Element, - convert_to_base: bool = True, - basic_types_mapping: dict[str, list[AttributeType]] | None = None) -> list[AttributeType]: +@overload +def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, type_elem: etree._Element, + basic_types_mapping: dict[str, list[AttributeType]] | None, + convert_to_base: Literal[True]) -> list[AttributeType]: + ... + + +@overload +def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, type_elem: etree._Element, + basic_types_mapping: dict[str, list[AttributeType]] | None, + convert_to_base: Literal[False]) -> list[_XSDAttributeType]: + ... + + +@overload +def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, type_elem: etree._Element, + basic_types_mapping: dict[str, list[AttributeType]] | None) -> list[AttributeType]: + ... + + +def _get_base_types( + xmlschema_evaluator: etree.XPathDocumentEvaluator, + type_elem: etree._Element, + basic_types_mapping: dict[str, list[AttributeType]] | None = None, + convert_to_base: bool = True, +) -> list[AttributeType] | list[_XSDAttributeType]: """ Analyses the given type element to deduce its base_types and length restrictions @@ -269,7 +304,7 @@ def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, length = _get_length(xmlschema_evaluator, type_elem) - possible_types = set() + possible_types: set[_XSDAttributeType | AttributeType] = set() for child in type_elem: child_type = _normalized_name(child.tag) @@ -293,9 +328,9 @@ def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, for found_type in types: if _is_base_type(found_type): - possible_types.add(AttributeType(base_type=found_type, length=length)) + possible_types.add(_XSDAttributeType(base_type=found_type, length=length)) elif found_type in basic_types_mapping: - possible_types.add(AttributeType(base_type=found_type, length=length)) + possible_types.add(_XSDAttributeType(base_type=found_type, length=length)) else: sub_types = _xpath_eval(xmlschema_evaluator, '//xsd:simpleType[@name=$name]', name=found_type) if len(sub_types) == 0: @@ -316,7 +351,7 @@ def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, if length != 1: possible_types.update( - AttributeType(base_type=base_type, length=length) for base_type, _ in new_types) + _XSDAttributeType(base_type=base_type, length=length) for base_type, _ in new_types) else: possible_types.update(new_types) @@ -1202,7 +1237,9 @@ def get_text_tags(xmlschema_evaluator: etree.XPathDocumentEvaluator, **kwargs: A return text_tags -def get_basic_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, **kwargs: Any) -> dict[str, list[AttributeType]]: +def get_basic_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, + input_basic_types: dict[str, list[AttributeType]] | None = None, + **kwargs: Any) -> dict[str, list[AttributeType]]: """ find all types, which can be traced back directly to a base_type @@ -1211,20 +1248,23 @@ def get_basic_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, **kwargs: :return: dictionary with type names and their corresponding type_definition meaning a dictionary with possible base types and evtl. length restriction """ - basic_type_elems = _xpath_eval(xmlschema_evaluator, '//xsd:simpleType[@name]') - complex_type_elems = _xpath_eval(xmlschema_evaluator, '//xsd:complexType/xsd:simpleContent') + basic_type_elems: list[etree._Element] = _xpath_eval(xmlschema_evaluator, '//xsd:simpleType[@name]') + complex_type_elems: list[etree._Element] = _xpath_eval(xmlschema_evaluator, '//xsd:complexType/xsd:simpleContent') basic_types = {} for type_elem in basic_type_elems + complex_type_elems: if 'name' in type_elem.attrib: - type_name = type_elem.attrib['name'] + type_name = str(type_elem.attrib['name']) else: - type_name = type_elem.getparent().attrib['name'] + parent = type_elem.getparent() + if parent is None: + raise ValueError('Could not find parent') + type_name = str(parent.attrib['name']) if _is_base_type(type_name): continue #Already a base type - types = _get_base_types(xmlschema_evaluator, type_elem, basic_types_mapping=kwargs.get('input_basic_types')) + types = _get_base_types(xmlschema_evaluator, type_elem, basic_types_mapping=input_basic_types) if type_name not in basic_types: basic_types[type_name] = types @@ -1232,10 +1272,10 @@ def get_basic_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, **kwargs: raise ValueError(f'Already defined type {type_name}') #Append the definitions form the inputschema since including it directly is very messy - if 'input_basic_types' in kwargs: - if any(key in basic_types for key in kwargs['input_basic_types']): + if input_basic_types is not None: + if any(key in basic_types for key in input_basic_types): raise ValueError('Doubled type definitions from Inputschema') - basic_types.update(kwargs['input_basic_types']) + basic_types.update(input_basic_types) return basic_types diff --git a/masci_tools/testing/bokeh.py b/masci_tools/testing/bokeh.py index 3d3349b03..121612e44 100644 --- a/masci_tools/testing/bokeh.py +++ b/masci_tools/testing/bokeh.py @@ -41,7 +41,7 @@ def _regression_bokeh_plot(bokeh_fig): if not pytestconfig.getoption('--add-bokeh-version'): if prev_version is not None: - basename = Path(prev_version) / filename + basename = Path(f'bokeh-{prev_version}') / filename warnings.warn(f'Results for bokeh version {basename.parent} not available.' f'Using the last available version {prev_version}' 'Use the option --add-bokeh-version to add results for this version') diff --git a/masci_tools/tools/cf_calculation.py b/masci_tools/tools/cf_calculation.py index a2804d9e8..fc5550ed3 100644 --- a/masci_tools/tools/cf_calculation.py +++ b/masci_tools/tools/cf_calculation.py @@ -138,7 +138,10 @@ def __init__(self, @property def denNorm(self): - """DEPRECATED: Use density_normalization instead""" + """Returns the density normalization + + DEPRECATED: Use density_normalization instead + """ return self.density_normalization def stevens_prefactor(self, l: int, m: int) -> float: diff --git a/masci_tools/tools/fleur_inpxml_converter.py b/masci_tools/tools/fleur_inpxml_converter.py index 9b3a648ee..3cfb0df3d 100644 --- a/masci_tools/tools/fleur_inpxml_converter.py +++ b/masci_tools/tools/fleur_inpxml_converter.py @@ -807,52 +807,24 @@ def _reorder_tree(parent: etree._Element, schema_dict: InputSchemaDict, base_xpa parent = _reorder_tags(parent, order) -@click.group('inpxml') -def inpxml(): - """ - Tool for converting inp.xml files to different versions +def convert_inpxml(xmltree: etree._ElementTree, schema_dict: InputSchemaDict, to_version: str) -> etree._ElementTree: """ + Convert the given xmltree to the given file version + :param xmltree: XML tree to convert + :param schema_dict: SchemaDict corresponding to the original file version + :param to_version: file version to which to convert -@inpxml.command('convert') -@click.argument('xml-file', type=click.Path(exists=True)) -@click.argument('to_version', type=str) -@click.option('--output-file', '-o', type=str, default='inp.xml', help='Name of the output file') -@click.option('--overwrite', is_flag=True, help='If the flag is given and the file already exists it is overwritten') -@click.pass_context -def convert_inpxml(ctx: click.Context, xml_file: FileLike, to_version: str, output_file: str, overwrite: bool) -> None: - """ - Convert the given XML_FILE file to version TO_VERSION - - XML_FILE is the file to convert - TO_VERSION is the file version of the finale input file + :returns: the XML tree converted to the given file version """ - INCLUDE_NSMAP = {'xi': 'http://www.w3.org/2001/XInclude'} - INCLUDE_TAG = etree.QName(INCLUDE_NSMAP['xi'], 'include') - FALLBACK_TAG = etree.QName(INCLUDE_NSMAP['xi'], 'fallback') - - xmltree, schema_dict = load_inpxml(xml_file) schema_dict_target = InputSchemaDict.fromVersion(to_version) #We want to leave comments in so we cannot use clear_xml for the xinclude feature #Here we just include and write out the complete xml file xmltree.xinclude() - from_version = evaluate_attribute(xmltree, schema_dict, 'fleurInputVersion') - try: - conversion = load_conversion(from_version, to_version) - except FileNotFoundError: - echo.echo_warning(f'No conversion available between versions {from_version} to {to_version}') - if click.confirm('Do you want to generate this conversion now'): - conversion = ctx.invoke(generate_inp_conversion, from_version=from_version, to_version=to_version) - else: - echo.echo_critical('Cannot convert') - - if Path(output_file).is_file(): - if not overwrite: - echo.echo_critical(f'The output file {output_file} already exists. Use the overwrite flag to ignore') - echo.echo_warning(f'The output file {output_file} already exists. Will be overwritten') + conversion = load_conversion(from_version, to_version) set_attrib_value(xmltree, schema_dict_target, 'fleurInputVersion', to_version) @@ -915,10 +887,61 @@ def convert_inpxml(ctx: click.Context, xml_file: FileLike, to_version: str, outp _reorder_tree(xmltree.getroot(), schema_dict_target) + validate_xml(xmltree, schema_dict_target.xmlschema, error_header='Input file does not validate against the schema') + + #If there was no relax.xml included we need to rewrite the xinclude tag for it + INCLUDE_NSMAP = {'xi': 'http://www.w3.org/2001/XInclude'} + INCLUDE_TAG = etree.QName(INCLUDE_NSMAP['xi'], 'include') + FALLBACK_TAG = etree.QName(INCLUDE_NSMAP['xi'], 'fallback') + if not tag_exists(xmltree, schema_dict_target, 'relaxation'): + xinclude_elem = etree.Element(INCLUDE_TAG, href='relax.xml', nsmap=INCLUDE_NSMAP) + xinclude_elem.append(etree.Element(FALLBACK_TAG)) + xmltree.getroot().append(xinclude_elem) + + etree.indent(xmltree) + return xmltree + + +@click.group('inpxml') +def inpxml(): + """ + Tool for converting inp.xml files to different versions + """ + + +@inpxml.command('convert') +@click.argument('xml-file', type=click.Path(exists=True)) +@click.argument('to_version', type=str) +@click.option('--output-file', '-o', type=str, default='inp.xml', help='Name of the output file') +@click.option('--overwrite', is_flag=True, help='If the flag is given and the file already exists it is overwritten') +@click.pass_context +def cmd_convert_inpxml(ctx: click.Context, xml_file: FileLike, to_version: str, output_file: str, + overwrite: bool) -> None: + """ + Convert the given XML_FILE file to version TO_VERSION + + XML_FILE is the file to convert + TO_VERSION is the file version of the finale input file + """ + xmltree, schema_dict = load_inpxml(xml_file) + from_version = evaluate_attribute(xmltree, schema_dict, 'fleurInputVersion') + try: - validate_xml(xmltree, - schema_dict_target.xmlschema, - error_header='Input file does not validate against the schema') + load_conversion(from_version, to_version) + except FileNotFoundError: + echo.echo_warning(f'No conversion available between versions {from_version} to {to_version}') + if click.confirm('Do you want to generate this conversion now'): + ctx.invoke(generate_inp_conversion, from_version=from_version, to_version=to_version) + else: + echo.echo_critical('Cannot convert') + + if Path(output_file).is_file(): + if not overwrite: + echo.echo_critical(f'The output file {output_file} already exists. Use the overwrite flag to ignore') + echo.echo_warning(f'The output file {output_file} already exists. Will be overwritten') + + try: + convert_inpxml(xmltree, schema_dict, to_version) echo.echo_success('The conversion was successful') echo.echo_info( 'It is not guaranteed that a FLEUR calculation will behave in the exact same way as the old input file\n' @@ -927,13 +950,6 @@ def convert_inpxml(ctx: click.Context, xml_file: FileLike, to_version: str, outp echo.echo_critical( f'inp.xml conversion did not finish successfully. The resulting file violates the XML schema with:\n {err}') - #If there was no relax.xml included we need to rewrite the xinclude tag for it - if not tag_exists(xmltree, schema_dict_target, 'relaxation'): - xinclude_elem = etree.Element(INCLUDE_TAG, href='relax.xml', nsmap=INCLUDE_NSMAP) - xinclude_elem.append(etree.Element(FALLBACK_TAG)) - xmltree.getroot().append(xinclude_elem) - - etree.indent(xmltree) xmltree.write(output_file, encoding='utf-8', pretty_print=True) echo.echo_success(f'Converted file written to {output_file}') diff --git a/masci_tools/util/typing.py b/masci_tools/util/typing.py index 791e510fb..40c002100 100644 --- a/masci_tools/util/typing.py +++ b/masci_tools/util/typing.py @@ -26,18 +26,18 @@ Type for xpath expressions """ -FileLike: TypeAlias = 'str | bytes | Path | os.PathLike[Any] | IO[Any]' +FileLike: TypeAlias = Union[str, bytes, Path, os.PathLike, IO[Any]] """ Type used for functions accepting file-like objects, i.e. handles or file paths """ -XMLFileLike: TypeAlias = 'etree._ElementTree | etree._Element | FileLike' +XMLFileLike: TypeAlias = Union[etree._ElementTree, etree._Element, FileLike] """ Type used for functions accepting xml-file-like objects, i.e. handles or file paths or already parsed xml objects """ -XMLLike: TypeAlias = 'etree._Element | etree._ElementTree' +XMLLike: TypeAlias = Union[etree._Element, etree._ElementTree] """ Type used for functions accepting xml objects from lxml """ diff --git a/masci_tools/util/xml/common_functions.py b/masci_tools/util/xml/common_functions.py index b6f97a64f..a7d14e7a5 100644 --- a/masci_tools/util/xml/common_functions.py +++ b/masci_tools/util/xml/common_functions.py @@ -380,8 +380,7 @@ def check_complex_xpath(node: XMLLike | etree.XPathElementEvaluator, base_xpath: results_complex = set(eval_xpath(node, complex_xpath, list_return=True)) #type:ignore if not results_base.issuperset(results_complex): - raise ValueError( - f"Complex xpath '{str(complex_xpath)}' is not compatible with the base_xpath '{str(base_xpath)}'") + raise ValueError(f"Complex xpath '{complex_xpath!r}' is not compatible with the base_xpath '{base_xpath!r}'") def abs_to_rel_xpath(xpath: str, new_root: str) -> str: diff --git a/masci_tools/util/xml/converters.py b/masci_tools/util/xml/converters.py index 50e6ed4f9..ba71d6212 100644 --- a/masci_tools/util/xml/converters.py +++ b/masci_tools/util/xml/converters.py @@ -177,8 +177,7 @@ def convert_from_xml_explicit( all_success = False continue - types: tuple[BaseType, - ...] = tuple(definition.base_type for definition in text_definitions) #type: ignore[misc] + types = tuple(definition.base_type for definition in text_definitions) lengths = {definition.length for definition in text_definitions} if len(text_definitions) == 1: @@ -260,7 +259,7 @@ def convert_to_xml_explicit(value: Any | Iterable[Any], all_success = False continue - types: tuple[BaseType, ...] = tuple(definition.base_type for definition in text_definitions) #type:ignore[misc] + types = tuple(definition.base_type for definition in text_definitions) converted_text, suc = convert_to_xml_single_values(val, types, logger=logger, float_format=float_format) all_success = all_success and suc @@ -505,6 +504,8 @@ def convert_from_fortran_complex(number_str: str) -> complex: :returns: complex number """ + number_str = number_str.strip() + RE_COMPLEX_NUMBER = r'\([-+]?(?:\d*\.\d+|\d+)\,[-+]?(?:\d*\.\d+|\d+)\)' RE_SINGLE_FLOAT = r'[-+]?(?:\d*\.\d+|\d+)' diff --git a/pyproject.toml b/pyproject.toml index 54bbc9081..614051144 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -199,7 +199,7 @@ split_arguments_when_comma_terminated = true indent_dictionary_value = false [bumpver] -current_version = "0.9.0" +current_version = "0.9.1" version_pattern = "MAJOR.MINOR.PATCH[TAGNUM]" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/tests/cmdline/test_inpxml_converter.py b/tests/cmdline/test_inpxml_converter.py index d0ae59701..0f2906859 100644 --- a/tests/cmdline/test_inpxml_converter.py +++ b/tests/cmdline/test_inpxml_converter.py @@ -31,12 +31,12 @@ def test_convert_inpxml(tmp_path, test_file, file_regression): """ Test of the generate-conversion command """ - from masci_tools.tools.fleur_inpxml_converter import convert_inpxml + from masci_tools.tools.fleur_inpxml_converter import cmd_convert_inpxml from click.testing import CliRunner runner = CliRunner() result = runner.invoke( - convert_inpxml, + cmd_convert_inpxml, [test_file('fleur/aiida_fleur/inpxml/FePt/inp.xml'), '0.34', '--output-file', os.fspath(tmp_path / 'inp.xml')]) print(result.output) @@ -46,3 +46,19 @@ def test_convert_inpxml(tmp_path, test_file, file_regression): content = f.read() file_regression.check(content, extension='.xml') + + +def test_convert_inpxml_function(file_regression, load_inpxml): + """ + Test of the generate-conversion command + """ + from masci_tools.tools.fleur_inpxml_converter import convert_inpxml + from lxml import etree + + xmltree, schema_dict = load_inpxml('fleur/aiida_fleur/inpxml/FePt/inp.xml', absolute=False) + xmltree = convert_inpxml(xmltree, schema_dict, '0.34') + + content = etree.tostring(xmltree, encoding='unicode', pretty_print=True) + + #This function should produce the same output as the full click command + file_regression.check(content, extension='.xml', basename='test_convert_inpxml') diff --git a/tests/xml/test_xml_converters.py b/tests/xml/test_xml_converters.py index efb82cc3f..c007715b2 100644 --- a/tests/xml/test_xml_converters.py +++ b/tests/xml/test_xml_converters.py @@ -55,7 +55,8 @@ def test_convert_to_fortran_bool(): @pytest.mark.parametrize('text,expected,error', (('(1.52,3.14)', 1.52 + 3.14j, False), ('(-1.52,+3.14)', -1.52 + 3.14j, False), - ('(1.52.12,-3.14)', None, True), ('(1.43,not-anumber)', None, True))) + (' (1.52,3.14) ', 1.52 + 3.14j, False), ('(1.52.12,-3.14)', None, True), + ('(1.43,not-anumber)', None, True))) def test_convert_from_fortran_complex(text, expected, error): """ Test of the convert_from_fortran_complex function