Merge pull request #139 from JuDFTteam/release-0.9.1

🚀 Release 0.9.1
JuDFTteam · Apr 5, 2022 · 9841edc · 9841edc
2 parents b113747 + b85b53b
commit 9841edc
Show file tree

Hide file tree

Showing 13 changed files with 168 additions and 82 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -42,7 +42,7 @@ repos:
       ]
 
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.31.0
+    rev: v2.31.1
     hooks:
     -   id: pyupgrade
         args: [

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,15 @@
 # Changelog
 
+## v.0.9.1
+[full changelog](https://github.com/JuDFTteam/masci-tools/compare/v0.9.0...v0.9.1)
+
+### Added
+- Standalone function `masci_tools.tools.fleur_inpxml_converter.convert_inpxml` to allow conversions of `inp.xml` files within a python runtime without needing to go via the commandline
+
+### Bugfixes
+- Fixed bug in bokeh testing fixtures using the wrong folder for fallback versions
+- Fixed bug not correctly converting complex numbers from the Fleur xml files if they have whitespace at beginning/end
+
 ## v.0.9.0
 [full changelog](https://github.com/JuDFTteam/masci-tools/compare/v0.8.0...v0.9.0)
 

diff --git a/masci_tools/__init__.py b/masci_tools/__init__.py
@@ -21,7 +21,7 @@
 __copyright__ = ('Copyright (c), Forschungszentrum Jülich GmbH, IAS-1/PGI-1, Germany. '
                  'All rights reserved.')
 __license__ = 'MIT license, see LICENSE.txt file.'
-__version__ = '0.9.0'
+__version__ = '0.9.1'
 __authors__ = 'The JuDFT team. Also see AUTHORS.txt file.'
 
 logging.getLogger(__name__).addHandler(logging.NullHandler())
diff --git a/masci_tools/io/parsers/fleur_schema/fleur_schema_parser_functions.py b/masci_tools/io/parsers/fleur_schema/fleur_schema_parser_functions.py
@@ -14,14 +14,19 @@
 """
 from __future__ import annotations
 
+import sys
 from masci_tools.util.case_insensitive_dict import CaseInsensitiveDict, CaseInsensitiveFrozenSet
 from functools import wraps
-from typing import Callable, NamedTuple, Any
+from typing import Callable, NamedTuple, Any, overload
 from lxml import etree
 try:
     from typing import Literal, TypedDict
 except ImportError:
     from typing_extensions import Literal, TypedDict  #type: ignore[misc]
+if sys.version_info >= (3, 10):
+    from typing import TypeAlias
+else:
+    from typing_extensions import TypeAlias
 import warnings
 import math
 
@@ -35,7 +40,9 @@
 # The types defined here should not be reduced further and are associated with one clear base type
 # AngularMomentumNumberType and MainQuantumNumberType are here because they are integers
 # but are implemented as xsd:string with a regex
-BASE_TYPES = {
+BaseType: TypeAlias = Literal['int', 'switch', 'string', 'float', 'float_expression', 'complex']
+
+BASE_TYPES: dict[BaseType, set[str]] = {
     'switch': {'FleurBool'},
     'int': {
         'xsd:nonNegativeInteger', 'xsd:positiveInteger', 'xsd:integer', 'AngularMomentumNumberType',
@@ -46,6 +53,7 @@
     'string': {'xsd:string'},
     'complex': {'FortranComplex'}
 }
+
 NAMESPACES = {'xsd': 'http://www.w3.org/2001/XMLSchema'}
 
 
@@ -66,12 +74,17 @@ def convert_str_version_number(version_str: str) -> tuple[int, int]:
     return tuple(int(part) for part in version_numbers)  #type: ignore[return-value]
 
 
-class AttributeType(NamedTuple):
-    """Type for describing the types of attributes/text"""
+class _XSDAttributeType(NamedTuple):
+    """Type for describing the types of attributes/text. Can contain unfinished conversions"""
     base_type: str
     length: int | Literal['unbounded'] | None
 
 
+class AttributeType(_XSDAttributeType):
+    """Type for describing the types of attributes/text"""
+    base_type: Literal['int', 'switch', 'string', 'float', 'float_expression', 'complex']
+
+
 class TagInfo(TypedDict):
     """Dict representing the entries for the tag information.
     """
@@ -247,10 +260,32 @@ def _get_parent_fleur_type(elem: etree._Element,
     return parent, parent_type
 
 
-def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator,
-                    type_elem: etree._Element,
-                    convert_to_base: bool = True,
-                    basic_types_mapping: dict[str, list[AttributeType]] | None = None) -> list[AttributeType]:
+@overload
+def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, type_elem: etree._Element,
+                    basic_types_mapping: dict[str, list[AttributeType]] | None,
+                    convert_to_base: Literal[True]) -> list[AttributeType]:
+    ...
+
+
+@overload
+def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, type_elem: etree._Element,
+                    basic_types_mapping: dict[str, list[AttributeType]] | None,
+                    convert_to_base: Literal[False]) -> list[_XSDAttributeType]:
+    ...
+
+
+@overload
+def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, type_elem: etree._Element,
+                    basic_types_mapping: dict[str, list[AttributeType]] | None) -> list[AttributeType]:
+    ...
+
+
+def _get_base_types(
+    xmlschema_evaluator: etree.XPathDocumentEvaluator,
+    type_elem: etree._Element,
+    basic_types_mapping: dict[str, list[AttributeType]] | None = None,
+    convert_to_base: bool = True,
+) -> list[AttributeType] | list[_XSDAttributeType]:
     """
     Analyses the given type element to deduce its base_types and length restrictions
 
@@ -269,7 +304,7 @@ def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator,
 
     length = _get_length(xmlschema_evaluator, type_elem)
 
-    possible_types = set()
+    possible_types: set[_XSDAttributeType | AttributeType] = set()
     for child in type_elem:
         child_type = _normalized_name(child.tag)
 
@@ -293,9 +328,9 @@ def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator,
             for found_type in types:
 
                 if _is_base_type(found_type):
-                    possible_types.add(AttributeType(base_type=found_type, length=length))
+                    possible_types.add(_XSDAttributeType(base_type=found_type, length=length))
                 elif found_type in basic_types_mapping:
-                    possible_types.add(AttributeType(base_type=found_type, length=length))
+                    possible_types.add(_XSDAttributeType(base_type=found_type, length=length))
                 else:
                     sub_types = _xpath_eval(xmlschema_evaluator, '//xsd:simpleType[@name=$name]', name=found_type)
                     if len(sub_types) == 0:
@@ -316,7 +351,7 @@ def _get_base_types(xmlschema_evaluator: etree.XPathDocumentEvaluator,
 
                     if length != 1:
                         possible_types.update(
-                            AttributeType(base_type=base_type, length=length) for base_type, _ in new_types)
+                            _XSDAttributeType(base_type=base_type, length=length) for base_type, _ in new_types)
                     else:
                         possible_types.update(new_types)
 
@@ -1202,7 +1237,9 @@ def get_text_tags(xmlschema_evaluator: etree.XPathDocumentEvaluator, **kwargs: A
     return text_tags
 
 
-def get_basic_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, **kwargs: Any) -> dict[str, list[AttributeType]]:
+def get_basic_types(xmlschema_evaluator: etree.XPathDocumentEvaluator,
+                    input_basic_types: dict[str, list[AttributeType]] | None = None,
+                    **kwargs: Any) -> dict[str, list[AttributeType]]:
     """
     find all types, which can be traced back directly to a base_type
 
@@ -1211,31 +1248,34 @@ def get_basic_types(xmlschema_evaluator: etree.XPathDocumentEvaluator, **kwargs:
     :return: dictionary with type names and their corresponding type_definition
              meaning a dictionary with possible base types and evtl. length restriction
     """
-    basic_type_elems = _xpath_eval(xmlschema_evaluator, '//xsd:simpleType[@name]')
-    complex_type_elems = _xpath_eval(xmlschema_evaluator, '//xsd:complexType/xsd:simpleContent')
+    basic_type_elems: list[etree._Element] = _xpath_eval(xmlschema_evaluator, '//xsd:simpleType[@name]')
+    complex_type_elems: list[etree._Element] = _xpath_eval(xmlschema_evaluator, '//xsd:complexType/xsd:simpleContent')
 
     basic_types = {}
     for type_elem in basic_type_elems + complex_type_elems:
         if 'name' in type_elem.attrib:
-            type_name = type_elem.attrib['name']
+            type_name = str(type_elem.attrib['name'])
         else:
-            type_name = type_elem.getparent().attrib['name']
+            parent = type_elem.getparent()
+            if parent is None:
+                raise ValueError('Could not find parent')
+            type_name = str(parent.attrib['name'])
 
         if _is_base_type(type_name):
             continue  #Already a base type
 
-        types = _get_base_types(xmlschema_evaluator, type_elem, basic_types_mapping=kwargs.get('input_basic_types'))
+        types = _get_base_types(xmlschema_evaluator, type_elem, basic_types_mapping=input_basic_types)
 
         if type_name not in basic_types:
             basic_types[type_name] = types
         else:
             raise ValueError(f'Already defined type {type_name}')
 
     #Append the definitions form the inputschema since including it directly is very messy
-    if 'input_basic_types' in kwargs:
-        if any(key in basic_types for key in kwargs['input_basic_types']):
+    if input_basic_types is not None:
+        if any(key in basic_types for key in input_basic_types):
             raise ValueError('Doubled type definitions from Inputschema')
-        basic_types.update(kwargs['input_basic_types'])
+        basic_types.update(input_basic_types)
 
     return basic_types
 

diff --git a/masci_tools/testing/bokeh.py b/masci_tools/testing/bokeh.py
@@ -41,7 +41,7 @@ def _regression_bokeh_plot(bokeh_fig):
 
                 if not pytestconfig.getoption('--add-bokeh-version'):
                     if prev_version is not None:
-                        basename = Path(prev_version) / filename
+                        basename = Path(f'bokeh-{prev_version}') / filename
                         warnings.warn(f'Results for bokeh version {basename.parent} not available.'
                                       f'Using the last available version {prev_version}'
                                       'Use the option --add-bokeh-version to add results for this version')

diff --git a/masci_tools/tools/cf_calculation.py b/masci_tools/tools/cf_calculation.py
@@ -138,7 +138,10 @@ def __init__(self,
 
     @property
     def denNorm(self):
-        """DEPRECATED: Use density_normalization instead"""
+        """Returns the density normalization
+
+        DEPRECATED: Use density_normalization instead
+        """
         return self.density_normalization
 
     def stevens_prefactor(self, l: int, m: int) -> float:

diff --git a/masci_tools/tools/fleur_inpxml_converter.py b/masci_tools/tools/fleur_inpxml_converter.py
@@ -807,52 +807,24 @@ def _reorder_tree(parent: etree._Element, schema_dict: InputSchemaDict, base_xpa
         parent = _reorder_tags(parent, order)
 
 
-@click.group('inpxml')
-def inpxml():
-    """
-    Tool for converting inp.xml files to different versions
+def convert_inpxml(xmltree: etree._ElementTree, schema_dict: InputSchemaDict, to_version: str) -> etree._ElementTree:
     """
+    Convert the given xmltree to the given file version
 
+    :param xmltree: XML tree to convert
+    :param schema_dict: SchemaDict corresponding to the original file version
+    :param to_version: file version to which to convert
 
-@inpxml.command('convert')
-@click.argument('xml-file', type=click.Path(exists=True))
-@click.argument('to_version', type=str)
-@click.option('--output-file', '-o', type=str, default='inp.xml', help='Name of the output file')
-@click.option('--overwrite', is_flag=True, help='If the flag is given and the file already exists it is overwritten')
-@click.pass_context
-def convert_inpxml(ctx: click.Context, xml_file: FileLike, to_version: str, output_file: str, overwrite: bool) -> None:
-    """
-    Convert the given XML_FILE file to version TO_VERSION
-
-    XML_FILE is the file to convert
-    TO_VERSION is the file version of the finale input file
+    :returns: the XML tree converted to the given file version
     """
-    INCLUDE_NSMAP = {'xi': 'http://www.w3.org/2001/XInclude'}
-    INCLUDE_TAG = etree.QName(INCLUDE_NSMAP['xi'], 'include')
-    FALLBACK_TAG = etree.QName(INCLUDE_NSMAP['xi'], 'fallback')
-
-    xmltree, schema_dict = load_inpxml(xml_file)
     schema_dict_target = InputSchemaDict.fromVersion(to_version)
 
     #We want to leave comments in so we cannot use clear_xml for the xinclude feature
     #Here we just include and write out the complete xml file
     xmltree.xinclude()
-
     from_version = evaluate_attribute(xmltree, schema_dict, 'fleurInputVersion')
 
-    try:
-        conversion = load_conversion(from_version, to_version)
-    except FileNotFoundError:
-        echo.echo_warning(f'No conversion available between versions {from_version} to {to_version}')
-        if click.confirm('Do you want to generate this conversion now'):
-            conversion = ctx.invoke(generate_inp_conversion, from_version=from_version, to_version=to_version)
-        else:
-            echo.echo_critical('Cannot convert')
-
-    if Path(output_file).is_file():
-        if not overwrite:
-            echo.echo_critical(f'The output file {output_file} already exists. Use the overwrite flag to ignore')
-        echo.echo_warning(f'The output file {output_file} already exists. Will be overwritten')
+    conversion = load_conversion(from_version, to_version)
 
     set_attrib_value(xmltree, schema_dict_target, 'fleurInputVersion', to_version)
 
@@ -915,10 +887,61 @@ def convert_inpxml(ctx: click.Context, xml_file: FileLike, to_version: str, outp
 
     _reorder_tree(xmltree.getroot(), schema_dict_target)
 
+    validate_xml(xmltree, schema_dict_target.xmlschema, error_header='Input file does not validate against the schema')
+
+    #If there was no relax.xml included we need to rewrite the xinclude tag for it
+    INCLUDE_NSMAP = {'xi': 'http://www.w3.org/2001/XInclude'}
+    INCLUDE_TAG = etree.QName(INCLUDE_NSMAP['xi'], 'include')
+    FALLBACK_TAG = etree.QName(INCLUDE_NSMAP['xi'], 'fallback')
+    if not tag_exists(xmltree, schema_dict_target, 'relaxation'):
+        xinclude_elem = etree.Element(INCLUDE_TAG, href='relax.xml', nsmap=INCLUDE_NSMAP)
+        xinclude_elem.append(etree.Element(FALLBACK_TAG))
+        xmltree.getroot().append(xinclude_elem)
+
+    etree.indent(xmltree)
+    return xmltree
+
+
+@click.group('inpxml')
+def inpxml():
+    """
+    Tool for converting inp.xml files to different versions
+    """
+
+
+@inpxml.command('convert')
+@click.argument('xml-file', type=click.Path(exists=True))
+@click.argument('to_version', type=str)
+@click.option('--output-file', '-o', type=str, default='inp.xml', help='Name of the output file')
+@click.option('--overwrite', is_flag=True, help='If the flag is given and the file already exists it is overwritten')
+@click.pass_context
+def cmd_convert_inpxml(ctx: click.Context, xml_file: FileLike, to_version: str, output_file: str,
+                       overwrite: bool) -> None:
+    """
+    Convert the given XML_FILE file to version TO_VERSION
+
+    XML_FILE is the file to convert
+    TO_VERSION is the file version of the finale input file
+    """
+    xmltree, schema_dict = load_inpxml(xml_file)
+    from_version = evaluate_attribute(xmltree, schema_dict, 'fleurInputVersion')
+
     try:
-        validate_xml(xmltree,
-                     schema_dict_target.xmlschema,
-                     error_header='Input file does not validate against the schema')
+        load_conversion(from_version, to_version)
+    except FileNotFoundError:
+        echo.echo_warning(f'No conversion available between versions {from_version} to {to_version}')
+        if click.confirm('Do you want to generate this conversion now'):
+            ctx.invoke(generate_inp_conversion, from_version=from_version, to_version=to_version)
+        else:
+            echo.echo_critical('Cannot convert')
+
+    if Path(output_file).is_file():
+        if not overwrite:
+            echo.echo_critical(f'The output file {output_file} already exists. Use the overwrite flag to ignore')
+        echo.echo_warning(f'The output file {output_file} already exists. Will be overwritten')
+
+    try:
+        convert_inpxml(xmltree, schema_dict, to_version)
         echo.echo_success('The conversion was successful')
         echo.echo_info(
             'It is not guaranteed that a FLEUR calculation will behave in the exact same way as the old input file\n'
@@ -927,13 +950,6 @@ def convert_inpxml(ctx: click.Context, xml_file: FileLike, to_version: str, outp
         echo.echo_critical(
             f'inp.xml conversion did not finish successfully. The resulting file violates the XML schema with:\n {err}')
 
-    #If there was no relax.xml included we need to rewrite the xinclude tag for it
-    if not tag_exists(xmltree, schema_dict_target, 'relaxation'):
-        xinclude_elem = etree.Element(INCLUDE_TAG, href='relax.xml', nsmap=INCLUDE_NSMAP)
-        xinclude_elem.append(etree.Element(FALLBACK_TAG))
-        xmltree.getroot().append(xinclude_elem)
-
-    etree.indent(xmltree)
     xmltree.write(output_file, encoding='utf-8', pretty_print=True)
     echo.echo_success(f'Converted file written to {output_file}')
 

diff --git a/masci_tools/util/typing.py b/masci_tools/util/typing.py
@@ -26,18 +26,18 @@
 Type for xpath expressions
 """
 
-FileLike: TypeAlias = 'str | bytes | Path | os.PathLike[Any] | IO[Any]'
+FileLike: TypeAlias = Union[str, bytes, Path, os.PathLike, IO[Any]]
 """
 Type used for functions accepting file-like objects, i.e. handles or file paths
 """
 
-XMLFileLike: TypeAlias = 'etree._ElementTree | etree._Element | FileLike'
+XMLFileLike: TypeAlias = Union[etree._ElementTree, etree._Element, FileLike]
 """
 Type used for functions accepting xml-file-like objects, i.e. handles or file paths
 or already parsed xml objects
 """
 
-XMLLike: TypeAlias = 'etree._Element | etree._ElementTree'
+XMLLike: TypeAlias = Union[etree._Element, etree._ElementTree]
 """
 Type used for functions accepting xml objects from lxml
 """