General improvements

* [CORE] Code autoformatting initial support: black and flynt tools added, with a support linting script * [CORE] Remove six (python 2 & 3 compatibility layer) dependency * [CORE] Remove references to Python 2.* version * [TOOL] Add zero in valid ranges, and better handling of short names in mp_seq* tools * [PASS] Use a single branch target for absolute branches in BranchNextPass * [PASS] Improve InitializeMemoryFloatPass pass * [PASS] Add force_value parameter in InitializeRegisterPass * [PASS] Fix alignment in AutoAlignPass * [PASS] Handle storage to storage instructions in GenericMemoryStreamsPass * [MISC] Include the end of the specified range when processing input range parameter * [FIX] Fix typo in add_allow_register call Signed-off-by: Ramon Bertran Monfort <rbertra@us.ibm.com>
IBM · Oct 26, 2023 · 1c2f9d7 · 1c2f9d7
1 parent ac035ec
commit 1c2f9d7
Show file tree

Hide file tree

Showing 90 changed files with 1,545 additions and 1,459 deletions.
diff --git a/.gitignore b/.gitignore
@@ -35,6 +35,7 @@ tests_*.xml
 /cover*xml
 /venv*
 /pylint*.out
+/mypy*.out
 /doc.out
 /pycodestyle*.out
 /pycodestyle.err

diff --git a/README.md b/README.md
@@ -30,7 +30,7 @@ Required commands
 -----------------
 
 * git
-* python (2.7, 3.6, 3.7, 3.8, 3.9)
+* python (3.7, 3.8, 3.9)
 * virtualenv: https://virtualenv.pypa.io/en/stable/
 
 First-time set up

diff --git a/dev_tools/ci/environment.sh b/dev_tools/ci/environment.sh
@@ -101,6 +101,19 @@ if [ "$TRAVIS" = "true" ] && [ "$CI" = "true" ]; then
     export MP_TESTING_COMPILER_POWER_V300
     MP_TESTING_COMPILER_POWER_V310=$(command -v powerpc64le-linux-gnu-gcc)
     export MP_TESTING_COMPILER_POWER_V310
+elif [ "$(uname -m)" = "ppc64le" ]; then
+    set +e
+    MP_TESTING_COMPILER_POWER=$(command -v gcc)
+    export MP_TESTING_COMPILER_POWER
+    MP_TESTING_COMPILER_POWER_V206=$(command -v gcc)
+    export MP_TESTING_COMPILER_POWER_V206
+    MP_TESTING_COMPILER_POWER_V207=$(command -v gcc)
+    export MP_TESTING_COMPILER_POWER_V207
+    MP_TESTING_COMPILER_POWER_V300=$(command -v gcc)
+    export MP_TESTING_COMPILER_POWER_V300
+    MP_TESTING_COMPILER_POWER_V310=$(command -v gcc)
+    export MP_TESTING_COMPILER_POWER_V310
+    set -e
 else
     set +e
     MP_TESTING_COMPILER_POWER=$(command -v powerpc64le-linux-gnu-gcc)

diff --git a/dev_tools/distribution/microprobe_core.py b/dev_tools/distribution/microprobe_core.py
@@ -76,13 +76,12 @@ def read(fname):
             'mp_target = microprobe.definitions.generic.tools.mp_target:main',
         ]},
     install_requires=[
-        'ordereddict',
         'PyYAML',
         'rxjson',
         'argparse',
         'fasteners',
         'cachetools',
-        'six'],
+        ],
     long_description=read(
         os.path.join(
             ".",

diff --git a/dev_tools/utils/check_fix_style.sh b/dev_tools/utils/check_fix_style.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env sh 
+#!/usr/bin/env sh
 # Copyright 2011-2021 IBM Corporation
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,10 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 set -e
-if [ "$WORKSPACE" = "" ]; then  
-    WORKSPACE=$(pwd)        
+if [ "$WORKSPACE" = "" ]; then
+    WORKSPACE=$(pwd)
 	export WORKSPACE
-fi                                                                                                                                                                                                                                                                                     
+fi
+
+input=$*
+
 # shellcheck source=dev_tools/ci/environment.sh
 . "$WORKSPACE/dev_tools/ci/environment.sh"
 
@@ -30,7 +33,7 @@ check_style () {
     sed -i "s/^# constants$/# Constants/g" "$file"
 
     isort -ac -rc "$file"
-    
+
     # check_str "$file" "^# Futures$"
     # check_str "$file" "^# Built-in modules$"
     # check_str "$file" "^# Third party modules$"
@@ -48,34 +51,47 @@ check_style () {
     check_str "$file" "^# Functions"
     check_str "$file" "^# Classes"
 
-    if [ $error -ne 0 ]; then
-        exit 1
-    fi
+    # if [ $error -ne 0 ]; then
+    #    exit 1
+    # fi
+
+    python "$(command -v flynt)" -ll 79 -tc -tj "$file"
+    sed -i "s/ : /: /g" "$file"
+
+    python "$(command -v black)" -l 79 --target-version py39 "$file"
+    sed -i "s/ : /: /g" "$file"
 
     echo Check pycodestyle "$file"
+    set +e
     python "$(command -v pycodestyle)" --count --statistics "$file"
+    set -e
 
     echo Check pylint "$file"
     set +e
-    python "$(command -v pylint)" "$file" -r n -f parseable -d I0011,R0902,R0913,W0511,R0912,C0111,R0101,R0915,C0302,R0914,R0904,C0112,W0703,W0125 
+    python "$(command -v pylint)" "$file" -r n -f parseable -d I0011,R0902,R0913,W0511,R0912,C0111,R0101,R0915,C0302,R0914,R0904,C0112,W0703,W0125,C0209
     error=$?
     set -e
 
     if [ $error -ne 0 ]; then
-        echo Check "$file" 
-        exit 1
+        echo Check "$file"
     fi
 
 }
 
 check_str () {
     if [ "$(grep -c "$2" < "$1")" -ne 1 ]; then
-        echo "$1 fails $2"
+        echo "$1 fails $2 $(grep -c "$2" < "$1")"
         error=1
     fi
 }
 
-for file in $(find "$WORKSPACE/src/" "$WORKSPACE/targets/" -type f -name "*.py" | grep -v ./venv | grep -v ./deploy | sort); do 
-     check_style "$file"
+for i in $input; do
+    if [ -f "$i" ]; then
+        check_style "$i"
+    fi
 done;
+exit 0
 
+for file in $(find "$WORKSPACE/src/" "$WORKSPACE/targets/" -type f -name "*.py" | grep -v ./venv | grep -v ./deploy | sort); do
+     check_style "$file"
+done;
diff --git a/doc/source/conf.py b/doc/source/conf.py
@@ -337,7 +337,7 @@
 # intersphinx_mapping = {'http://docs.python.org/': None}
 
 intersphinx_mapping = {
-    'python': ('http://docs.python.org/2.7/', None),
+    'python': ('http://docs.python.org/3.9/', None),
 }
 
 autogen_output_path = "_rsts"

diff --git a/doc/source/devel_code_style.rst b/doc/source/devel_code_style.rst
@@ -8,11 +8,14 @@ project. In case of doubt, please :doc:`support_contact` the main developers.
 Code format checker
 -------------------
 
-We use *pylint* and *pycodestyle* as our code checkers. These tools plus the
-auto-format-on-save feature included in Eclipse will keep the code style of
-the project. Below there are the parameters used to configure these
-tools correctly. The goal is to have zero errors and warnings, and also
-comply with all the conventions suggested by these tools.
+We use *pylint* and *pycodestyle* as our code checkers, as well as *black*
+and *flynt* to automatically help in the formatting of code. You can 
+automatically fix code formatting issues by issuing the following command::
+
+  > ./dev_tools/utils/check_fix_style.sh $file
+
+The goal is to have zero errors and warnings, and also comply with all the
+conventions suggested by these tools.
 
 .. note::
 
@@ -25,9 +28,11 @@ project files::
   > ./dev_tools/ci/code_conventions_001_pycodestyle.sh
   > ./dev_tools/ci/code_conventions_002_pylint.sh
   > ./dev_tools/ci/code_conventions_003_documentation.sh
+  > ./dev_tools/ci/code_conventions_004_mypy.sh
 
 You can run them locally and fix any errors reported before committing
-changes to the repository.
+changes to the repository. These scripts are executed as a part of our
+continuous integration framework, and they will fail if an error is reported.
 
 .. note::
 
@@ -52,124 +57,15 @@ Pycodestyle
 We also use `pycodestyle <https://pypi.python.org/pypi/pycodestyle>`_ to
 verify that the code written follows the python style guide.
 
-Eclipse formating rules
-^^^^^^^^^^^^^^^^^^^^^^^
-
-In case you use the *pydev* extension of eclipse IDE to develop Microprobe,
-you can find below screenshots of the formatting configurations used so that it
-is more easy to follow the conventions and avoid pylint/pycodestyle errors.
-
-.. figure:: images/code_style_1.png
-   :alt: Code style tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
-
-   Code style tab
-
-.. figure:: images/code_style_2.png
-   :alt: Block comments tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
-
-   Block comments tab
-
-.. figure:: images/code_style_3.png
-   :alt: Code formatter tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
-
-   Code formatter tab
-
-.. figure:: images/code_style_4.png
-   :alt: Doctrings tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
-
-   Docstrings tab
-
-.. figure:: images/code_style_5.png
-   :alt: Imports tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
-
-   Imports tab
-
-.. figure:: images/code_style_6.png
-   :alt: Typing tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
-
-   Typing tab
-
-.. figure:: images/code_style_7.png
-   :alt: Tabs tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
-
-   Tabs tab
-
-.. figure:: images/code_style_8.png
-   :alt: Save actions tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
-
-   Save actions tab
-
-.. figure:: images/code_style_9.png
-   :alt: Code analysis unused tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
-
-   Code analysis unused tab
-
-.. figure:: images/code_style_10.png
-   :alt: Code analysis undefined tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
-
-   Code analysis undefined tab
-
-.. figure:: images/code_style_11.png
-   :alt: Code analysis imports tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
-
-   Code analysis imports tab
-
-.. figure:: images/code_style_12.png
-   :alt: Code analysis others tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
-
-   Code analysis others tab
-
-.. figure:: images/code_style_13.png
-   :alt: Code analysis pep8 tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
+Auto-formatting with black anf flynt
+------------------------------------
 
-   Code analysis pep8 tab
+*black* and *flynt* can be used as an auto-formatter tools. To do so, 
+you can use the following command::
 
-.. figure:: images/code_style_14.png
-   :alt: Pylint tab
-   :width: 90 %
-   :figwidth: 90 %
-   :align: center
+  > ./dev_tools/utils/check_fix_style.sh $file
 
-   Pylint tab
+Then, you can manually fix any remaining errors and warnings.
 
 Templates
 ---------
@@ -472,6 +368,6 @@ Useful links
 ------------
 
 - `rst features <http://docutils.sourceforge.net/docs/user/rst/quickref.html>`_
-- `docstring format <http://www.python.org/dev/peps/pep-0257/>`_
+- `docstring format <http://www.python.org/dev/peps/pe3.9257/>`_
 - `Sphinx examples <http://packages.python.org/an_example_pypi_project/sphinx.html>`_
 - `Sphinx docstring format <http://sphinx.pocoo.org/markup/desc.html>`_
diff --git a/doc/source/devel_repository.rst b/doc/source/devel_repository.rst
@@ -32,7 +32,7 @@ Required commands
 -----------------
 
 * **git**
-* **python** (>=2.7, 3.6, 3.7, 3.8, 3.9)
+* **python** (>=3.7, 3.8, 3.9)
 * **virtualenv**: https://virtualenv.pypa.io/en/stable/
 
 -----------------

diff --git a/doc/source/examples.rst b/doc/source/examples.rst
@@ -11,3 +11,6 @@ for new users, so that we can extend the current set of examples.
 .. toctree::
    :maxdepth: 1
 
+   examples_power
+   examples_riscv
+   examples_z
diff --git a/doc/source/start.rst b/doc/source/start.rst
@@ -11,3 +11,4 @@ Getting started
    start_install
    start_update
    start_check
+   start_quick_private
diff --git a/doc/source/start_install_git.rst b/doc/source/start_install_git.rst
@@ -35,7 +35,7 @@ Required commands
 -----------------
 
 * **git**
-* **python** 2.7 or >=3.6
+* **python** >=3.7
 * **virtualenv**: https://virtualenv.pypa.io/en/stable/
 
 -----------------

diff --git a/doc/source/start_organization.rst b/doc/source/start_organization.rst
@@ -49,3 +49,27 @@ microprobe_target_riscv               Package       All RISCV target/tools defin
 microprobe_target_power               Package       All PowerPC target/tools definitions
 ====================================  ============  ===========================================
 
+**POWER packages**:
+
+====================================  ============  ===========================================
+Name                                  Type          Description
+====================================  ============  ===========================================
+microprobe_target_power_priv          Package       All private POWER target/tools definitions
+====================================  ============  ===========================================
+
+**Private packages**:
+
+====================================  ============  ===========================================
+Name                                  Type          Description
+====================================  ============  ===========================================
+microprobe_target_private             Package       All private target/tools definitions
+====================================  ============  ===========================================
+
+**Z packages**:
+
+====================================  ============  ===========================================
+Name                                  Type          Description
+====================================  ============  ===========================================
+microprobe_target_z                   Package       All Z target/tools definitions
+====================================  ============  ===========================================
+