Add support for recent Clang and Python releases (#618)

* Address Clang and Clang-Tidy diagnostics

* Update wrap.py for Python3.12 support

* Address wrap.py syntax errors

Remove Python2 built-in function 'xrange()'. Fix exception handlers. Fix
Pylint checks E0602 (undefined-variable), W0631 (undefined-loop-variable).

* Address Pylint warnings

Close opened file handles and fix Pylint checks W0622
(redefined-builtin) and R1721 (unnecessary-comprehension).

* Fix typos in wrap.py docstrings and comments

include/caliper/cali.h

@@ -592,7 +592,7 @@ void cali_configset_set(cali_configset_t cfg, const char* key, const char* value
  *
  * \return ID of the created channel.
  */
-cali_id_t cali_create_channel(const char* name, int flags, const cali_configset_t cfg);
+cali_id_t cali_create_channel(const char* name, int flags, cali_configset_t cfg);
 
 /**
  * \brief Delete a channel. Frees associated resources, e.g. blackboards,

include/caliper/common/Variant.h

@@ -97,7 +97,7 @@ class Variant
 
     static Variant unpack(const unsigned char* buf, size_t* inc, bool* ok = nullptr)
     {
-        return Variant(cali_variant_unpack(buf, inc, ok));
+        return {cali_variant_unpack(buf, inc, ok)};
     }
 
     static Variant from_string(cali_attr_type type, const char* str);

src/caliper/aggregate_over_mpi.cpp

@@ -167,7 +167,7 @@ void aggregate_over_mpi(CaliperMetadataDB& metadb, Aggregator& aggr, MPI_Comm co
     MPI_Comm_size(comm, &commsize);
     MPI_Comm_rank(comm, &rank);
 
-    for (int step = 0, steppow2 = 1; steppow2 < commsize; ++step, steppow2 *= 2) {
+    for (int steppow2 = 1; steppow2 < commsize; steppow2 *= 2) {
         // receive and merge
         if (rank % (2 * steppow2) == 0 && rank + steppow2 < commsize)
             ::receive_and_merge(rank + steppow2, metadb, aggr, comm);

src/services/mpiwrap/wrap.py

@@ -35,7 +35,7 @@
 '''Usage: wrap.py [-Gfgd] [-i pmpi_init] [-c mpicc_name] [-o file] wrapper.w [...]
  Python script for creating PMPI wrappers. Roughly follows the syntax of
    the Argonne PMPI wrapper generator, with some enhancements.
- Options:"
+ Options:
    -d             Just dump function declarations parsed out of mpi.h
    -G             Generate GOTCHA instead of PMPI wrappers
    -f             Generate fortran wrappers in addition to C wrappers.
@@ -74,29 +74,29 @@
 # In general, all MPI calls we care about return int.  We include double
 # to grab MPI_Wtick and MPI_Wtime, but we'll ignore the f2c and c2f calls
 # that return MPI_Datatypes and other such things.
-# MPI_Aint_add and MPI_Aint_diff return MPI_Aint, so include that too. 
+# MPI_Aint_add and MPI_Aint_diff return MPI_Aint, so include that too.
 rtypes = ['int', 'double', 'MPI_Aint' ]
 
 # If we find these strings in a declaration, exclude it from consideration.
 exclude_strings = [ "c2f", "f2c", "f082c", "f082f", "f2f08", "f90", "typedef", "MPI_T_", "MPI_Comm_spawn" ]
 
 # Regular expressions for start and end of declarations in mpi.h. These are
 # used to get the declaration strings out for parsing with formal_re below.
-begin_decl_re = re.compile("(" + "|".join(rtypes) + ")\s+(MPI_\w+)\s*\(")
+begin_decl_re = re.compile("(" + "|".join(rtypes) + r")\s+(MPI_\w+)\s*\(")
 exclude_re =    re.compile("|".join(exclude_strings))
-end_decl_re =   re.compile("\).*\;")
+end_decl_re =   re.compile(r"\).*\;")
 
 # Regular Expression for splitting up args. Matching against this
 # returns three groups: type info, arg name, and array info
 formal_re = re.compile(
-    "\s*(" +                       # Start type
-    "(?:const)?\s*" +              # Initial const
-    "\w+"                          # Type name (note: doesn't handle 'long long', etc. right now)
-    ")\s*(" +                      # End type, begin pointers
-    "(?:\s*\*(?:\s*const)?)*" +    # Look for 0 or more pointers with optional 'const'
-    ")\s*"                         # End pointers
-    "(?:(\w+)\s*)?" +              # Argument name. Optional.
-     "(\[.*\])?\s*$"               # Array type.  Also optional. Works for multidimensions b/c it's greedy.
+    r"\s*(" +                       # Start type
+    r"(?:const)?\s*" +              # Initial const
+    r"\w+"                          # Type name (note: doesn't handle 'long long', etc. right now)
+    r")\s*(" +                      # End type, begin pointers
+    r"(?:\s*\*(?:\s*const)?)*" +    # Look for 0 or more pointers with optional 'const'
+    r")\s*"                         # End pointers
+    r"(?:(\w+)\s*)?" +              # Argument name. Optional.
+    r"(\[.*\])?\s*$"                # Array type.  Also optional. Works for multidimensions b/c it's greedy.
     )
 
 # Fortran wrapper suffix
@@ -145,15 +145,15 @@
 #undef WRAP_PY_MPI_Fint
 '''
 
-gotcha_wrapper_includes = ''' 
+gotcha_wrapper_includes = '''
 #include <gotcha/gotcha.h>
 
-''' 
+'''
 
 # Macros used to suppress MPI deprecation warnings.
 wrapper_diagnosics_macros = '''
 /* Macros to enable and disable compiler warnings for deprecated functions.
- * These macros will be used to silent warnings about deprecated MPI functions,
+ * These macros will be used to silence warnings about deprecated MPI functions,
  * as these should be wrapped, even if they are deprecated.
  *
  * Note: The macros support GCC and clang compilers only. For other compilers
@@ -185,7 +185,7 @@
                         "MPI_Op", "MPI_Request", "MPI_Status", "MPI_Datatype", "MPI_Win" ])
 
 # MPI Calls that have array parameters, and mappings from the array parameter positions to the position
-# of the 'count' paramters that determine their size
+# of the 'count' parameters that determine their size
 mpi_array_calls = {
     "MPI_Startall"           : { 1:0 },
     "MPI_Testall"            : { 1:0, 3:0 },
@@ -225,10 +225,10 @@ def find_matching_paren(string, index, lparen='(', rparen=')'):
         return -1
 
 
-def isindex(str):
+def isindex(string):
     """True if a string is something we can index an array with."""
     try:
-        int(str)
+        int(string)
         return True
     except ValueError:
         return False
@@ -246,9 +246,9 @@ def conversion_prefix(handle_type):
         return handle_type
 
 # Special join function for joining lines together.  Puts "\n" at the end too.
-def joinlines(list, sep="\n"):
-    if list:
-        return sep.join(list) + sep
+def joinlines(lines, sep="\n"):
+    if lines:
+        return sep.join(lines) + sep
     else:
         return ""
 
@@ -257,16 +257,16 @@ def joinlines(list, sep="\n"):
 
 class Token:
     """Represents tokens; generated from input by lexer and fed to parse()."""
-    def __init__(self, type, value, line=0):
-        self.type = type    # Type of token
+    def __init__(self, ctype, value, line=0):
+        self.type = ctype   # Type of token
         self.value = value  # Text value
         self.line = line
 
     def __str__(self):
         return "'%s'" % re.sub(r'\n', "\\\\n", self.value)
 
-    def isa(self, type):
-        return self.type == type
+    def isa(self, ctype):
+        return self.type == ctype
 
 
 class LineTrackingLexer(object):
@@ -275,8 +275,8 @@ def __init__(self, lexicon):
         self.line_no = -1
         self.scanner = re.Scanner(lexicon)
 
-    def make_token(self, type, value):
-        token = Token(type, value, self.line_no)
+    def make_token(self, ctype, value):
+        token = Token(ctype, value, self.line_no)
         self.line_no += value.count("\n")
         return token
 
@@ -371,9 +371,9 @@ def __contains__(self, key):
     def __setitem__(self, key, value):
         self.map[key] = value
 
-    def include(self, map):
+    def include(self, new_elements):
         """Add entire contents of the map (or scope) to this scope."""
-        self.map.update(map)
+        self.map.update(new_elements)
 
 ################################################################################
 # MPI Semantics:
@@ -388,12 +388,12 @@ class Param:
        Doesn't represent a full parse, only the initial type information,
        name, and array info of the argument split up into strings.
     """
-    def __init__(self, type, pointers, name, array, pos):
-        self.type = type               # Name of arg's type (might include things like 'const')
+    def __init__(self, ctype, pointers, name, array, pos):
+        self.type = ctype              # Name of arg's type (might include things like 'const')
         self.pointers = pointers       # Pointers
         self.name = name               # Formal parameter name (from header or autogenerated)
         self.array = array             # Any array type information after the name
-        self.pos = pos                 # Position of arg in declartion
+        self.pos = pos                 # Position of arg in declaration
         self.decl = None               # This gets set later by Declaration
 
     def setDeclaration(self, decl):
@@ -465,7 +465,7 @@ def castType(self):
             if arr.count('[') > 1:
                 pointers += '(*)'   # need extra parens for, e.g., int[][3] -> int(*)[3]
             else:
-                pointers += '*'     # justa single array; can pass pointer.
+                pointers += '*'     # just a single array; can pass pointer.
             arr = arr.replace('[]', '')
         return "%s%s%s" % (self.type, pointers, arr)
 
@@ -572,7 +572,7 @@ def enumerate_mpi_declarations(mpicc, includes):
 
     # Run the mpicc -E on the temp file and pipe the output
     # back to this process for parsing.
-    string_includes = ["-I"+dir for dir in includes]
+    string_includes = ["-I" + path for path in includes]
     mpicc_cmd = "%s -E %s" % (mpicc, " ".join(string_includes))
     try:
         popen = subprocess.Popen("%s %s" % (mpicc_cmd, tmpname), shell=True,
@@ -621,13 +621,13 @@ def enumerate_mpi_declarations(mpicc, includes):
                         sys.stderr.write("MATCH FAILED FOR: '%s' in %s\n" % (arg, fn_name))
                         sys.exit(1)
 
-                    type, pointers, name, array = match.groups()
-                    types.add(type)
+                    ctype, pointers, name, array = match.groups()
+                    types.add(ctype)
                     all_pointers.add(pointers)
                     # If there's no name, make one up.
                     if not name: name = "arg_" + str(arg_num)
 
-                    decl.addArgument(Param(type.strip(), pointers, name, array, arg_num))
+                    decl.addArgument(Param(ctype.strip(), pointers, name, array, arg_num))
                 arg_num += 1
 
             yield decl
@@ -676,7 +676,7 @@ def write_gotcha_c_wrapper(out, decl, return_val, write_body):
 def write_pmpi_c_wrapper(out, decl, return_val, write_body):
     """Write the C wrapper for an MPI function."""
     # Write the PMPI prototype here in case mpi.h doesn't define it
-    # (sadly the case with some MPI implementaitons)
+    # (sadly the case with some MPI implementations)
     out.write(decl.pmpi_prototype(default_modifiers))
     out.write(";\n")
 
@@ -732,9 +732,9 @@ def __init__(self, decl, return_val):
         self.actuals = []
         self.mpich_actuals = []
 
-    def addTemp(self, type, name):
+    def addTemp(self, ctype, name):
         """Adds a temp var with a particular name.  Adds the same var only once."""
-        temp = "    %s %s;" % (type, name)
+        temp = "    %s %s;" % (ctype, name)
         self.temps.add(temp)
 
     def addActual(self, actual):
@@ -897,20 +897,21 @@ def decorate(fun):
         return fun
     return decorate
 
-def handle_list(list_name, list, args):
+def handle_list(list_name, array, args):
     """This function handles indexing lists used as macros in the wrapper generator.
        There are two syntaxes:
        {{<list_name>}}          Evaluates to the whole list, e.g. 'foo, bar, baz'
        {{<list_name> <index>}}  Evaluates to a particular element of a list.
     """
     if not args:
-        return list
+        return array
     else:
         len(args) == 1 or syntax_error("Wrong number of args for list expression.")
         try:
-            return list[int(args[0])]
+            index = int(args[0])
+            return array[index]
         except ValueError:
-            syntax_error("Invald index value: '%s'" % args[0])
+            syntax_error("Invalid index value: '%s'" % args[0])
         except IndexError:
             syntax_error("Index out of range in '%s': %d" % (list_name, index))
 
@@ -923,9 +924,9 @@ def __init__(self, decl):
 
     def __call__(self, out, scope, args, children):
         len(args) == 2 or syntax_error("Wrong number of args in apply macro.")
-        type, macro_name = args
+        ctype, macro_name = args
         for arg in self.decl.args:
-            if arg.cType() == type:
+            if arg.cType() == ctype:
                 out.write("%s(%s);\n" % (macro_name, arg.name))
 
 def include_decl(scope, decl):
@@ -951,7 +952,7 @@ def all_but(fn_list):
     """Return a list of all mpi functions except those in fn_list"""
     all_mpi = set(mpi_functions.keys())
     diff = all_mpi - set(fn_list)
-    return [x for x in sorted(diff)]
+    return list(sorted(diff))
 
 @macro("foreachfn", has_body=True)
 def foreachfn(out, scope, args, children):
@@ -1144,7 +1145,7 @@ def __init__(self):
 
     def iwrite(self, file, level, text):
         """Write indented text."""
-        for x in xrange(level):
+        for x in range(level):
             file.write("  ")
         file.write(text)
 
@@ -1228,9 +1229,9 @@ def push_tokens(self, iterable):
         self.tokens = itertools.chain(iter(iterable), iter([self.next]), self.tokens)
         self.gettok()
 
-    def accept(self, id):
+    def accept(self, idx):
         """Puts the next symbol in self.token if we like it.  Then calls gettok()"""
-        if self.next.isa(id):
+        if self.next.isa(idx):
             self.token = self.next
             self.gettok()
             return True
@@ -1239,9 +1240,9 @@ def accept(self, id):
     def unexpected_token(self):
         syntax_error("Unexpected token: %s." % self.next)
 
-    def expect(self, id):
+    def expect(self, idx):
         """Like accept(), but fails if we don't like the next token."""
-        if self.accept(id):
+        if self.accept(idx):
             return True
         else:
             if self.next:
@@ -1268,7 +1269,7 @@ def macro(self, accept_body_macros=True):
 
         if not accept_body_macros and self.is_body_macro(chunk.macro):
             syntax_error("Cannot use body macros in expression context: '%s'" % chunk.macro)
-            eys.exit(1)
+            sys.exit(1)
 
         while True:
             if self.accept(LBRACE):
@@ -1331,7 +1332,7 @@ def usage():
 try:
     opts, args = getopt.gnu_getopt(sys.argv[1:], "Gfsgdwc:o:i:I:")
 except getopt.GetoptError as err:
-    sys.stderr.write(err + "\n")
+    sys.stderr.write(str(err) + "\n")
     usage()
 
 for opt, arg in opts:
@@ -1348,7 +1349,7 @@ def usage():
         if stripped: includes.append(stripped)
     if opt == "-i":
         if not arg in pmpi_init_bindings:
-            sys.stderr.write("ERROR: PMPI_Init binding must be one of:\n    %s\n" % " ".join(possible_bindings))
+            sys.stderr.write("ERROR: PMPI_Init binding must be one of:\n    %s\n" % " ".join(pmpi_init_bindings))
             usage()
         else:
             pmpi_init_binding = arg
@@ -1371,14 +1372,15 @@ def usage():
     sys.exit(1)
 
 # If we're just dumping prototypes, we can just exit here.
-if dump_prototypes: sys.exit(0)
+if dump_prototypes:
+    sys.exit(0)
 
 # Open the output file here if it was specified
 if output_filename:
     try:
         output = open(output_filename, "w")
-    except IOError:
-        sys.stderr.write("Error: couldn't open file " + arg + " for writing.\n")
+    except:
+        sys.stderr.write("Error: couldn't open file " + output_filename + " for writing.\n")
         sys.exit(1)
 
 try:
@@ -1396,21 +1398,21 @@ def usage():
     # Parse each file listed on the command line and execute
     # it once it's parsed.
     fileno = 0
-    for f in args:
-        cur_filename = f
-        file = open(cur_filename)
+    for cur_filename in args:
+        f = open(cur_filename)
 
         # Outer scope contains fileno and the fundamental macros.
         outer_scope = Scope()
         outer_scope["fileno"] = str(fileno)
         outer_scope.include(macros)
 
         parser = Parser(macros)
-        chunks = parser.parse(file.read())
+        chunks = parser.parse(f.read())
 
         for chunk in chunks:
             chunk.evaluate(output, Scope(outer_scope))
         fileno += 1
+        f.close()
 
 except WrapSyntaxError:
     output.close()