Merge pull request #154 from paulgazz/2to3_conversion

Convert all python source to version 3

kmax/alg.py

@@ -9,11 +9,12 @@
 from collections import defaultdict
 
 import pdb
+from functools import reduce
 trace = pdb.set_trace
 import z3
 import kmax.vcommon as CM
 
-from datastructures import CondDef, Multiverse, VarEntry, BoolVar, Results
+from .datastructures import CondDef, Multiverse, VarEntry, BoolVar, Results
 
 import kmax.settings
 mlog = CM.getLogger(__name__, kmax.settings.logger_level)
@@ -166,11 +167,11 @@ def getSymbTable(self, printCond=None):
     def get_presence_conditions(self, vars, pcs, cond, zcond, visited=set()):
         names = set()
         for var in vars:
-            if var in self.variables.keys():
+            if var in list(self.variables.keys()):
                 names = names.union(self.get_var_equiv_set(var))
 
         # ignore undefined vars
-        names = [ name for name in names if name in self.variables.keys() ]
+        names = [ name for name in names if name in list(self.variables.keys()) ]
 
         # prevent cycles, e.g., linux-3.19/net/mac80211/Makefile
         names = [ name for name in names if name not in visited ]
@@ -190,7 +191,7 @@ def get_presence_conditions(self, vars, pcs, cond, zcond, visited=set()):
                 for token in tokens:
                     and_cond = conj(cond, bdd_condition)
                     and_zcond = zconj(zcond, z3_condition)
-                    if token not in pcs.keys():
+                    if token not in list(pcs.keys()):
                         pcs[token] = and_zcond
                     else:
                         pcs[token] = zdisj(pcs[token], and_zcond)
@@ -216,7 +217,7 @@ def deduplicate_and_add_path(self, presence_conditions, path=None, updated_prese
         for token in presence_conditions:
             # resolve any uses of ../ or ./
             filename = os.path.join(path, token)
-            if filename not in updated_presence_conditions.keys():
+            if filename not in list(updated_presence_conditions.keys()):
                 updated_presence_conditions[filename] = presence_conditions[token]
             else:
                 updated_presence_conditions[filename] = zdisj(updated_presence_conditions[filename], presence_conditions[token])
@@ -823,12 +824,10 @@ def add_var(self, name, presence_cond, presence_zcond, token, value):
         assert value is not None, value
 
         update_vars = lambda name: \
-                    map(lambda (old_value, old_cond, old_zcond, old_flavor): 
-                            VarEntry(old_value, 
-                                    conj(old_cond, neg(presence_cond)),
-                                    zconj(old_zcond, z3.Not(presence_zcond)),
-                                    old_flavor), 
-                        self.variables[name])
+                    [VarEntry(old_value_old_cond_old_zcond_old_flavor[0], 
+                                    conj(old_value_old_cond_old_zcond_old_flavor[1], neg(presence_cond)),
+                                    zconj(old_value_old_cond_old_zcond_old_flavor[2], z3.Not(presence_zcond)),
+                                    old_value_old_cond_old_zcond_old_flavor[3]) for old_value_old_cond_old_zcond_old_flavor in self.variables[name]]
 
         if token == "=":
             # Recursively-expanded variable defs are expanded at use-time
@@ -1240,10 +1239,10 @@ def extract(self, path):
         presence_conditions = {}
         kbuild.get_presence_conditions([ "subdir-y", "subdir-m", "SPECIAL-subdir-simple" ], presence_conditions, kbuild.T, ZSolver.T)
         subdirs_pcs = kbuild.deduplicate_and_add_path(presence_conditions, path)
-        subdirs = subdirs_pcs.keys()
+        subdirs = list(subdirs_pcs.keys())
 
         subdirs_pcs_fixed = {}
-        for subdir in subdirs_pcs.keys():
+        for subdir in list(subdirs_pcs.keys()):
             # add / to end of subdirs
             if not subdir.endswith("/"):
                 fixed_subdir = subdir + "/"
@@ -1256,27 +1255,27 @@ def extract(self, path):
             self.results.units_by_type['subdirs'] = subdirs
 
             # mapping from unit type name to structure holding them
-            self.results.units_by_type['compilation_units'] = self.results.presence_conditions.keys()
+            self.results.units_by_type['compilation_units'] = list(self.results.presence_conditions.keys())
 
             kbuild.process_stmts(parser.parsestring("SPECIAL-extra := $(extra-y)", makefile.name), kbuild.T, ZSolver.T)
             presence_conditions = {}
             kbuild.get_presence_conditions([ "SPECIAL-extra", "extra-y" ], presence_conditions, kbuild.T, ZSolver.T)
-            self.results.units_by_type['extra'] = kbuild.deduplicate_and_add_path(presence_conditions, path).keys()
+            self.results.units_by_type['extra'] = list(kbuild.deduplicate_and_add_path(presence_conditions, path).keys())
 
             kbuild.process_stmts(parser.parsestring("SPECIAL-hostprogs := $(hostprogs-y) $(hostprogs-m) $(hostprogs) $(always)", makefile.name), kbuild.T, ZSolver.T)
             presence_conditions = {}
             kbuild.get_presence_conditions([ "SPECIAL-hostprogs", "hostprogs-y", "hostprogs-m", "hostprogs", "always" ], presence_conditions, kbuild.T, ZSolver.T)
-            self.results.units_by_type['hostprog_units'] = kbuild.deduplicate_and_add_path(presence_conditions, path).keys()
+            self.results.units_by_type['hostprog_units'] = list(kbuild.deduplicate_and_add_path(presence_conditions, path).keys())
 
             kbuild.process_stmts(parser.parsestring("SPECIAL-targets := $(targets)", makefile.name), kbuild.T, ZSolver.T)
             presence_conditions = {}
             kbuild.get_presence_conditions([ "SPECIAL-targets", "targets" ], presence_conditions, kbuild.T, ZSolver.T)
-            self.results.units_by_type['targets'] = kbuild.deduplicate_and_add_path(presence_conditions, path).keys()
+            self.results.units_by_type['targets'] = list(kbuild.deduplicate_and_add_path(presence_conditions, path).keys())
 
             kbuild.process_stmts(parser.parsestring("SPECIAL-clean-files := $(clean-files)", makefile.name), kbuild.T, ZSolver.T)
             presence_conditions = {}
             kbuild.get_presence_conditions([ "SPECIAL-clean-files", "clean-files" ], presence_conditions, kbuild.T, ZSolver.T)
-            self.results.units_by_type['clean_files'] = kbuild.deduplicate_and_add_path(presence_conditions, path).keys()
+            self.results.units_by_type['clean_files'] = list(kbuild.deduplicate_and_add_path(presence_conditions, path).keys())
 
             # finds compilation units in obj-, lib-.  such units were
             # specified to be buildable, but the configuration options
@@ -1306,7 +1305,7 @@ def extract(self, path):
 
             presence_conditions = {}
             kbuild.get_presence_conditions(unconfigurable_variables, presence_conditions, kbuild.T, ZSolver.T)
-            self.results.units_by_type['unconfigurable_units'] = kbuild.deduplicate_and_add_path(presence_conditions, path).keys()
+            self.results.units_by_type['unconfigurable_units'] = list(kbuild.deduplicate_and_add_path(presence_conditions, path).keys())
 
         if kmax.settings.do_table:
             mlog.info(kbuild.getSymbTable(printCond=kbuild.bdd_to_str))

kmax/arch.py

@@ -731,7 +731,7 @@ def generate_kextract(self):
       self.__logger.debug("Automatically detecting the kextract module version suitable for the kernel.")
       kextract_version=self.__detect_kextract_version()
       self.__logger.debug("Using kextract module version: %s" % kextract_version)
-    elif self.__kextract_version not in kmax.kextractcommon.module_versions.keys():
+    elif self.__kextract_version not in list(kmax.kextractcommon.module_versions.keys()):
       raise Arch.UnknownKextractVersion(self.__kextract_version)
     else:
       kextract_version=self.__kextract_version

kmax/common.py

@@ -54,24 +54,24 @@ def debug(msg, ending="\n"):
 
 def get_kmax_constraints(kmax_formulas, kbuild_path, view=False):
   """Returns an array of z3 formulas that, ANDed together, represent conditions under which the unit is included in a build, as specified by Kbuild.  Note that this excludes any Kconfig constraints, which are handled by kclause instead."""
-  if kbuild_path in kmax_formulas.keys():
+  if kbuild_path in list(kmax_formulas.keys()):
     kmax_constraints = []
     # add the condition for the compilation unit and each of its parent directories
     comp_unit_constraint = z3.parse_smt2_string(kmax_formulas[kbuild_path])
     kmax_constraints.extend(comp_unit_constraint)
     if view:
-      print("%s\n%s\n" % (kbuild_path, comp_unit_constraint))
+      print(("%s\n%s\n" % (kbuild_path, comp_unit_constraint)))
     if '/' in kbuild_path:
       subpath, basename = kbuild_path.rsplit('/', 1)
       elems = subpath.rsplit('/')
       for i in range(0, len(elems)):
         subarray = elems[0:(len(elems) - i)]
         subsubpath = '/'.join(subarray) + "/"
-        if subsubpath in kmax_formulas.keys():
+        if subsubpath in list(kmax_formulas.keys()):
           subsubpath_constraint = z3.parse_smt2_string(kmax_formulas[subsubpath])
           kmax_constraints.extend(subsubpath_constraint)
           if view:
-            print("%s\n%s\n" % (subsubpath, subsubpath_constraint))
+            print(("%s\n%s\n" % (subsubpath, subsubpath_constraint)))
         else:
           debug("%s has no kmax formula, assuming it is unconstrained." % (subsubpath))
     return kmax_constraints

kmax/datastructures.py

@@ -77,7 +77,7 @@ def dedup(self):
                 cache[val] = (cond, zcond)
 
         mv = Multiverse([CondDef(c, zc, v)
-                         for v, (c, zc)  in cache.iteritems()])
+                         for v, (c, zc)  in cache.items()])
         assert mv
         return mv
 
@@ -145,7 +145,7 @@ def __str__(self, details=False):
             # legacy output format
             f = lambda k, s: "{}: {}".format(k, ', '.join(s) if details else len(s))
             delim = '\n' if details else ', '
-            ss = delim.join(f(k, s) for k, s in self.__dict__.iteritems() if s)
+            ss = delim.join(f(k, s) for k, s in self.__dict__.items() if s)
             if self.presence_conditions:
                 ss += '\n{} presence conditions: \n{}'.format(len(self.presence_conditions), self.z3_str(self.presence_conditions))
             # if self.unit_pcs:
@@ -155,7 +155,7 @@ def __str__(self, details=False):
             return ss
         else:
             z3_pcs = {}
-            for filename in self.presence_conditions.keys():
+            for filename in list(self.presence_conditions.keys()):
                 solver = z3.Solver()
                 solver.add(self.presence_conditions[filename])
                 z3_pcs[filename] = solver.to_smt2()
@@ -257,5 +257,5 @@ def z3_str(self, path_conds):
         #         if subsubpath in subdir_pcs: print subsubpath, subdir_pcs[subsubpath]
 
         result = '\n'.join(self.get_line_format(filename).format(filename, self.to_exp(z3.simplify(path_conds[filename])))
-                           for filename in path_conds.keys())
+                           for filename in list(path_conds.keys()))
         return result

kmax/expression_converter.py

@@ -98,7 +98,7 @@ def visit_Constant(self, node):
     if result is not None:
       node.z3 = result
     else:
-      print(traceback.format_exc())
+      print((traceback.format_exc()))
       sys.stderr.write("error: cannot process value \"%s\"\n" % (value))
       exit(1)
       node.z3 = None
@@ -116,7 +116,7 @@ def visit_Str(self, node):  # deprecated since 3.8, replaced by constant
     if result is not None:
       node.z3 = result
     else:
-      print(traceback.format_exc())
+      print((traceback.format_exc()))
       sys.stderr.write("error: cannot process value \"%s\"\n" % (value))
       exit(1)
       node.z3 = None
@@ -235,7 +235,7 @@ def convert_to_z3(expr):
 
 if __name__ == '__main__':
   # tests
-  print(convert_to_z3('((((not ((((CONFIG_GCOV_KERNEL and CONFIG_CC_IS_GCC and 1)) and (((1 and CONFIG_GCC_VERSION < "40700")) or ((1)))))) or (CONFIG_GCOV_FORMAT_3_4 or CONFIG_GCOV_FORMAT_4_7))) and (((not (CONFIG_GCOV_FORMAT_3_4 or CONFIG_GCOV_FORMAT_4_7)) or ((CONFIG_GCOV_KERNEL and CONFIG_CC_IS_GCC and 1)))))'))
+  print((convert_to_z3('((((not ((((CONFIG_GCOV_KERNEL and CONFIG_CC_IS_GCC and 1)) and (((1 and CONFIG_GCC_VERSION < "40700")) or ((1)))))) or (CONFIG_GCOV_FORMAT_3_4 or CONFIG_GCOV_FORMAT_4_7))) and (((not (CONFIG_GCOV_FORMAT_3_4 or CONFIG_GCOV_FORMAT_4_7)) or ((CONFIG_GCOV_KERNEL and CONFIG_CC_IS_GCC and 1)))))')))
   # print(convert_to_z3('((((not ((((1)) and (((1)) or ((1 and CONFIG_GCC_PLUGINS)) or ((1 and CONFIG_GCC_PLUGINS and ( not CONFIG_KASAN or CONFIG_KASAN_STACK!="1"))) or ((1 and CONFIG_GCC_PLUGINS and ( not CONFIG_KASAN or CONFIG_KASAN_STACK!="1"))) or ((1 and CONFIG_CC_HAS_AUTO_VAR_INIT)))))) or (CONFIG_INIT_STACK_NONE or CONFIG_GCC_PLUGIN_STRUCTLEAK_USER or CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF or CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL or CONFIG_INIT_STACK_ALL))) and (((not (CONFIG_INIT_STACK_NONE or CONFIG_GCC_PLUGIN_STRUCTLEAK_USER or CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF or CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL or CONFIG_INIT_STACK_ALL)) or ((1)))))'))
   # print(convert_to_z3('((((not ((((1)) and (((1 and CONFIG_X86_64)) or ((1)) or ((1 and CONFIG_EXPERT and  not CONFIG_STACKDEPOT)))))) or (CONFIG_UNWINDER_ORC or CONFIG_UNWINDER_FRAME_POINTER or CONFIG_UNWINDER_GUESS))) and (((not (CONFIG_UNWINDER_ORC or CONFIG_UNWINDER_FRAME_POINTER or CONFIG_UNWINDER_GUESS)) or ((1)))))'))
   # print(get_identifiers('((((not ((((1)) and (((1 and CONFIG_X86_64)) or ((1)) or ((1 and CONFIG_EXPERT and  not CONFIG_STACKDEPOT)))))) or (CONFIG_UNWINDER_ORC or CONFIG_UNWINDER_FRAME_POINTER or CONFIG_UNWINDER_GUESS))) and (((not (CONFIG_UNWINDER_ORC or CONFIG_UNWINDER_FRAME_POINTER or CONFIG_UNWINDER_GUESS)) or ((1)))))'))

kmax/find_selectable.py

@@ -78,7 +78,7 @@ def visit_Str(self, node):  # deprecated since 3.8, replaced by constant
 
   def visit_Name(self, node):
     option = node.id
-    if option in self.find_selectable.selectable_options.keys():
+    if option in list(self.find_selectable.selectable_options.keys()):
       node.selectable = self.find_selectable.selectable_options[option]
     else:
       node.selectable = self.find_selectable.get_selectable_one(option)
@@ -113,13 +113,13 @@ def get_selectable(self, pending_options):
     return self.selectable_options
 
   def get_selectable_one(self, current_option):
-    if current_option not in self.selectable_options.keys():  # not yet memoized
+    if current_option not in list(self.selectable_options.keys()):  # not yet memoized
       # if no (direct or reverse) dependencies, assume it is selectable
-      if current_option not in self.dep_exprs.keys() and current_option not in self.rev_dep_exprs.keys():
+      if current_option not in list(self.dep_exprs.keys()) and current_option not in list(self.rev_dep_exprs.keys()):
         self.selectable_options[current_option] = True
       else:
         result = False
-        if current_option in self.dep_exprs.keys():
+        if current_option in list(self.dep_exprs.keys()):
           try:
             tree = ast.parse(self.dep_exprs[current_option])
           except RuntimeError as e:
@@ -137,7 +137,7 @@ def get_selectable_one(self, current_option):
           visitor.visit(tree)
           result = result or visitor.result()
 
-        if current_option in self.rev_dep_exprs.keys():
+        if current_option in list(self.rev_dep_exprs.keys()):
           try:
             tree = ast.parse(self.rev_dep_exprs[current_option])
           except RuntimeError as e:

kmax/kextractcommon.py

@@ -15,7 +15,7 @@
 module_versions["4.12.8"] = kextractor_4_12_8
 module_versions["4.18"] = kextractor_4_18
 
-available_versions = "Available versions: %s" % (", ".join(module_versions.keys()))
+available_versions = "Available versions: %s" % (", ".join(list(module_versions.keys())))
 
 def pick_version(kernel_version: str):
   """Given a kernel_version string, returns the kextract version string to use."""

kmax/klocalizer.py

@@ -250,7 +250,7 @@ def __fetch_kmax_constraints(self, path):
         current_path = os.path.join(parent_path, elem)
         if i < len(elems) - 1:
           current_path = current_path + "/"
-        if current_path not in self.__kmax_formulas.keys():
+        if current_path not in list(self.__kmax_formulas.keys()):
           # run kmax on the parent to find the constraint
           paths_to_try = []
           # paths are relative to ksrc
@@ -587,7 +587,7 @@ def compile_constraints(self, arch: Arch):
         token_pattern = regex.compile("CONFIG_[A-Za-z0-9_]+")
         for kmax_constraint in self.__kmax_constraints:
           used_vars = z3.z3util.get_vars(kmax_constraint)
-          vars_not_in_arch = [ used_var for used_var in used_vars if str(used_var) not in kconfig_types.keys() and token_pattern.match(str(used_var)) ]
+          vars_not_in_arch = [ used_var for used_var in used_vars if str(used_var) not in list(kconfig_types.keys()) and token_pattern.match(str(used_var)) ]
           for used_var in vars_not_in_arch:
             constraints.append(z3.Not(used_var))
 

kmax/patch.py

@@ -1,4 +1,4 @@
-from common import SourceFileType, FileChangeType
+from .common import SourceFileType, FileChangeType
 import whatthepatch
 
 def is_maybe_kernel(filepath) -> bool:

kmax/superc.py

@@ -1,5 +1,5 @@
-from common import BasicLogger
-from vcommon import run, write_content_to_file, check_if_compiles
+from .common import BasicLogger
+from .vcommon import run, write_content_to_file, check_if_compiles
 from kmax.arch import Arch
 import os
 import pathlib

kmax/vcommon.py

@@ -7,6 +7,7 @@
 from shutil import which
 
 import logging
+from functools import reduce
 
 def write_content_to_file(filepath: str, content: str):
     with open(filepath, 'w') as f:
@@ -183,9 +184,9 @@ def are_units_compiled():
 
 def pause(s=None):
     try: #python2
-        raw_input("Press any key to continue ..." if s is None else s)
-    except NameError:
         input("Press any key to continue ..." if s is None else s)
+    except NameError:
+        eval(input("Press any key to continue ..." if s is None else s))
 
 
 def whoami():
@@ -197,7 +198,7 @@ def vcmd(cmd, inp=None, shell=True):
 
 def vload(filename,mode='rb'):
     try:
-        import cPickle as pickle
+        import pickle as pickle
     except ImportError:  #Python3
         import pickle
 
@@ -208,7 +209,7 @@ def vload(filename,mode='rb'):
 
 def vsave(filename,sobj,mode='wb'):
     try:
-        import cPickle as pickle
+        import pickle as pickle
     except ImportError:  #Python3
         import pickle