Make Labcontroller compatible with Python 2 and Python 3

LabController/Makefile

@@ -4,17 +4,25 @@
 # the Free Software Foundation; either version 2 of the License, or
 # (at your option) any later version.
 
+BKR_PY3 ?= 0
+COMMAND := python2
+
+ifeq ($(BKR_PY3),1)
+	COMMAND := python3
+endif
+
+
 .PHONY: build
 build:
-	python2 setup.py build
+	$(COMMAND) setup.py build
 
 .PHONY: install
 install: build
-	python2 setup.py install -O1 --skip-build --root $(DESTDIR)
+	$(COMMAND) setup.py install -O1 --skip-build --root $(DESTDIR)
 
 .PHONY: clean
 clean:
-	python2 setup.py clean
+	$(COMMAND) setup.py clean
 	rm -rf build
 
 .PHONY: check

LabController/setup.py

@@ -1,16 +1,21 @@
 from setuptools import setup, find_packages
-import commands
 from glob import glob
 
+try:
+    from subprocess import getstatusoutput
+except ImportError:
+    from commands import getstatusoutput
+
+
 def systemd_unit_dir():
-    status, output = commands.getstatusoutput('pkg-config --variable systemdsystemunitdir systemd')
+    status, output = getstatusoutput('pkg-config --variable systemdsystemunitdir systemd')
     if status or not output:
         return None # systemd not found
     return output.strip()
 
 def systemd_tmpfiles_dir():
     # There doesn't seem to be a specific pkg-config variable for this
-    status, output = commands.getstatusoutput('pkg-config --variable prefix systemd')
+    status, output = getstatusoutput('pkg-config --variable prefix systemd')
     if status or not output:
         return None # systemd not found
     return output.strip() + '/lib/tmpfiles.d'

LabController/src/bkr/labcontroller/expire_distros.py

@@ -5,16 +5,16 @@
 # (at your option) any later version.
 
 import sys, os
-import xmlrpclib
-import urllib2
-import urlparse
+
+from six.moves import urllib
+from six.moves import xmlrpc_client
 
 
 def check_http(url):
     try:
-        urllib2.urlopen(url, timeout=120)
+        urllib.request.urlopen(url, timeout=120)
         return True
-    except urllib2.HTTPError as e:
+    except urllib.error.HTTPError as e:
         if e.code in (404, 410):
             return False
         else:
@@ -23,9 +23,9 @@ def check_http(url):
 
 def check_ftp(url):
     try:
-        urllib2.urlopen(url, timeout=120)
+        urllib.request.urlopen(url, timeout=120)
         return True
-    except urllib2.URLError as e:
+    except urllib.error.URLError as e:
         if '550' in e.reason:
             return False
         else:
@@ -41,7 +41,7 @@ def check_nfs(tree):
     Make sure the tree is accessible, check that the server is up first.
     """
 
-    _, nfs_server, nfs_path, _, _, _ = urlparse.urlparse(tree)
+    _, nfs_server, nfs_path, _, _, _ = urllib.parse.urlparse(tree)
     # Beaker uses a non-standard syntax for NFS URLs, inherited from Cobbler:
     # nfs://server:/path
     # so we need to strip a trailing colon from the hostname portion.
@@ -64,7 +64,7 @@ def check_url(url):
     Returns True if the given URL exists.
     """
 
-    scheme = urlparse.urlparse(url).scheme
+    scheme = urllib.parse.urlparse(url).scheme
     if scheme == 'nfs' or scheme.startswith('nfs+'):
         return check_nfs(url)
     elif scheme == 'http' or scheme == 'https':
@@ -79,7 +79,7 @@ def check_all_trees(ignore_errors=False,
                     dry_run=False,
                     lab_controller='http://localhost:8000',
                     remove_all=False):
-    proxy = xmlrpclib.ServerProxy(lab_controller, allow_none=True)
+    proxy = xmlrpc_client.ServerProxy(lab_controller, allow_none=True)
     rdistro_trees = []
     distro_trees = proxy.get_distro_trees()
     if not remove_all:
@@ -93,7 +93,7 @@ def check_all_trees(ignore_errors=False,
                         print('{0} is missing [Distro Tree ID {1}]'.format(
                             url,
                             distro_tree['distro_tree_id']))
-                except (urllib2.URLError, urllib2.HTTPError, NFSServerInaccessible) as e:
+                except (urllib.error.URLError, urllib.error.HTTPError, NFSServerInaccessible) as e:
                     if ignore_errors:
                         # suppress exception, assume the tree still exists
                         accessible = True

LabController/src/bkr/labcontroller/log_storage.py

@@ -1,48 +1,49 @@
-
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 2 of the License, or
 # (at your option) any later version.
 
 # We are talking about job logs here, not logs produced by the daemons.
 
-import os, os.path
 import errno
+import os
+import os.path
+
 from bkr.common.helpers import makedirs_ignore
 
-class LogFile(object):
 
+class LogFile(object):
     def __init__(self, path, register_func, create=True):
-        self.path = path #: absolute path where the log will be stored
-        self.register_func = register_func #: called only if the file was created
-        self.create = create #: create the file if it doesn't exist
+        self.path = path  #: absolute path where the log will be stored
+        self.register_func = register_func  #: called only if the file was created
+        self.create = create  #: create the file if it doesn't exist
 
     def __repr__(self):
-        return '%s(%r)' % (self.__class__.__name__, self.path)
+        return "%s(%r)" % (self.__class__.__name__, self.path)
 
     def open_ro(self):
         """
         If you just want to read the log, call this instead of entering the context manager.
         """
-        return open(self.path, 'r')
+        return open(self.path, "r")
 
     def __enter__(self):
-        makedirs_ignore(os.path.dirname(self.path), 0755)
+        makedirs_ignore(os.path.dirname(self.path), 0o755)
         created = False
         if self.create:
             try:
-                # stdio does not have any mode string which corresponds to this 
+                # stdio does not have any mode string which corresponds to this
                 # combination of flags, so we have to use raw os.open :-(
-                fd = os.open(self.path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
+                fd = os.open(self.path, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o644)
                 created = True
-            except (OSError, IOError), e:
+            except (OSError, IOError) as e:
                 if e.errno != errno.EEXIST:
                     raise
                 fd = os.open(self.path, os.O_RDWR)
         else:
             fd = os.open(self.path, os.O_RDWR)
         try:
-            self.f = os.fdopen(fd, 'r+')
+            self.f = os.fdopen(fd, "r+")
         except Exception:
             os.close(fd)
             raise
@@ -65,68 +66,99 @@ def truncate(self, size):
 
     def update_chunk(self, data, offset):
         if offset < 0:
-            raise ValueError('Offset cannot be negative')
+            raise ValueError("Offset cannot be negative")
         self.f.seek(offset, os.SEEK_SET)
-        # XXX the original uploadFile acquires an exclusive lock while writing, 
+        # XXX the original uploadFile acquires an exclusive lock while writing,
         # for no reason that I can discern
         self.f.write(data)
         self.f.flush()
 
+
 class LogStorage(object):
 
     """
     Handles storage of job logs on the local filesystem.
 
-    The old XML-RPC API doesn't include the recipe ID with the task or result 
-    upload calls. So for now, everything is stored flat. Eventually it would be 
+    The old XML-RPC API doesn't include the recipe ID with the task or result
+    upload calls. So for now, everything is stored flat. Eventually it would be
     nice to arrange things hierarchically with everything under recipe instead.
     """
 
     def __init__(self, base_dir, base_url, hub):
         self.base_dir = base_dir
-        if not base_url.endswith('/'):
-            base_url += '/' # really it is always a directory
+        if not base_url.endswith("/"):
+            base_url += "/"  # really it is always a directory
         self.base_url = base_url
         self.hub = hub
 
     def recipe(self, recipe_id, path, create=True):
-        path = os.path.normpath(path.lstrip('/'))
-        if path.startswith('../'):
-            raise ValueError('Upload path not allowed: %s' % path)
-        recipe_base_dir = os.path.join(self.base_dir, 'recipes',
-                (recipe_id[:-3] or '0') + '+', recipe_id, '')
-        recipe_base_url = '%srecipes/%s+/%s/' % (self.base_url,
-                recipe_id[:-3] or '0', recipe_id)
-        return LogFile(os.path.join(recipe_base_dir, path),
-                lambda: self.hub.recipes.register_file(recipe_base_url,
-                    recipe_id, os.path.dirname(path), os.path.basename(path),
-                    recipe_base_dir),
-                create=create)
+        path = os.path.normpath(path.lstrip("/"))
+        if path.startswith("../"):
+            raise ValueError("Upload path not allowed: %s" % path)
+        recipe_base_dir = os.path.join(
+            self.base_dir, "recipes", (recipe_id[:-3] or "0") + "+", recipe_id, ""
+        )
+        recipe_base_url = "%srecipes/%s+/%s/" % (
+            self.base_url,
+            recipe_id[:-3] or "0",
+            recipe_id,
+        )
+        return LogFile(
+            os.path.join(recipe_base_dir, path),
+            lambda: self.hub.recipes.register_file(
+                recipe_base_url,
+                recipe_id,
+                os.path.dirname(path),
+                os.path.basename(path),
+                recipe_base_dir,
+            ),
+            create=create,
+        )
 
     def task(self, task_id, path, create=True):
-        path = os.path.normpath(path.lstrip('/'))
-        if path.startswith('../'):
-            raise ValueError('Upload path not allowed: %s' % path)
-        task_base_dir = os.path.join(self.base_dir, 'tasks',
-                (task_id[:-3] or '0') + '+', task_id, '')
-        task_base_url = '%stasks/%s+/%s/' % (self.base_url,
-                task_id[:-3] or '0', task_id)
-        return LogFile(os.path.join(task_base_dir, path),
-                lambda: self.hub.recipes.tasks.register_file(task_base_url,
-                    task_id, os.path.dirname(path), os.path.basename(path),
-                    task_base_dir),
-                create=create)
+        path = os.path.normpath(path.lstrip("/"))
+        if path.startswith("../"):
+            raise ValueError("Upload path not allowed: %s" % path)
+        task_base_dir = os.path.join(
+            self.base_dir, "tasks", (task_id[:-3] or "0") + "+", task_id, ""
+        )
+        task_base_url = "%stasks/%s+/%s/" % (
+            self.base_url,
+            task_id[:-3] or "0",
+            task_id,
+        )
+        return LogFile(
+            os.path.join(task_base_dir, path),
+            lambda: self.hub.recipes.tasks.register_file(
+                task_base_url,
+                task_id,
+                os.path.dirname(path),
+                os.path.basename(path),
+                task_base_dir,
+            ),
+            create=create,
+        )
 
     def result(self, result_id, path, create=True):
-        path = os.path.normpath(path.lstrip('/'))
-        if path.startswith('../'):
-            raise ValueError('Upload path not allowed: %s' % path)
-        result_base_dir = os.path.join(self.base_dir, 'results',
-                (result_id[:-3] or '0') + '+', result_id, '')
-        result_base_url = '%sresults/%s+/%s/' % (self.base_url,
-                result_id[:-3] or '0', result_id)
-        return LogFile(os.path.join(result_base_dir, path),
-                lambda: self.hub.recipes.tasks.register_result_file(result_base_url,
-                    result_id, os.path.dirname(path), os.path.basename(path),
-                    result_base_dir),
-                create=create)
+        path = os.path.normpath(path.lstrip("/"))
+        if path.startswith("../"):
+            raise ValueError("Upload path not allowed: %s" % path)
+        result_base_dir = os.path.join(
+            self.base_dir, "results", (result_id[:-3] or "0") + "+", result_id, ""
+        )
+        result_base_url = "%sresults/%s+/%s/" % (
+            self.base_url,
+            result_id[:-3] or "0",
+            result_id,
+        )
+        return LogFile(
+            os.path.join(result_base_dir, path),
+            lambda: self.hub.recipes.tasks.register_result_file(
+                result_base_url,
+                result_id,
+                os.path.dirname(path),
+                os.path.basename(path),
+                result_base_dir,
+            ),
+            create=create,
+        )