Add zuul-change-dump.py.tmpl script to zuul-scheduler

Change-Id: I0ee783b7ed0f6a3eb96bff9cf4acc7d5b7432d98

controllers/static/zuul/zuul-change-dump.py.tmpl

@@ -0,0 +1,106 @@
+#!/bin/env python3
+# Copyright 2013 OpenStack Foundation
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+# Copyright 2016 Red Hat
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import requests
+import argparse
+import os
+import time
+
+
+dump_file = "/var/lib/zuul/zuul-queues-dump.sh"
+
+
+def get_tenants(args):
+    req = requests.get("%s/tenants" % args.url)
+    tenants = req.json()
+    status = {}
+    for tenant in tenants:
+        req = requests.get("%s/tenant/%s/status" % (args.url, tenant["name"]))
+        status[tenant["name"]] = req.json()
+    if os.path.isfile(args.dump_file):
+        os.rename(args.dump_file, "%s.orig" % args.dump_file)
+    return (tenants, status)
+
+
+def retry_get_tenants(args):
+    count, max_count = 0, 5
+    while True:
+        try:
+            return get_tenants(args)
+        except Exception as e:
+            if count > max_count:
+                raise e
+            count += 1
+            print("Request fails (%s), retrying %d/%d" %
+                  (str(e), count, max_count))
+            time.sleep(count * 2)
+
+
+def dump(args):
+    (tenants, status) = retry_get_tenants(args)
+    of = open(args.dump_file, "w")
+    of.write("#/bin/sh\nset -ex\n")
+    for tenant in tenants:
+        for pipeline in status[tenant["name"]].get('pipelines', []):
+            for queue in pipeline['change_queues']:
+                for head in queue['heads']:
+                    for change in head:
+                        if (not change['live'] or
+                                not change.get('id') or
+                                ',' not in change['id']):
+                            continue
+                        cid, cps = change['id'].split(',')
+                        cmd = (
+                            "zuul-client enqueue --tenant %s "
+                            "--pipeline %s --project %s --change %s,%s" % (
+                                tenant["name"],
+                                pipeline['name'],
+                                change['project_canonical'],
+                                cid, cps)
+                        )
+                        if ";" in cmd or "|" in cmd:
+                            raise RuntimeError("Forbidden char in [%s]" % cmd)
+                        print(cmd)
+                        of.write("%s\n" % cmd)
+    of.write(
+        "curl %s/info 2>&1 | grep 'capabilities' > /dev/null\n" % args.url)
+    of.write("echo SUCCESS: zuul queues restored\n")
+    of.close()
+    os.chmod(args.dump_file, 0o755)
+
+
+def load(args):
+    if not os.path.isfile(args.dump_file):
+        print("%s: no such file, please dump first" % args.dump_file)
+    if os.stat(args.dump_file).st_mtime + 172800 < time.time():
+        if not args.force:
+            raise RuntimeError("%s is too old, use --force to use it" %
+                               args.dump_file)
+    os.system(dump_file)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--force', action="store_const", const=True)
+    parser.add_argument('--url', default="{{ .ZuulWebURL }}/api")
+    parser.add_argument('--dump_file', default=dump_file)
+    parser.add_argument('action', choices=("dump", "load"))
+    args = parser.parse_args()
+    if args.action == "dump":
+        dump(args)
+    else:
+        load(args)

controllers/zuul.go

@@ -58,6 +58,9 @@ var (
 	//go:embed static/zuul/logging.yaml.tmpl
 	zuulLoggingConfig string
 
+	//go:embed static/zuul/zuul-change-dump.py.tmpl
+	zuulChangeDump string
+
 	// Common config sections for all Zuul components
 	commonIniConfigSections = []string{"zookeeper", "keystore", "database"}
 
@@ -206,6 +209,12 @@ func (r *SFController) mkZuulContainer(service string, corporateCMExists bool) a
 				MountPath: "/usr/local/bin/fetch-config-repo.sh",
 				ReadOnly:  true,
 			},
+			apiv1.VolumeMount{
+				Name:      "tooling-vol",
+				SubPath:   "zuul-change-dump.py",
+				MountPath: "/usr/local/bin/zuul-change-dump.py",
+				ReadOnly:  true,
+			},
 		)
 		envs = append(envs, r.getTenantsEnvs()...)
 	}
@@ -514,6 +523,9 @@ func (r *SFController) EnsureZuulScheduler(cfg *ini.File) bool {
 	schedulerToolingData["init-container.sh"] = zuulSchedulerInitContainerScript
 	schedulerToolingData["generate-zuul-tenant-yaml.sh"] = zuulGenerateTenantConfig
 	schedulerToolingData["fetch-config-repo.sh"] = fetchConfigRepoScript
+	schedulerToolingData["zuul-change-dump.py"], _ = utils.ParseString(zuulChangeDump, struct {
+		ZuulWebURL string
+	}{ZuulWebURL: "https://" + r.cr.Spec.FQDN + "/zuul"})
 
 	r.EnsureConfigMap("zuul-scheduler-tooling", schedulerToolingData)