optionally persist metadata sends from metadata plugin (#57)

* optionally persist metadata sends from metadata plugin

src/signalfx_metadata.py

@@ -72,10 +72,26 @@
 HOST_TYPE_INSTANCE = "host-meta-data"
 TOP_TYPE_INSTANCE = "top-info"
 TYPE = "objects"
-NEXT_METADATA_SEND = 0
-NEXT_METADATA_SEND_INTERVAL = [random.randint(0, 60), 60,
-                               3600 + random.randint(0, 60),
-                               86400 + random.randint(0, 600)]
+
+
+def DEFAULT_NEXT_METADATA_SEND():
+    """returns the default next metadata send"""
+    return 0
+
+
+NEXT_METADATA_SEND = DEFAULT_NEXT_METADATA_SEND()
+NEXT_METADATA_SEND_KEY = "NEXT_METADATA_SEND"
+
+
+def DEFAULT_NEXT_METADATA_SEND_INTERVAL():
+    """returns the default next metadata send intervals"""
+    return [random.randint(0, 60), 60,
+            3600 + random.randint(0, 60),
+            86400 + random.randint(0, 600)]
+
+
+NEXT_METADATA_SEND_INTERVAL = DEFAULT_NEXT_METADATA_SEND_INTERVAL()
+NEXT_METADATA_SEND_INTERVAL_KEY = "NEXT_METADATA_SEND_INTERVAL"
 LAST = 0
 AWS = False
 AWS_SET = False
@@ -97,6 +113,10 @@
 PERCORECPUUTIL = False
 OVERALLCPUUTIL = True
 ETC_PATH = "{0}etc".format(os.sep)
+SAVED_HOST = None
+SAVED_HOST_KEY = "SAVED_HOST"
+PERSISTENCE_PATH = None
+PERSISTENCE_FILE = "sfx_metadata_state.json"
 
 
 class mdict(dict):
@@ -848,9 +868,14 @@ def plugin_config(conf):
             psutil.PROCFS_PATH = kv.values[0]
             debug("Setting proc path to %s for psutil" % psutil.PROCFS_PATH)
         elif kv.key == 'EtcPath':
+            global ETC_PATH
             ETC_PATH = kv.values[0].rstrip(os.pathsep).rstrip(os.sep)
             debug("Setting etc path to %s for os release detection"
                   % ETC_PATH)
+        elif kv.key == 'PersistencePath':
+            global PERSISTENCE_PATH
+            PERSISTENCE_PATH = kv.values[0]
+            load_persistent_data()
 
     if not POST_URLS:
         POST_URLS = [DEFAULT_POST_URL]
@@ -883,6 +908,44 @@ def plugin_config(conf):
     get_aws_info()
 
 
+def load_persistent_data():
+    """Load persistent data from a specified path"""
+    try:
+        with open(os.path.join(PERSISTENCE_PATH, PERSISTENCE_FILE), 'r') as js:
+            persist = json.load(js)
+            debug("Loaded the following persistent data %s" % persist)
+            if SAVED_HOST_KEY in persist:
+                global SAVED_HOST
+                SAVED_HOST = persist[SAVED_HOST_KEY]
+            if NEXT_METADATA_SEND_KEY in persist:
+                global NEXT_METADATA_SEND
+                NEXT_METADATA_SEND = persist[NEXT_METADATA_SEND_KEY]
+            if NEXT_METADATA_SEND_INTERVAL_KEY in persist:
+                global NEXT_METADATA_SEND_INTERVAL
+                NEXT_METADATA_SEND_INTERVAL = \
+                    persist[NEXT_METADATA_SEND_INTERVAL_KEY]
+
+    except Exception as e:
+        debug("Unable to load persistence data %s" % e)
+
+
+def save_persistent_data():
+    """Persist data about the metadata plugin to a file"""
+    if PERSISTENCE_PATH:
+        try:
+            with open(os.path.join(PERSISTENCE_PATH,
+                                   PERSISTENCE_FILE), 'w') as f:
+                persist = {
+                    SAVED_HOST_KEY: HOST,
+                    NEXT_METADATA_SEND_KEY: NEXT_METADATA_SEND,
+                    NEXT_METADATA_SEND_INTERVAL_KEY:
+                        NEXT_METADATA_SEND_INTERVAL
+                }
+                json.dump(persist, f)
+        except Exception as e:
+            debug("Unable to save persistent data: %s" % e)
+
+
 def compact(thing):
     return json.dumps(thing, separators=(',', ':'))
 
@@ -923,9 +986,9 @@ def send():
     if NEXT_METADATA_SEND == 0:
         dither = NEXT_METADATA_SEND_INTERVAL.pop(0)
         NEXT_METADATA_SEND = time.time() + dither
-        log(
-            "adding small dither of %s seconds before sending notifications"
+        log("adding small dither of %s seconds before sending notifications"
             % dither)
+        save_persistent_data()
     if NEXT_METADATA_SEND < time.time():
         send_notifications()
         if len(NEXT_METADATA_SEND_INTERVAL) > 1:
@@ -936,11 +999,21 @@ def send():
 
         log("till next metadata %s seconds"
             % str(NEXT_METADATA_SEND - time.time()))
+        save_persistent_data()
 
     global LAST
     LAST = time.time()
 
 
+def reset_metadata_send():
+    """Reset the next metadata send and the metadata send intervals"""
+    debug("Resetting the next metadata send time and metadata send intervals.")
+    global NEXT_METADATA_SEND
+    NEXT_METADATA_SEND = DEFAULT_NEXT_METADATA_SEND()
+    global NEXT_METADATA_SEND_INTERVAL
+    NEXT_METADATA_SEND_INTERVAL = DEFAULT_NEXT_METADATA_SEND_INTERVAL()
+
+
 def all_interfaces():
 
     """
@@ -1522,8 +1595,16 @@ def steal_host_from_notifications(notif):
     # "host" from collectd.conf steal it from notifications we've put on the
     # bus so we can use it for our own
     global HOST
+    global SAVED_HOST
     if not HOST and notif.host:
         HOST = notif.host
+        # if host is identified and it's different from the saved_host,
+        # reset the metadata send interval and next metadata send time
+        if SAVED_HOST and SAVED_HOST != HOST:
+            debug(("The saved hostname '{0}' does not match the current "
+                   "hostname '{1}'.").format(SAVED_HOST, HOST))
+            reset_metadata_send()
+            SAVED_HOST = HOST
         DOGSTATSD_INSTANCE.set_host(notif.host)
         log("found host " + HOST)