From cb3923db1748db8b42655d3d6a9cfcfc855ea046 Mon Sep 17 00:00:00 2001 From: Charlie Smith Date: Wed, 21 Jun 2017 15:52:32 -0400 Subject: [PATCH] optionally persist metadata sends from metadata plugin (#57) * optionally persist metadata sends from metadata plugin --- src/signalfx_metadata.py | 93 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 6 deletions(-) diff --git a/src/signalfx_metadata.py b/src/signalfx_metadata.py index 0f0c61c..97b64a2 100755 --- a/src/signalfx_metadata.py +++ b/src/signalfx_metadata.py @@ -72,10 +72,26 @@ HOST_TYPE_INSTANCE = "host-meta-data" TOP_TYPE_INSTANCE = "top-info" TYPE = "objects" -NEXT_METADATA_SEND = 0 -NEXT_METADATA_SEND_INTERVAL = [random.randint(0, 60), 60, - 3600 + random.randint(0, 60), - 86400 + random.randint(0, 600)] + + +def DEFAULT_NEXT_METADATA_SEND(): + """returns the default next metadata send""" + return 0 + + +NEXT_METADATA_SEND = DEFAULT_NEXT_METADATA_SEND() +NEXT_METADATA_SEND_KEY = "NEXT_METADATA_SEND" + + +def DEFAULT_NEXT_METADATA_SEND_INTERVAL(): + """returns the default next metadata send intervals""" + return [random.randint(0, 60), 60, + 3600 + random.randint(0, 60), + 86400 + random.randint(0, 600)] + + +NEXT_METADATA_SEND_INTERVAL = DEFAULT_NEXT_METADATA_SEND_INTERVAL() +NEXT_METADATA_SEND_INTERVAL_KEY = "NEXT_METADATA_SEND_INTERVAL" LAST = 0 AWS = False AWS_SET = False @@ -97,6 +113,10 @@ PERCORECPUUTIL = False OVERALLCPUUTIL = True ETC_PATH = "{0}etc".format(os.sep) +SAVED_HOST = None +SAVED_HOST_KEY = "SAVED_HOST" +PERSISTENCE_PATH = None +PERSISTENCE_FILE = "sfx_metadata_state.json" class mdict(dict): @@ -848,9 +868,14 @@ def plugin_config(conf): psutil.PROCFS_PATH = kv.values[0] debug("Setting proc path to %s for psutil" % psutil.PROCFS_PATH) elif kv.key == 'EtcPath': + global ETC_PATH ETC_PATH = kv.values[0].rstrip(os.pathsep).rstrip(os.sep) debug("Setting etc path to %s for os release detection" % ETC_PATH) + elif kv.key == 'PersistencePath': + global PERSISTENCE_PATH + PERSISTENCE_PATH = kv.values[0] + load_persistent_data() if not POST_URLS: POST_URLS = [DEFAULT_POST_URL] @@ -883,6 +908,44 @@ def plugin_config(conf): get_aws_info() +def load_persistent_data(): + """Load persistent data from a specified path""" + try: + with open(os.path.join(PERSISTENCE_PATH, PERSISTENCE_FILE), 'r') as js: + persist = json.load(js) + debug("Loaded the following persistent data %s" % persist) + if SAVED_HOST_KEY in persist: + global SAVED_HOST + SAVED_HOST = persist[SAVED_HOST_KEY] + if NEXT_METADATA_SEND_KEY in persist: + global NEXT_METADATA_SEND + NEXT_METADATA_SEND = persist[NEXT_METADATA_SEND_KEY] + if NEXT_METADATA_SEND_INTERVAL_KEY in persist: + global NEXT_METADATA_SEND_INTERVAL + NEXT_METADATA_SEND_INTERVAL = \ + persist[NEXT_METADATA_SEND_INTERVAL_KEY] + + except Exception as e: + debug("Unable to load persistence data %s" % e) + + +def save_persistent_data(): + """Persist data about the metadata plugin to a file""" + if PERSISTENCE_PATH: + try: + with open(os.path.join(PERSISTENCE_PATH, + PERSISTENCE_FILE), 'w') as f: + persist = { + SAVED_HOST_KEY: HOST, + NEXT_METADATA_SEND_KEY: NEXT_METADATA_SEND, + NEXT_METADATA_SEND_INTERVAL_KEY: + NEXT_METADATA_SEND_INTERVAL + } + json.dump(persist, f) + except Exception as e: + debug("Unable to save persistent data: %s" % e) + + def compact(thing): return json.dumps(thing, separators=(',', ':')) @@ -923,9 +986,9 @@ def send(): if NEXT_METADATA_SEND == 0: dither = NEXT_METADATA_SEND_INTERVAL.pop(0) NEXT_METADATA_SEND = time.time() + dither - log( - "adding small dither of %s seconds before sending notifications" + log("adding small dither of %s seconds before sending notifications" % dither) + save_persistent_data() if NEXT_METADATA_SEND < time.time(): send_notifications() if len(NEXT_METADATA_SEND_INTERVAL) > 1: @@ -936,11 +999,21 @@ def send(): log("till next metadata %s seconds" % str(NEXT_METADATA_SEND - time.time())) + save_persistent_data() global LAST LAST = time.time() +def reset_metadata_send(): + """Reset the next metadata send and the metadata send intervals""" + debug("Resetting the next metadata send time and metadata send intervals.") + global NEXT_METADATA_SEND + NEXT_METADATA_SEND = DEFAULT_NEXT_METADATA_SEND() + global NEXT_METADATA_SEND_INTERVAL + NEXT_METADATA_SEND_INTERVAL = DEFAULT_NEXT_METADATA_SEND_INTERVAL() + + def all_interfaces(): """ @@ -1522,8 +1595,16 @@ def steal_host_from_notifications(notif): # "host" from collectd.conf steal it from notifications we've put on the # bus so we can use it for our own global HOST + global SAVED_HOST if not HOST and notif.host: HOST = notif.host + # if host is identified and it's different from the saved_host, + # reset the metadata send interval and next metadata send time + if SAVED_HOST and SAVED_HOST != HOST: + debug(("The saved hostname '{0}' does not match the current " + "hostname '{1}'.").format(SAVED_HOST, HOST)) + reset_metadata_send() + SAVED_HOST = HOST DOGSTATSD_INSTANCE.set_host(notif.host) log("found host " + HOST)