check_openshift_es_stats

#!/usr/bin/python3

import argparse
import nagiosplugin
import requests
import operator

from vshn_npo import constants
from vshn_npo import utils


def _bytes_to_mb(value):
  return int(round(value / (1024.0 * 1024), 0))


class _SingleCapacity:
  """Logic for single capacity metric.

  """
  def __init__(self, size, used, uom="MB"):
    self._size = size
    self._used = used
    self._uom = uom

  @property
  def size(self):
    return self._size

  @property
  def available(self):
    return max(0, self._size - self._used)

  @property
  def available_percent(self):
    """Calculate percentage of available capacity.

    """
    return min(100, 100.0 - self.used_percent)

  @property
  def used(self):
    return self._used

  @property
  def used_percent(self):
    """Calculate percentage of used capacity.

    """
    size = self.size

    if size > 0:
      return int(100.0 * self.used / size)

    return 0

  def stats(self, name_fn, context_fn):
    """Generate metrics.

    """
    for value_name, value in [
      ("used", self.used),
      ("available", self.available),
    ]:
      name = name_fn(value_name)

      yield nagiosplugin.Metric(name, value, uom=self._uom, min=0, max=self.size,
                                context=context_fn(value_name))

    for value_name, value in [
      ("used-percent", self.used_percent),
      ("available-percent", self.available_percent),
    ]:
      name = name_fn(value_name)

      yield nagiosplugin.Metric(name, value, uom="%", min=0, max=100,
                                context=context_fn(value_name))


class _NodeStats:
  """Per-node statistics computation.

  """
  def __init__(self, data, name_suffix):
    self._data = data
    self._name_suffix = name_suffix

    # TODO: Monitor per-path space, not only globally
    allpaths = data["fs"]["total"]
    fs_size = _bytes_to_mb(allpaths["total_in_bytes"])
    fs_available = _bytes_to_mb(allpaths["available_in_bytes"])

    memdata = self._mem_data
    jvm_heap_size = _bytes_to_mb(memdata["heap_max_in_bytes"])
    jvm_heap_used = _bytes_to_mb(memdata["heap_used_in_bytes"])

    self.fs = _SingleCapacity(fs_size, fs_size - fs_available)
    self.jvm_heap = _SingleCapacity(jvm_heap_size, jvm_heap_used)

  @property
  def _process_cpu_percent(self):
    return self._data["process"]["cpu"]["percent"]

  @property
  def _mem_data(self):
    return self._data["jvm"]["mem"]

  def _gc_stats(self):
    gc = self._data["jvm"]["gc"]

    uptime_in_millis = float(self._data["jvm"]["uptime_in_millis"])

    for gc_name, gc_data in gc["collectors"].items():
      collection_count = int(gc_data["collection_count"])
      collection_time_in_millis = int(gc_data["collection_time_in_millis"])
      collection_time_avg = \
          round(float(collection_time_in_millis) / collection_count / 1000.0, 5)
      collection_time_percent = \
          round(100.0 * collection_time_in_millis / uptime_in_millis, 5)

      name_fn = lambda middle: \
          ("jvm-gc-collector-{}-{}-{}".
           format(gc_name, middle, self._name_suffix))

      yield nagiosplugin.Metric(name_fn("collection-count"),
                                collection_count, min=0,
                                context="default")

      yield nagiosplugin.Metric(name_fn("collection-time"),
                                round(collection_time_in_millis / 1000.0),
                                uom="s", min=0,
                                context="default")

      yield nagiosplugin.Metric(name_fn("collection-time-avg"),
                                collection_time_avg, uom="s", min=0,
                                context="jvm-gc-collection-time-avg")

      yield nagiosplugin.Metric(name_fn("collection-time-percent"),
                                collection_time_percent, uom="%", min=0, max=100,
                                context="jvm-gc-collection-time-percent")

  def stats(self):
    """Generate metrics.

    """
    suffix = self._name_suffix

    non_heap_used_in_bytes = _bytes_to_mb(self._mem_data["non_heap_used_in_bytes"])

    yield nagiosplugin.Metric("process-cpu-percent-{}".format(suffix),
                              self._process_cpu_percent, uom="%", min=0,
                              context="process-cpu-percent")

    yield nagiosplugin.Metric("jvm-non-heap-used-{}".format(suffix),
                              non_heap_used_in_bytes, uom="MB", min=0,
                              context="default")

    yield from self.fs.stats(lambda value_name: "fs-{}-{}".format(value_name, suffix),
                             lambda value_name: "fs-{}".format(value_name))

    yield from self.jvm_heap.stats(lambda value_name: "jvm-heap-{}-{}".format(value_name, suffix),
                                   lambda value_name: "jvm-heap-{}".format(value_name))

    yield from self._gc_stats()


def capacity_stats(name, items, uom="MB"):
  """Calculate metrics over any number of capacity metrics.

  """
  total_size = 0
  total_used = 0
  total_available = 0

  for i in items:
    total_size += i.size
    total_used += i.used
    total_available += i.available

  for name_pattern, value in [
    ("total-{}-used", total_used),
    ("total-{}-available", max(0, total_size - total_used)),
  ]:
    metric_name = name_pattern.format(name)

    yield nagiosplugin.Metric(metric_name, value, uom=uom,
                              min=0, max=total_size,
                              context=metric_name)

  if total_size > 0:
    for value_name, value in [
      ("used", total_used),
      ("available", total_available),
    ]:
      metric_name = "total-{}-{}-percent".format(name, value_name)
      total_percent = int(100.0 * value / total_size)

      yield nagiosplugin.Metric(metric_name, total_percent, uom="%",
                                min=0, max=100, context=metric_name)

  if items:
    for selector_name, selector_fn in [
      ("min", min),
      ("max", max),
    ]:
      name_fn = lambda middle: "node-{}-{}-{}".format(name, middle, selector_name)
      value_fn = lambda key_fn: selector_fn(map(key_fn, items))

      for name_middle, key_fn in [
        ("used", operator.attrgetter("used")),
        ("available", operator.attrgetter("available")),
      ]:
        yield nagiosplugin.Metric(name_fn(name_middle), value_fn(key_fn),
                                  uom=uom, min=0,
                                  context="default")

      for name_middle, key_fn in [
        ("used-percent", operator.attrgetter("used_percent")),
        ("available-percent", operator.attrgetter("available_percent")),
      ]:
        yield nagiosplugin.Metric(name_fn(name_middle), value_fn(key_fn),
                                  uom="%", min=0, max=100,
                                  context="default")


class StatsQuery(nagiosplugin.Resource):
  def __init__(self, endpoint, token, strip_hostname_prefix):
    self._endpoint = endpoint
    self._token = token
    self._strip_hostname_prefix = strip_hostname_prefix

  def probe(self):
    url = "{}/_nodes/stats/fs,jvm,process".format(self._endpoint)
    headers = {
        "Authorization": "Bearer {}".format(self._token),
        }

    response = requests.get(url, headers=headers, allow_redirects=False)
    utils.raise_for_elasticsearch_response(response)

    data = response.json()

    nodes = []

    # Data structure documentation:
    # https://www.elastic.co/guide/en/elasticsearch/reference/2.4/cluster-nodes-stats.html#fs-info
    for node_data in data["nodes"].values():
      # In a cluster outside OpenShift there could be more than one
      # Elasticsearch instance per host, but in OpenShift every container has
      # its own name.
      suffix = node_data["name"]

      for i in self._strip_hostname_prefix:
        if suffix.startswith(i):
          suffix = suffix[len(i):]
          break

      node = _NodeStats(node_data, suffix)
      nodes.append(node)

      yield from node.stats()

    for capacity_name, attr_name in [
      ("fs", "fs"),
      ("jvm-heap", "jvm_heap"),
    ]:
      yield from \
          capacity_stats(capacity_name, [getattr(n, attr_name) for n in nodes])


class _CapacityArgs:
  """Utility to generate and use CLI arguments for capacity metrics.

  """
  _METRICS = frozenset({
    "used",
    "used-percent",
    "available",
    "available-percent",
  })
  _METRIC_STATUS = [
    (metric, status_name)
    for metric in _METRICS
    for status_name in ["warn", "critical"]
  ]

  def __init__(self, name):
    self._name = name

  def _make_arg_dest(self, scope_name, metric, status_name):
    return ("{}_{}_{}_{}".
            format(self._name, scope_name, metric, status_name).
            replace("-", "_"))

  def add_arguments(self, global_args, node_args):
    """Add arguments to given argument groups.

    """
    for metric, status_name in self._METRIC_STATUS:
      arg_name = "--total-{}-{}-{}".format(self._name, metric, status_name)
      arg_dest = self._make_arg_dest("total", metric, status_name)
      global_args.add_argument(arg_name, metavar="RANGE", dest=arg_dest)

      arg_name = "--{}-{}-{}".format(self._name, metric, status_name)
      arg_dest = self._make_arg_dest("node", metric, status_name)
      node_args.add_argument(arg_name, metavar="RANGE", dest=arg_dest)

  def make_contexts(self, args):
    for metric in self._METRICS:
      for ctx_name_prefix, scope_name in [
        ("total-", "total"),
        ("", "node"),
      ]:
        ctx_name = "{}{}-{}".format(ctx_name_prefix, self._name, metric)

        warn_dest = self._make_arg_dest(scope_name, metric, "warn")
        crit_dest = self._make_arg_dest(scope_name, metric, "critical")

        yield nagiosplugin.ScalarContext(ctx_name,
                                         warning=getattr(args, warn_dest),
                                         critical=getattr(args, crit_dest))


@nagiosplugin.guarded
def main():
  parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  utils.add_verbose_argument(parser)
  utils.add_token_arguments(parser)
  parser.add_argument("--strip-hostname-prefix", metavar="PREFIX",
                      action="append",
                      help="Strip given prefix from node names")

  glargs = parser.add_argument_group("Global")
  nodeargs = parser.add_argument_group("Per node")

  for name, desc in [
    ("cpu-usage", "CPU usage"),
    ("jvm-gc-collection-time-avg",
     "Average JVM GC collection duration"),
    ("jvm-gc-collection-time-percent",
     "Percentage of JVM GC collection time relative to uptime"),
  ]:
    nodeargs.add_argument("--{}-warn".format(name), metavar="RANGE")
    nodeargs.add_argument("--{}-critical".format(name), metavar="RANGE")

  capacity_args = [
    _CapacityArgs("fs"),
    _CapacityArgs("jvm-heap"),
  ]

  for i in capacity_args:
    i.add_arguments(glargs, nodeargs)

  parser.add_argument("--endpoint", required=True, metavar="URL",
                      help="API endpoint")

  args = parser.parse_args()

  # Workaround for https://bugs.python.org/issue16399
  if not args.strip_hostname_prefix:
    args.strip_hostname_prefix = [
      "logging-es-data-master-",
      "logging-es-",
    ]

  utils.setup_basic_logging(args.verbose)

  token = utils.extract_token_argument(args)

  checks = [
      StatsQuery(args.endpoint, token, args.strip_hostname_prefix),
      nagiosplugin.ScalarContext("process-cpu-percent",
                                 warning=args.cpu_usage_warn,
                                 critical=args.cpu_usage_critical),
      nagiosplugin.ScalarContext("jvm-gc-collection-time-avg",
                                 warning=args.jvm_gc_collection_time_avg_warn,
                                 critical=args.jvm_gc_collection_time_avg_critical),
      nagiosplugin.ScalarContext("jvm-gc-collection-time-percent",
                                 warning=args.jvm_gc_collection_time_percent_warn,
                                 critical=args.jvm_gc_collection_time_percent_critical),
      ]

  for i in capacity_args:
    checks.extend(i.make_contexts(args))

  nagiosplugin.Check(*checks).main(verbose=args.verbose, timeout=None)


if __name__ == "__main__":
  main()

# vim: set sw=2 sts=2 et :