feat(rest api): Implement prometheus/openmetrics exporter

code/components/jomjol_flowcontroll/ClassFlowControll.cpp

@@ -862,6 +862,15 @@ bool ClassFlowControll::StartMQTTService()
 #endif //ENABLE_MQTT
 
 
+/**
+ * @returns a vector of all current sequences
+ **/
+const std::vector<NumberPost*> &ClassFlowControll::getNumbers()
+{
+    return *flowpostprocessing->GetNumbers();
+}
+
+
 /* Return all available numbers names (number sequences)*/
 std::string ClassFlowControll::getNumbersName()
 {

code/components/jomjol_flowcontroll/ClassFlowControll.h

@@ -78,6 +78,7 @@ class ClassFlowControll : public ClassFlow
 	std::string TranslateAktstatus(std::string _input);
 	bool getStatusSetupModus() {return SetupModeActive;};
 
+	const std::vector<NumberPost*> &getNumbers();
 	std::string getNumbersName();
 	std::string getNumbersName(int _number);
 	int getNumbersSize();

code/components/jomjol_flowcontroll/MainFlowControl.cpp

@@ -1011,6 +1011,12 @@ void setTaskAutoFlowState(int _value)
 }
 
 
+int getTaskAutoFlowState()
+{
+    return taskAutoFlowState;
+}
+
+
 std::string getProcessStatus(void)
 {
     std::string process_status;

code/components/jomjol_flowcontroll/MainFlowControl.h

@@ -19,6 +19,7 @@ esp_err_t triggerFlowStartByMqtt(std::string _topic);
 void triggerFlowStartByGpio();
 
 void setTaskAutoFlowState(int _value);
+int getTaskAutoFlowState();
 
 std::string getProcessStatus();
 int getFlowCycleCounter();

code/components/jomjol_helper/Helper.cpp

@@ -645,6 +645,17 @@ std::string UrlDecode(const std::string& value)
 }
 
 
+// from https://stackoverflow.com/a/14678800
+void replaceAll(std::string& s, const std::string& toReplace, const std::string& replaceWith)
+{
+    size_t pos = 0;
+    while ((pos = s.find(toReplace, pos)) != std::string::npos) {
+         s.replace(pos, toReplace.length(), replaceWith);
+         pos += replaceWith.length();
+    }
+}
+
+
 bool replaceString(std::string& s, std::string const& toReplace, std::string const& replaceWith)
 {
     return replaceString(s, toReplace, replaceWith, true);

code/components/jomjol_helper/Helper.h

@@ -47,6 +47,7 @@ const char* get404(void);
 
 std::string UrlDecode(const std::string& value);
 
+void replaceAll(std::string& s, const std::string& toReplace, const std::string& replaceWith);
 bool replaceString(std::string& s, std::string const& toReplace, std::string const& replaceWith);
 bool replaceString(std::string& s, std::string const& toReplace, std::string const& replaceWith, bool logIt);
 bool isInString(std::string& s, std::string const& toFind);

code/components/openmetrics/CMakeLists.txt

@@ -0,0 +1,7 @@
+FILE(GLOB_RECURSE app_sources ${CMAKE_CURRENT_SOURCE_DIR}/*.*)
+
+idf_component_register(SRCS ${app_sources}
+                    INCLUDE_DIRS "."
+                    REQUIRES jomjol_helper)
+
+

code/components/openmetrics/openmetrics.cpp

@@ -0,0 +1,118 @@
+#include "openmetrics.h"
+#include "../../include/defines.h"
+
+#include <esp_log.h>
+
+#include "MainFlowControl.h"
+#include "system.h"
+#include "connect_wlan.h"
+
+
+static const char *TAG = "OPENMETRICS";
+
+
+/**
+ * create a singe metric from the given input
+ **/
+std::string createMetric(const std::string &metricName, const std::string &help, const std::string &type, const std::string &value)
+{
+    return "# HELP " + metricName + " " + help + "\n" +
+           "# TYPE " + metricName + " " + type + "\n" +
+           metricName + " " + value + "\n";
+}
+
+
+/**
+ * Generate the MetricFamily from all available sequences
+ * @returns the string containing the text wire format of the MetricFamily
+ **/
+std::string createSequenceMetrics(std::string prefix, const std::vector<NumberPost *> &sequences)
+{
+    std::string res;
+
+    for (const auto &sequence : sequences) {
+        std::string sequenceName = sequence->name;
+
+        // except newline, double quote, and backslash (https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#abnf)
+        // to keep it simple, these characters are just removed from the label
+        replaceAll(sequenceName, "\\", "");
+        replaceAll(sequenceName, "\"", "");
+        replaceAll(sequenceName, "\n", "");
+        res += prefix + "_flow_value{sequence=\"" + sequenceName + "\"} " + sequence->sActualValue + "\n";
+    }
+
+    // prepend metadata if a valid metric was created
+    if (res.length() > 0) {
+        res = "# HELP " + prefix + "_flow_value current value of meter readout\n# TYPE " + prefix + "_flow_value gauge\n" + res;
+    }
+
+    return res;
+}
+
+
+/**
+ * Generates a http response containing the OpenMetrics (https://openmetrics.io/) text wire format
+ * according to https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#text-format.
+ *
+ * A MetricFamily with a Metric for each Sequence is provided. If no valid value is available, the metric is not provided.
+ * MetricPoints are provided without a timestamp. Additional metrics with some device information is also provided.
+ *
+ * The metric name prefix is 'ai_on_the_edge_device_'.
+ *
+ * example configuration for Prometheus (`prometheus.yml`):
+ *
+ *    - job_name: watermeter
+ *      static_configs:
+ *        - targets: ['watermeter.fritz.box']
+ *
+*/
+esp_err_t handler_openmetrics(httpd_req_t *req)
+{
+    if (getTaskAutoFlowState() <= FLOW_TASK_STATE_INIT) {
+        httpd_resp_set_hdr(req, "Access-Control-Allow-Origin", "*");
+        httpd_resp_send_err(req, HTTPD_403_FORBIDDEN, "E95: Request rejected, flow not initialized");
+        return ESP_FAIL;
+    }
+
+    httpd_resp_set_hdr(req, "Access-Control-Allow-Origin", "*");
+    httpd_resp_set_type(req, "text/plain"); // application/openmetrics-text is not yet supported by prometheus so we use text/plain for now
+
+    const std::string metricNamePrefix = "ai_on_the_edge_device";
+
+    // get current measurement values
+    std::string response = createSequenceMetrics(metricNamePrefix, flowctrl.getNumbers());
+
+    // CPU temperature
+    response += createMetric(metricNamePrefix + "_cpu_temperature_celsius", "current cpu temperature in celsius", "gauge", std::to_string((int)getSOCTemperature()));
+
+    // WiFi signal strength
+    response += createMetric(metricNamePrefix + "_rssi_dbm", "current WiFi signal strength in dBm", "gauge", std::to_string(get_WIFI_RSSI()));
+
+    // memory info
+    response += createMetric(metricNamePrefix + "_memory_heap_free_bytes", "available heap memory", "gauge", std::to_string(getESPHeapSizeTotalFree()));
+
+    // device uptime
+    response += createMetric(metricNamePrefix + "_uptime_seconds", "device uptime in seconds", "gauge", std::to_string((long)getUptime()));
+
+    // data aquisition round
+    response += createMetric(metricNamePrefix + "_cycles_total", "data aquisition cycles since device startup", "counter", std::to_string(getFlowCycleCounter()));
+
+    // the response always contains at least the metadata (HELP, TYPE) for the MetricFamily so no length check is needed
+    httpd_resp_send(req, response.c_str(), response.length());
+
+    return ESP_OK;
+}
+
+
+void register_openmetrics_uri(httpd_handle_t server)
+{
+    ESP_LOGI(TAG, "Registering URI handlers");
+
+    httpd_uri_t camuri = { };
+    camuri.method    = HTTP_GET;
+
+    camuri.uri = "/metrics";
+    camuri.handler = handler_openmetrics;
+    camuri.user_ctx = NULL;
+    httpd_register_uri_handler(server, &camuri);
+}

code/components/openmetrics/openmetrics.h

@@ -0,0 +1,14 @@
+#ifndef OPENMETRICS_H
+#define OPENMETRICS_H
+
+#include <string>
+#include <vector>
+
+#include "ClassFlowDefineTypes.h"
+
+std::string createMetric(const std::string &metricName, const std::string &help, const std::string &type, const std::string &value);
+std::string createSequenceMetrics(std::string prefix, const std::vector<NumberPost *> &numbers);
+
+void register_openmetrics_uri(httpd_handle_t server);
+
+#endif // OPENMETRICS_H

code/main/main.cpp

@@ -45,6 +45,8 @@
 #include "server_mqtt.h"
 #endif //ENABLE_MQTT
 
+#include "openmetrics.h"
+
 #include "Helper.h"
 #include "system.h"
 #include "statusled.h"
@@ -341,10 +343,13 @@ extern "C" void app_main(void)
     register_server_main_flow_task_uri(server);
     register_server_file_uri(server, "/sdcard");
     register_server_ota_sdcard_uri(server);
+
     #ifdef ENABLE_MQTT
-        register_server_mqtt_uri(server);
+    register_server_mqtt_uri(server);
     #endif //ENABLE_MQTT
 
+    register_openmetrics_uri(server);
+
     gpio_handler_create(server);
 
     ESP_LOGD(TAG, "Before reg server main");

code/main/server_main.cpp

@@ -616,7 +616,7 @@ httpd_handle_t start_webserver(void)
     config.server_port = 80;
     config.ctrl_port = 32768;
     config.max_open_sockets = 5; //20210921 --> previously 7
-    config.max_uri_handlers = 20; // previously 42
+    config.max_uri_handlers = 21;
     config.max_resp_headers = 8;
     config.backlog_conn = 5;
     config.lru_purge_enable = true; // this cuts old connections if new ones are needed.

code/test/components/openmetrics/test_openmetrics.cpp

@@ -0,0 +1,69 @@
+#include <unity.h>
+#include "openmetrics.h"
+
+
+void test_createMetric()
+{
+    // simple happy path
+    const char *expected = "# HELP metric_name short description\n# TYPE metric_name gauge\nmetric_name 123.456\n";
+    std::string result = createMetric("metric_name", "short description", "gauge", "123.456");
+    TEST_ASSERT_EQUAL_STRING(expected, result.c_str());
+}
+
+
+/**
+ * test the replaceString function as it's a dependency to sanitize sequence names
+ */
+void test_replaceString()
+{
+    std::string sample = "hello\\world\\";
+    replaceAll(sample, "\\", "");
+    TEST_ASSERT_EQUAL_STRING("helloworld", sample.c_str());
+
+    sample = "hello\"world\"";
+    replaceAll(sample, "\"", "");
+    TEST_ASSERT_EQUAL_STRING("helloworld", sample.c_str());
+
+    sample = "hello\nworld\n";
+    replaceAll(sample, "\n", "");
+    TEST_ASSERT_EQUAL_STRING("helloworld", sample.c_str());
+
+    sample = "\\\\\\\\\\\\\\\\\\hello\\world\\\\\\\\\\\\\\\\\\\\";
+    replaceAll(sample, "\\", "");
+    TEST_ASSERT_EQUAL_STRING("helloworld", sample.c_str());
+}
+
+
+void test_createSequenceMetrics()
+{
+    std::vector<NumberPost *> sequences;
+    NumberPost *number_1 = new NumberPost;
+    number_1->name = "main";
+    number_1->sActualValue = "123.456";
+    sequences.push_back(number_1);
+
+    const std::string metricNamePrefix = "ai_on_the_edge_device";
+    const std::string metricName = metricNamePrefix + "_flow_value";
+
+    std::string expected1 = "# HELP " + metricName + " current value of meter readout\n# TYPE " + metricName + " gauge\n" +
+                             metricName + "{sequence=\"" + number_1->name + "\"} " + number_1->sActualValue + "\n";
+    TEST_ASSERT_EQUAL_STRING(expected1.c_str(), createSequenceMetrics(metricNamePrefix, sequences).c_str());
+
+    NumberPost *number_2 = new NumberPost;
+    number_2->name = "secondary";
+    number_2->sActualValue = "1.0";
+    sequences.push_back(number_2);
+
+    std::string expected2 = "# HELP " + metricName + " current value of meter readout\n# TYPE " + metricName + " gauge\n" +
+                             metricName + "{sequence=\"" + number_1->name + "\"} " + number_1->sActualValue + "\n" +
+                             metricName + "{sequence=\"" + number_2->name + "\"} " + number_2->sActualValue + "\n";
+    TEST_ASSERT_EQUAL_STRING(expected2.c_str(), createSequenceMetrics(metricNamePrefix, sequences).c_str());
+}
+
+
+void test_openmetrics()
+{
+    test_createMetric();
+    test_replaceString();
+    test_createSequenceMetrics();
+}

code/test/test_suite_flowcontroll.cpp

@@ -26,6 +26,7 @@
 #include "components/jomjol-flowcontroll/test_flow_pp_negative.cpp"
 #include "components/jomjol-flowcontroll/test_PointerEvalAnalogToDigitNew.cpp"
 #include "components/jomjol-flowcontroll/test_getReadoutRawString.cpp"
+#include "components/openmetrics/test_openmetrics.cpp"
 
 esp_err_t initNVSFlash();
 esp_err_t initSDCard();
@@ -70,6 +71,8 @@ void task_UnityTesting(void *pvParameter)
         RUN_TEST(test_doFlowPP4);
         printf("---------------------------------------------------------------------------\n");
         RUN_TEST(test_doFlowPP5);
+        printf("---------------------------------------------------------------------------\n");
+        RUN_TEST(test_openmetrics);
     UNITY_END();
 
     while(1);