diff --git a/Makefile b/Makefile index a132224..f946252 100644 --- a/Makefile +++ b/Makefile @@ -4,8 +4,7 @@ MG_DOCKER_IMAGE_NAME_PREFIX ?= magistrala BUILD_DIR = build SERVICES = opcua lora influxdb-writer influxdb-reader mongodb-writer \ - mongodb-reader cassandra-writer cassandra-reader postgres-writer postgres-reader \ - timescale-writer timescale-reader twins provision certs smtp-notifier smpp-notifier + mongodb-reader cassandra-writer cassandra-reader twins smtp-notifier smpp-notifier TEST_API_SERVICES = certs notifiers provision readers twins TEST_API = $(addprefix test_api_,$(TEST_API_SERVICES)) DOCKERS = $(addprefix docker_,$(SERVICES)) diff --git a/api/openapi/notifiers.yml b/api/openapi/notifiers.yml new file mode 100644 index 0000000..6a4c099 --- /dev/null +++ b/api/openapi/notifiers.yml @@ -0,0 +1,291 @@ +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +openapi: 3.0.1 +info: + title: Magistrala Notifiers service + description: | + HTTP API for Notifiers service. + Some useful links: + - [The Magistrala repository](https://github.com/absmach/magistrala) + contact: + email: info@abstractmachines.fr + license: + name: Apache 2.0 + url: https://github.com/absmach/magistrala/blob/main/LICENSE + version: 0.14.0 + +servers: + - url: http://localhost:9014 + - url: https://localhost:9014 + - url: http://localhost:9015 + - url: https://localhost:9015 + +tags: + - name: notifiers + description: Everything about your Notifiers + externalDocs: + description: Find out more about notifiers + url: https://docs.magistrala.abstractmachines.fr/ + +paths: + /subscriptions: + post: + operationId: createSubscription + summary: Create subscription + description: Creates a new subscription give a topic and contact. + tags: + - notifiers + requestBody: + $ref: "#/components/requestBodies/Create" + responses: + "201": + $ref: "#/components/responses/Create" + "400": + description: Failed due to malformed JSON. + "401": + description: Missing or invalid access token provided. + "403": + description: Failed to perform authorization over the entity. + "409": + description: Failed due to using an existing topic and contact. + "415": + description: Missing or invalid content type. + "422": + description: Database can't process request. + "500": + $ref: "#/components/responses/ServiceError" + get: + operationId: listSubscriptions + summary: List subscriptions + description: List subscriptions given list parameters. + tags: + - notifiers + parameters: + - $ref: "#/components/parameters/Topic" + - $ref: "#/components/parameters/Contact" + - $ref: "#/components/parameters/Offset" + - $ref: "#/components/parameters/Limit" + responses: + "200": + $ref: "#/components/responses/Page" + "400": + description: Failed due to malformed query parameters. + "401": + description: Missing or invalid access token provided. + "403": + description: Failed to perform authorization over the entity. + "404": + description: A non-existent entity request. + "422": + description: Database can't process request. + "500": + $ref: "#/components/responses/ServiceError" + /subscriptions/{id}: + get: + operationId: viewSubscription + summary: Get subscription with the provided id + description: Retrieves a subscription with the provided id. + tags: + - notifiers + parameters: + - $ref: "#/components/parameters/Id" + responses: + "200": + $ref: "#/components/responses/View" + "400": + description: Failed due to malformed ID. + "401": + description: Missing or invalid access token provided. + "403": + description: Failed to perform authorization over the entity. + "404": + description: A non-existent entity request. + "422": + description: Database can't process request. + "500": + $ref: "#/components/responses/ServiceError" + delete: + operationId: removeSubscription + summary: Delete subscription with the provided id + description: Removes a subscription with the provided id. + tags: + - notifiers + parameters: + - $ref: "#/components/parameters/Id" + responses: + "204": + description: Subscription removed + "401": + description: Missing or invalid access token provided. + "403": + description: Failed to perform authorization over the entity. + "404": + description: A non-existent entity request. + "422": + description: Database can't process request. + "500": + $ref: "#/components/responses/ServiceError" + /health: + get: + summary: Retrieves service health check info. + tags: + - health + responses: + "200": + $ref: "#/components/responses/HealthRes" + "500": + $ref: "#/components/responses/ServiceError" + +components: + schemas: + Subscription: + type: object + properties: + id: + type: string + format: ulid + example: 01EWDVKBQSG80B6PQRS9PAAY35 + description: ULID id of the subscription. + owner_id: + type: string + format: uuid + example: 18167738-f7a8-4e96-a123-58c3cd14de3a + description: An id of the owner who created subscription. + topic: + type: string + example: topic.subtopic + description: Topic to which the user subscribes. + contact: + type: string + example: user@example.com + description: The contact of the user to which the notification will be sent. + CreateSubscription: + type: object + properties: + topic: + type: string + example: topic.subtopic + description: Topic to which the user subscribes. + contact: + type: string + example: user@example.com + description: The contact of the user to which the notification will be sent. + Page: + type: object + properties: + subscriptions: + type: array + minItems: 0 + uniqueItems: true + items: + $ref: "#/components/schemas/Subscription" + total: + type: integer + description: Total number of items. + offset: + type: integer + description: Number of items to skip during retrieval. + limit: + type: integer + description: Maximum number of items to return in one page. + + parameters: + Id: + name: id + description: Unique identifier. + in: path + schema: + type: string + format: ulid + required: true + Limit: + name: limit + description: Size of the subset to retrieve. + in: query + schema: + type: integer + default: 10 + maximum: 100 + minimum: 1 + required: false + Offset: + name: offset + description: Number of items to skip during retrieval. + in: query + schema: + type: integer + default: 0 + minimum: 0 + required: false + Topic: + name: topic + description: Topic name. + in: query + schema: + type: string + required: false + Contact: + name: contact + description: Subscription contact. + in: query + schema: + type: string + required: false + + requestBodies: + Create: + description: JSON-formatted document describing the new subscription to be created + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreateSubscription" + + responses: + Create: + description: Created a new subscription. + headers: + Location: + content: + text/plain: + schema: + type: string + description: Created subscription relative URL + example: /subscriptions/{id} + View: + description: View subscription. + content: + application/json: + schema: + $ref: "#/components/schemas/Subscription" + links: + delete: + operationId: removeSubscription + parameters: + id: $response.body#/id + Page: + description: Data retrieved. + content: + application/json: + schema: + $ref: "#/components/schemas/Page" + ServiceError: + description: Unexpected server-side error occurred. + HealthRes: + description: Service Health Check. + content: + application/health+json: + schema: + $ref: "./schemas/HealthInfo.yml" + + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT + description: | + * Users access: "Authorization: Bearer " + +security: + - bearerAuth: [] diff --git a/api/openapi/readers.yml b/api/openapi/readers.yml new file mode 100644 index 0000000..e82de3f --- /dev/null +++ b/api/openapi/readers.yml @@ -0,0 +1,304 @@ +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +openapi: 3.0.1 +info: + title: Magistrala reader service + description: | + HTTP API for reading messages. + Some useful links: + - [The Magistrala repository](https://github.com/absmach/magistrala) + contact: + email: info@abstractmachines.fr + license: + name: Apache 2.0 + url: https://github.com/absmach/magistrala/blob/main/LICENSE + version: 0.14.0 + +servers: + - url: http://localhost:9003 + - url: https://localhost:9003 + - url: http://localhost:9005 + - url: https://localhost:9005 + - url: http://localhost:9007 + - url: https://localhost:9007 + - url: http://localhost:9009 + - url: https://localhost:9009 + - url: http://localhost:9011 + - url: https://localhost:9011 + +tags: + - name: readers + description: Everything about your Readers + externalDocs: + description: Find out more about readers + url: https://docs.magistrala.abstractmachines.fr/ + +paths: + /channels/{chanId}/messages: + get: + operationId: getMessages + summary: Retrieves messages sent to single channel + description: | + Retrieves a list of messages sent to specific channel. Due to + performance concerns, data is retrieved in subsets. The API readers must + ensure that the entire dataset is consumed either by making subsequent + requests, or by increasing the subset size of the initial request. + tags: + - readers + parameters: + - $ref: "#/components/parameters/ChanId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - $ref: "#/components/parameters/Publisher" + - $ref: "#/components/parameters/Name" + - $ref: "#/components/parameters/Value" + - $ref: "#/components/parameters/BoolValue" + - $ref: "#/components/parameters/StringValue" + - $ref: "#/components/parameters/DataValue" + - $ref: "#/components/parameters/From" + - $ref: "#/components/parameters/To" + - $ref: "#/components/parameters/Aggregation" + - $ref: "#/components/parameters/Interval" + responses: + "200": + $ref: "#/components/responses/MessagesPageRes" + "400": + description: Failed due to malformed query parameters. + "401": + description: Missing or invalid access token provided. + "500": + $ref: "#/components/responses/ServiceError" + /health: + get: + operationId: health + summary: Retrieves service health check info. + tags: + - health + responses: + "200": + $ref: "#/components/responses/HealthRes" + "500": + $ref: "#/components/responses/ServiceError" + +components: + schemas: + MessagesPage: + type: object + properties: + total: + type: number + description: Total number of items that are present on the system. + offset: + type: number + description: Number of items that were skipped during retrieval. + limit: + type: number + description: Size of the subset that was retrieved. + messages: + type: array + minItems: 0 + uniqueItems: true + items: + type: object + properties: + channel: + type: integer + description: Unique channel id. + publisher: + type: integer + description: Unique publisher id. + protocol: + type: string + description: Protocol name. + name: + type: string + description: Measured parameter name. + unit: + type: string + description: Value unit. + value: + type: number + description: Measured value in number. + stringValue: + type: string + description: Measured value in string format. + boolValue: + type: boolean + description: Measured value in boolean format. + dataValue: + type: string + description: Measured value in binary format. + valueSum: + type: number + description: Sum value. + time: + type: number + description: Time of measurement. + updateTime: + type: number + description: Time of updating measurement. + + parameters: + ChanId: + name: chanId + description: Unique channel identifier. + in: path + schema: + type: string + format: uuid + required: true + Limit: + name: limit + description: Size of the subset to retrieve. + in: query + schema: + type: integer + default: 10 + maximum: 100 + minimum: 1 + required: false + Offset: + name: offset + description: Number of items to skip during retrieval. + in: query + schema: + type: integer + default: 0 + minimum: 0 + required: false + Publisher: + name: Publisher + description: Unique thing identifier. + in: query + schema: + type: string + format: uuid + required: false + Name: + name: name + description: SenML message name. + in: query + schema: + type: string + required: false + Value: + name: v + description: SenML message value. + in: query + schema: + type: string + required: false + BoolValue: + name: vb + description: SenML message bool value. + in: query + schema: + type: boolean + required: false + StringValue: + name: vs + description: SenML message string value. + in: query + schema: + type: string + required: false + DataValue: + name: vd + description: SenML message data value. + in: query + schema: + type: string + required: false + Comparator: + name: comparator + description: Value comparison operator. + in: query + schema: + type: string + default: eq + enum: + - eq + - lt + - le + - gt + - ge + required: false + From: + name: from + description: SenML message time in nanoseconds (integer part represents seconds). + in: query + schema: + type: number + example: 1709218556069 + required: false + To: + name: to + description: SenML message time in nanoseconds (integer part represents seconds). + in: query + schema: + type: number + example: 1709218757503 + required: false + Aggregation: + name: aggregation + description: Aggregation function. + in: query + schema: + type: string + enum: + - MAX + - AVG + - MIN + - SUM + - COUNT + - max + - min + - sum + - avg + - count + example: MAX + required: false + Interval: + name: interval + description: Aggregation interval. + in: query + schema: + type: string + example: 10s + required: false + + responses: + MessagesPageRes: + description: Data retrieved. + content: + application/json: + schema: + $ref: "#/components/schemas/MessagesPage" + ServiceError: + description: Unexpected server-side error occurred. + HealthRes: + description: Service Health Check. + content: + application/health+json: + schema: + $ref: "./schemas/HealthInfo.yml" + + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT + description: | + * Users access: "Authorization: Bearer " + + thingAuth: + type: http + scheme: bearer + bearerFormat: uuid + description: | + * Things access: "Authorization: Thing " + +security: + - bearerAuth: [] + - thingAuth: [] diff --git a/api/openapi/twins.yml b/api/openapi/twins.yml new file mode 100644 index 0000000..f582958 --- /dev/null +++ b/api/openapi/twins.yml @@ -0,0 +1,430 @@ +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +openapi: 3.0.1 +info: + title: Magistrala twins service + description: | + HTTP API for managing digital twins and their states. + Some useful links: + - [The Magistrala repository](https://github.com/absmach/magistrala) + contact: + email: info@abstractmachines.fr + license: + name: Apache 2.0 + url: https://github.com/absmach/magistrala/blob/main/LICENSE + version: 0.14.0 + +servers: + - url: http://localhost:9018 + - url: https://localhost:9018 + +tags: + - name: twins + description: Everything about your Twins + externalDocs: + description: Find out more about twins + url: https://docs.magistrala.abstractmachines.fr/ + +paths: + /twins: + post: + operationId: createTwin + summary: Adds new twin + description: | + Adds new twin to the list of twins owned by user identified using + the provided access token. + tags: + - twins + requestBody: + $ref: "#/components/requestBodies/TwinReq" + responses: + "201": + $ref: "#/components/responses/TwinCreateRes" + "400": + description: Failed due to malformed JSON. + "401": + description: Missing or invalid access token provided. + "415": + description: Missing or invalid content type. + "422": + description: Database can't process request. + "500": + $ref: "#/components/responses/ServiceError" + + get: + operationId: getTwins + summary: Retrieves twins + description: | + Retrieves a list of twins. Due to performance concerns, data + is retrieved in subsets. + tags: + - twins + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - $ref: "#/components/parameters/Name" + - $ref: "#/components/parameters/Metadata" + responses: + "200": + $ref: "#/components/responses/TwinsPageRes" + "400": + description: Failed due to malformed query parameters. + "401": + description: Missing or invalid access token provided. + "422": + description: Database can't process request. + "500": + $ref: "#/components/responses/ServiceError" + + /twins/{twinID}: + get: + operationId: getTwin + summary: Retrieves twin info + tags: + - twins + parameters: + - $ref: "#/components/parameters/TwinID" + responses: + "200": + $ref: "#/components/responses/TwinRes" + "400": + description: Failed due to malformed twin's ID. + "401": + description: Missing or invalid access token provided. + "404": + description: Twin does not exist. + "422": + description: Database can't process request. + "500": + $ref: "#/components/responses/ServiceError" + put: + operationId: updateTwin + summary: Updates twin info + description: | + Update is performed by replacing the current resource data with values + provided in a request payload. Note that the twin's ID cannot be changed. + tags: + - twins + parameters: + - $ref: "#/components/parameters/TwinID" + requestBody: + $ref: "#/components/requestBodies/TwinReq" + responses: + "200": + description: Twin updated. + "400": + description: Failed due to malformed twin's ID or malformed JSON. + "401": + description: Missing or invalid access token provided. + "404": + description: Twin does not exist. + "415": + description: Missing or invalid content type. + "422": + description: Database can't process request. + "500": + $ref: "#/components/responses/ServiceError" + delete: + operationId: removeTwin + summary: Removes a twin + description: Removes a twin. + tags: + - twins + parameters: + - $ref: "#/components/parameters/TwinID" + responses: + "204": + description: Twin removed. + "400": + description: Failed due to malformed twin's ID. + "401": + description: Missing or invalid access token provided + "404": + description: Twin does not exist. + "415": + description: Missing or invalid content type. + "422": + description: Database can't process request. + "500": + $ref: "#/components/responses/ServiceError" + + /states/{twinID}: + get: + operationId: getStates + summary: Retrieves states of twin with id twinID + description: | + Retrieves a list of states. Due to performance concerns, data + is retrieved in subsets. + tags: + - states + parameters: + - $ref: "#/components/parameters/TwinID" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + responses: + "200": + $ref: "#/components/responses/StatesPageRes" + "400": + description: Failed due to malformed query parameters. + "401": + description: Missing or invalid access token provided. + "404": + description: Twin does not exist. + "422": + description: Database can't process request. + "500": + $ref: "#/components/responses/ServiceError" + /health: + get: + summary: Retrieves service health check info. + tags: + - health + responses: + "200": + $ref: "#/components/responses/HealthRes" + "500": + $ref: "#/components/responses/ServiceError" + +components: + parameters: + Limit: + name: limit + description: Size of the subset to retrieve. + in: query + schema: + type: integer + default: 10 + maximum: 100 + minimum: 1 + required: false + Offset: + name: offset + description: Number of items to skip during retrieval. + in: query + schema: + type: integer + default: 0 + minimum: 0 + required: false + Name: + name: name + description: Twin name + in: query + schema: + type: string + required: false + Metadata: + name: metadata + description: | + Metadata filter. Filtering is performed matching the parameter with + metadata on top level. Parameter is json. + in: query + schema: + type: string + minimum: 0 + required: false + TwinID: + name: twinID + description: Unique twin identifier. + in: path + schema: + type: string + format: uuid + minimum: 1 + required: true + + schemas: + Attribute: + type: object + properties: + name: + type: string + description: Name of the attribute. + channel: + type: string + description: Magistrala channel used by attribute. + subtopic: + type: string + description: Subtopic used by attribute. + persist_state: + type: boolean + description: Trigger state creation based on the attribute. + Definition: + type: object + properties: + delta: + type: number + description: Minimal time delay before new state creation. + attributes: + type: array + minItems: 0 + items: + $ref: "#/components/schemas/Attribute" + TwinReqObj: + type: object + properties: + name: + type: string + description: Free-form twin name. + metadata: + type: object + description: Arbitrary, object-encoded twin's data. + definition: + $ref: "#/components/schemas/Definition" + TwinResObj: + type: object + properties: + owner: + type: string + description: Email address of Magistrala user that owns twin. + id: + type: string + format: uuid + description: Unique twin identifier generated by the service. + name: + type: string + description: Free-form twin name. + revision: + type: number + description: Oridnal revision number of twin. + created: + type: string + format: date + description: Twin creation date and time. + updated: + type: string + format: date + description: Twin update date and time. + definitions: + type: array + minItems: 0 + items: + $ref: "#/components/schemas/Definition" + metadata: + type: object + description: Arbitrary, object-encoded twin's data. + TwinsPage: + type: object + properties: + twins: + type: array + minItems: 0 + items: + $ref: "#/components/schemas/TwinResObj" + total: + type: integer + description: Total number of items. + offset: + type: integer + description: Number of items to skip during retrieval. + limit: + type: integer + description: Maximum number of items to return in one page. + required: + - twins + State: + type: object + properties: + twin_id: + type: string + format: uuid + description: ID of twin state belongs to. + id: + type: number + description: State position in a time row of states. + created: + type: string + format: date + description: State creation date. + payload: + type: object + description: Object-encoded states's payload. + StatesPage: + type: object + properties: + states: + type: array + minItems: 0 + items: + $ref: "#/components/schemas/State" + total: + type: integer + description: Total number of items. + offset: + type: integer + description: Number of items to skip during retrieval. + limit: + type: integer + description: Maximum number of items to return in one page. + required: + - states + + requestBodies: + TwinReq: + description: JSON-formatted document describing the twin to create or update. + content: + application/json: + schema: + $ref: "#/components/schemas/TwinReqObj" + required: true + + responses: + TwinCreateRes: + description: Created twin's relative URL (i.e. /twins/{twinID}). + headers: + Location: + content: + text/plain: + schema: + type: string + TwinRes: + description: Data retrieved. + content: + application/json: + schema: + $ref: "#/components/schemas/TwinResObj" + links: + update: + operationId: updateTwin + parameters: + twinID: $response.body#/id + delete: + operationId: removeTwin + parameters: + twinID: $response.body#/id + states: + operationId: getStates + parameters: + twinID: $response.body#/id + TwinsPageRes: + description: Data retrieved. + content: + application/json: + schema: + $ref: "#/components/schemas/TwinsPage" + StatesPageRes: + description: Data retrieved. + content: + application/json: + schema: + $ref: "#/components/schemas/StatesPage" + ServiceError: + description: Unexpected server-side error occurred. + HealthRes: + description: Service Health Check. + content: + application/health+json: + schema: + $ref: "./schemas/HealthInfo.yml" + + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT + description: | + * Users access: "Authorization: Bearer " + +security: + - bearerAuth: [] diff --git a/api/schemas/HealthInfo.yml b/api/schemas/HealthInfo.yml new file mode 100644 index 0000000..9c4e858 --- /dev/null +++ b/api/schemas/HealthInfo.yml @@ -0,0 +1,30 @@ +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +type: object +properties: + status: + type: string + description: Service status. + enum: + - pass + version: + type: string + description: Service version. + example: v0.14.0 + commit: + type: string + description: Service commit hash. + example: 73362210dd2e04e389eaddb802cab3fe03976593 + description: + type: string + description: Service description. + example: service + build_time: + type: string + description: Service build time. + example: 2024-02-01_12:18:15 + instance_id: + type: string + description: Service instance ID. + example: 8edbf8af-7db7-4218-bb4f-a8a929ff5266 diff --git a/certs/README.md b/certs/README.md deleted file mode 100644 index 52124be..0000000 --- a/certs/README.md +++ /dev/null @@ -1,130 +0,0 @@ -# Certs Service - -Issues certificates for things. `Certs` service can create certificates to be used when `Magistrala` is deployed to support mTLS. -Certificate service can create certificates using PKI mode - where certificates issued by PKI, when you deploy `Vault` as PKI certificate management `cert` service will proxy requests to `Vault` previously checking access rights and saving info on successfully created certificate. - -## PKI mode - -When `MG_CERTS_VAULT_HOST` is set it is presumed that `Vault` is installed and `certs` service will issue certificates using `Vault` API. -First you'll need to set up `Vault`. -To setup `Vault` follow steps in [Build Your Own Certificate Authority (CA)](https://learn.hashicorp.com/tutorials/vault/pki-engine). - -For lab purposes you can use docker-compose and script for setting up PKI in [https://github.com/absmach/magistrala/blob/main/docker/addons/vault/README.md](https://github.com/absmach/magistrala/blob/main/docker/addons/vault/README.md) - -```bash -MG_CERTS_VAULT_HOST= -MG_CERTS_VAULT_NAMESPACE= -MG_CERTS_VAULT_APPROLE_ROLEID= -MG_CERTS_VAULT_APPROLE_SECRET= -MG_CERTS_VAULT_THINGS_CERTS_PKI_PATH= -MG_CERTS_VAULT_THINGS_CERTS_PKI_ROLE_NAME= -``` - -The certificates can also be revoked using `certs` service. To revoke a certificate you need to provide `thing_id` of the thing for which the certificate was issued. - -```bash -curl -s -S -X DELETE http://localhost:9019/certs/revoke -H "Authorization: Bearer $TOK" -H 'Content-Type: application/json' -d '{"thing_id":"c30b8842-507c-4bcd-973c-74008cef3be5"}' -``` - -## Configuration - -The service is configured using the environment variables presented in the following table. Note that any unset variables will be replaced with their default values. - - -| Variable | Description | Default | -| :---------------------------------------- | --------------------------------------------------------------------------- | ---------------------------------------------------------------------- | -| MG_CERTS_LOG_LEVEL | Log level for the Certs (debug, info, warn, error) | info | -| MG_CERTS_HTTP_HOST | Service Certs host | "" | -| MG_CERTS_HTTP_PORT | Service Certs port | 9019 | -| MG_CERTS_HTTP_SERVER_CERT | Path to the PEM encoded server certificate file | "" | -| MG_CERTS_HTTP_SERVER_KEY | Path to the PEM encoded server key file | "" | -| MG_AUTH_GRPC_URL | Auth service gRPC URL | [localhost:8181](localhost:8181) | -| MG_AUTH_GRPC_TIMEOUT | Auth service gRPC request timeout in seconds | 1s | -| MG_AUTH_GRPC_CLIENT_CERT | Path to the PEM encoded auth service gRPC client certificate file | "" | -| MG_AUTH_GRPC_CLIENT_KEY | Path to the PEM encoded auth service gRPC client key file | "" | -| MG_AUTH_GRPC_SERVER_CERTS | Path to the PEM encoded auth server gRPC server trusted CA certificate file | "" | -| MG_CERTS_SIGN_CA_PATH | Path to the PEM encoded CA certificate file | ca.crt | -| MG_CERTS_SIGN_CA_KEY_PATH | Path to the PEM encoded CA key file | ca.key | -| MG_CERTS_VAULT_HOST | Vault host | http://vault:8200 | -| MG_CERTS_VAULT_NAMESPACE | Vault namespace in which pki is present | magistrala | -| MG_CERTS_VAULT_APPROLE_ROLEID | Vault AppRole auth RoleID | magistrala | -| MG_CERTS_VAULT_APPROLE_SECRET | Vault AppRole auth Secret | magistrala | -| MG_CERTS_VAULT_THINGS_CERTS_PKI_PATH | Vault PKI path for issuing Things Certificates | pki_int | -| MG_CERTS_VAULT_THINGS_CERTS_PKI_ROLE_NAME | Vault PKI Role Name for issuing Things Certificates | magistrala_things_certs | -| MG_CERTS_DB_HOST | Database host | localhost | -| MG_CERTS_DB_PORT | Database port | 5432 | -| MG_CERTS_DB_PASS | Database password | magistrala | -| MG_CERTS_DB_USER | Database user | magistrala | -| MG_CERTS_DB_NAME | Database name | certs | -| MG_CERTS_DB_SSL_MODE | Database SSL mode | disable | -| MG_CERTS_DB_SSL_CERT | Database SSL certificate | "" | -| MG_CERTS_DB_SSL_KEY | Database SSL key | "" | -| MG_CERTS_DB_SSL_ROOT_CERT | Database SSL root certificate | "" | -| MG_THINGS_URL | Things service URL | [localhost:9000](localhost:9000) | -| MG_JAEGER_URL | Jaeger server URL | [http://localhost:14268/api/traces](http://localhost:14268/api/traces) | -| MG_JAEGER_TRACE_RATIO | Jaeger sampling ratio | 1.0 | -| MG_SEND_TELEMETRY | Send telemetry to magistrala call home server | true | -| MG_CERTS_INSTANCE_ID | Service instance ID | "" | - -## Deployment - -The service is distributed as Docker container. Check the [`certs`](https://github.com/absmach/magistrala/blob/main/docker/addons/bootstrap/docker-compose.yml) service section in docker-compose file to see how the service is deployed. - -Running this service outside of container requires working instance of the auth service, things service, postgres database, vault and Jaeger server. -To start the service outside of the container, execute the following shell script: - -```bash -# download the latest version of the service -git clone https://github.com/absmach/magistrala - -cd magistrala - -# compile the certs -make certs - -# copy binary to bin -make install - -# set the environment variables and run the service -MG_CERTS_LOG_LEVEL=info \ -MG_CERTS_HTTP_HOST=localhost \ -MG_CERTS_HTTP_PORT=9019 \ -MG_CERTS_HTTP_SERVER_CERT="" \ -MG_CERTS_HTTP_SERVER_KEY="" \ -MG_AUTH_GRPC_URL=localhost:8181 \ -MG_AUTH_GRPC_TIMEOUT=1s \ -MG_AUTH_GRPC_CLIENT_CERT="" \ -MG_AUTH_GRPC_CLIENT_KEY="" \ -MG_AUTH_GRPC_SERVER_CERTS="" \ -MG_CERTS_SIGN_CA_PATH=ca.crt \ -MG_CERTS_SIGN_CA_KEY_PATH=ca.key \ -MG_CERTS_VAULT_HOST=http://vault:8200 \ -MG_CERTS_VAULT_NAMESPACE=magistrala \ -MG_CERTS_VAULT_APPROLE_ROLEID=magistrala \ -MG_CERTS_VAULT_APPROLE_SECRET=magistrala \ -MG_CERTS_VAULT_THINGS_CERTS_PKI_PATH=pki_int \ -MG_CERTS_VAULT_THINGS_CERTS_PKI_ROLE_NAME=magistrala_things_certs \ -MG_CERTS_DB_HOST=localhost \ -MG_CERTS_DB_PORT=5432 \ -MG_CERTS_DB_PASS=magistrala \ -MG_CERTS_DB_USER=magistrala \ -MG_CERTS_DB_NAME=certs \ -MG_CERTS_DB_SSL_MODE=disable \ -MG_CERTS_DB_SSL_CERT="" \ -MG_CERTS_DB_SSL_KEY="" \ -MG_CERTS_DB_SSL_ROOT_CERT="" \ -MG_THINGS_URL=localhost:9000 \ -MG_JAEGER_URL=http://localhost:14268/api/traces \ -MG_JAEGER_TRACE_RATIO=1.0 \ -MG_SEND_TELEMETRY=true \ -MG_CERTS_INSTANCE_ID="" \ -$GOBIN/magistrala-certs -``` - -Setting `MG_CERTS_HTTP_SERVER_CERT` and `MG_CERTS_HTTP_SERVER_KEY` will enable TLS against the service. The service expects a file in PEM format for both the certificate and the key. - -Setting `MG_AUTH_GRPC_CLIENT_CERT` and `MG_AUTH_GRPC_CLIENT_KEY` will enable TLS against the auth service. The service expects a file in PEM format for both the certificate and the key. Setting `MG_AUTH_GRPC_SERVER_CERTS` will enable TLS against the auth service trusting only those CAs that are provided. The service expects a file in PEM format of trusted CAs. - -## Usage - -For more information about service capabilities and its usage, please check out the [Certs section](https://docs.magistrala.abstractmachines.fr/certs/). diff --git a/certs/api/doc.go b/certs/api/doc.go deleted file mode 100644 index 943cf19..0000000 --- a/certs/api/doc.go +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package api contains implementation of certs service HTTP API. -package api diff --git a/certs/api/endpoint.go b/certs/api/endpoint.go deleted file mode 100644 index fea0c63..0000000 --- a/certs/api/endpoint.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package api - -import ( - "context" - - "github.com/absmach/magistrala/certs" - "github.com/absmach/magistrala/pkg/apiutil" - "github.com/absmach/magistrala/pkg/errors" - "github.com/go-kit/kit/endpoint" -) - -func issueCert(svc certs.Service) endpoint.Endpoint { - return func(ctx context.Context, request interface{}) (interface{}, error) { - req := request.(addCertsReq) - if err := req.validate(); err != nil { - return nil, errors.Wrap(apiutil.ErrValidation, err) - } - res, err := svc.IssueCert(ctx, req.token, req.ThingID, req.TTL) - if err != nil { - return certsRes{}, errors.Wrap(apiutil.ErrValidation, err) - } - - return certsRes{ - CertSerial: res.Serial, - ThingID: res.ThingID, - ClientCert: res.ClientCert, - ClientKey: res.ClientKey, - Expiration: res.Expire, - created: true, - }, nil - } -} - -func listSerials(svc certs.Service) endpoint.Endpoint { - return func(ctx context.Context, request interface{}) (interface{}, error) { - req := request.(listReq) - if err := req.validate(); err != nil { - return nil, errors.Wrap(apiutil.ErrValidation, err) - } - - page, err := svc.ListSerials(ctx, req.token, req.thingID, req.offset, req.limit) - if err != nil { - return certsPageRes{}, errors.Wrap(apiutil.ErrValidation, err) - } - res := certsPageRes{ - pageRes: pageRes{ - Total: page.Total, - Offset: page.Offset, - Limit: page.Limit, - }, - Certs: []certsRes{}, - } - - for _, cert := range page.Certs { - cr := certsRes{ - CertSerial: cert.Serial, - } - res.Certs = append(res.Certs, cr) - } - return res, nil - } -} - -func viewCert(svc certs.Service) endpoint.Endpoint { - return func(ctx context.Context, request interface{}) (interface{}, error) { - req := request.(viewReq) - if err := req.validate(); err != nil { - return nil, errors.Wrap(apiutil.ErrValidation, err) - } - - cert, err := svc.ViewCert(ctx, req.token, req.serialID) - if err != nil { - return certsPageRes{}, errors.Wrap(apiutil.ErrValidation, err) - } - - certRes := certsRes{ - CertSerial: cert.Serial, - ThingID: cert.ThingID, - ClientCert: cert.ClientCert, - Expiration: cert.Expire, - } - - return certRes, nil - } -} - -func revokeCert(svc certs.Service) endpoint.Endpoint { - return func(ctx context.Context, request interface{}) (interface{}, error) { - req := request.(revokeReq) - if err := req.validate(); err != nil { - return nil, errors.Wrap(apiutil.ErrValidation, err) - } - res, err := svc.RevokeCert(ctx, req.token, req.certID) - if err != nil { - return nil, err - } - return revokeCertsRes{ - RevocationTime: res.RevocationTime, - }, nil - } -} diff --git a/certs/api/endpoint_test.go b/certs/api/endpoint_test.go deleted file mode 100644 index 39aa944..0000000 --- a/certs/api/endpoint_test.go +++ /dev/null @@ -1,528 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package api_test - -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/absmach/magistrala/certs" - httpapi "github.com/absmach/magistrala/certs/api" - "github.com/absmach/magistrala/certs/mocks" - mglog "github.com/absmach/magistrala/logger" - "github.com/absmach/magistrala/pkg/apiutil" - "github.com/absmach/magistrala/pkg/errors" - svcerr "github.com/absmach/magistrala/pkg/errors/service" - "github.com/absmach/mg-contrib/pkg/testsutil" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -var ( - contentType = "application/json" - valid = "valid" - invalid = "invalid" - thingID = testsutil.GenerateUUID(&testing.T{}) - serial = testsutil.GenerateUUID(&testing.T{}) - ttl = "1h" - cert = certs.Cert{ - OwnerID: testsutil.GenerateUUID(&testing.T{}), - ThingID: thingID, - ClientCert: valid, - IssuingCA: valid, - CAChain: []string{valid}, - ClientKey: valid, - PrivateKeyType: valid, - Serial: serial, - Expire: time.Now().Add(time.Hour), - } -) - -type testRequest struct { - client *http.Client - method string - url string - contentType string - token string - body io.Reader -} - -func (tr testRequest) make() (*http.Response, error) { - req, err := http.NewRequest(tr.method, tr.url, tr.body) - if err != nil { - return nil, err - } - if tr.token != "" { - req.Header.Set("Authorization", apiutil.BearerPrefix+tr.token) - } - if tr.contentType != "" { - req.Header.Set("Content-Type", tr.contentType) - } - - return tr.client.Do(req) -} - -func newCertServer() (*httptest.Server, *mocks.Service) { - svc := new(mocks.Service) - logger := mglog.NewMock() - - mux := httpapi.MakeHandler(svc, logger, "") - return httptest.NewServer(mux), svc -} - -func TestIssueCert(t *testing.T) { - cs, svc := newCertServer() - defer cs.Close() - - validReqString := `{"thing_id": "%s","ttl": "%s"}` - invalidReqString := `{"thing_id": "%s","ttl": %s}` - - cases := []struct { - desc string - token string - contentType string - thingID string - ttl string - request string - status int - svcRes certs.Cert - svcErr error - err error - }{ - { - desc: "issue cert successfully", - token: valid, - contentType: contentType, - thingID: thingID, - ttl: ttl, - request: fmt.Sprintf(validReqString, thingID, ttl), - status: http.StatusCreated, - svcRes: cert, - svcErr: nil, - err: nil, - }, - { - desc: "issue with invalid token", - token: invalid, - contentType: contentType, - thingID: thingID, - ttl: ttl, - request: fmt.Sprintf(validReqString, thingID, ttl), - status: http.StatusUnauthorized, - svcRes: certs.Cert{}, - svcErr: svcerr.ErrAuthentication, - err: svcerr.ErrAuthentication, - }, - { - desc: "issue with empty token", - token: "", - contentType: contentType, - request: fmt.Sprintf(validReqString, thingID, ttl), - status: http.StatusUnauthorized, - svcRes: certs.Cert{}, - svcErr: nil, - err: apiutil.ErrBearerToken, - }, - { - desc: "issue with empty thing id", - token: valid, - contentType: contentType, - request: fmt.Sprintf(validReqString, "", ttl), - status: http.StatusBadRequest, - svcRes: certs.Cert{}, - svcErr: nil, - err: apiutil.ErrMissingID, - }, - { - desc: "issue with empty ttl", - token: valid, - contentType: contentType, - request: fmt.Sprintf(validReqString, thingID, ""), - status: http.StatusBadRequest, - svcRes: certs.Cert{}, - svcErr: nil, - err: apiutil.ErrMissingCertData, - }, - { - desc: "issue with invalid ttl", - token: valid, - contentType: contentType, - request: fmt.Sprintf(validReqString, thingID, invalid), - status: http.StatusBadRequest, - svcRes: certs.Cert{}, - svcErr: nil, - err: apiutil.ErrInvalidCertData, - }, - { - desc: "issue with invalid content type", - token: valid, - contentType: "application/xml", - request: fmt.Sprintf(validReqString, thingID, ttl), - status: http.StatusUnsupportedMediaType, - svcRes: certs.Cert{}, - svcErr: nil, - err: apiutil.ErrUnsupportedContentType, - }, - { - desc: "issue with invalid request body", - token: valid, - contentType: contentType, - request: fmt.Sprintf(invalidReqString, thingID, ttl), - status: http.StatusInternalServerError, - svcRes: certs.Cert{}, - svcErr: nil, - err: apiutil.ErrValidation, - }, - } - - for _, tc := range cases { - req := testRequest{ - client: cs.Client(), - method: http.MethodPost, - url: fmt.Sprintf("%s/certs", cs.URL), - contentType: tc.contentType, - token: tc.token, - body: strings.NewReader(tc.request), - } - svcCall := svc.On("IssueCert", mock.Anything, tc.token, tc.thingID, tc.ttl).Return(tc.svcRes, tc.svcErr) - res, err := req.make() - assert.Nil(t, err, fmt.Sprintf("%s: unexpected error %s", tc.desc, err)) - var errRes respBody - err = json.NewDecoder(res.Body).Decode(&errRes) - assert.Nil(t, err, fmt.Sprintf("%s: unexpected error while decoding response body: %s", tc.desc, err)) - if errRes.Err != "" || errRes.Message != "" { - err = errors.Wrap(errors.New(errRes.Err), errors.New(errRes.Message)) - } - assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err)) - assert.Equal(t, tc.status, res.StatusCode, fmt.Sprintf("%s: expected status code %d got %d", tc.desc, tc.status, res.StatusCode)) - svcCall.Unset() - } -} - -func TestViewCert(t *testing.T) { - cs, svc := newCertServer() - defer cs.Close() - - cases := []struct { - desc string - token string - serialID string - status int - svcRes certs.Cert - svcErr error - err error - }{ - { - desc: "view cert successfully", - token: valid, - serialID: serial, - status: http.StatusOK, - svcRes: cert, - svcErr: nil, - err: nil, - }, - { - desc: "view with invalid token", - token: invalid, - serialID: serial, - status: http.StatusUnauthorized, - svcRes: certs.Cert{}, - svcErr: svcerr.ErrAuthentication, - err: svcerr.ErrAuthentication, - }, - { - desc: "view with empty token", - token: "", - serialID: serial, - status: http.StatusUnauthorized, - svcRes: certs.Cert{}, - svcErr: nil, - err: apiutil.ErrBearerToken, - }, - { - desc: "view non-existing cert", - token: valid, - serialID: invalid, - status: http.StatusNotFound, - svcRes: certs.Cert{}, - svcErr: svcerr.ErrNotFound, - err: svcerr.ErrNotFound, - }, - } - for _, tc := range cases { - req := testRequest{ - client: cs.Client(), - method: http.MethodGet, - url: fmt.Sprintf("%s/certs/%s", cs.URL, tc.serialID), - token: tc.token, - } - svcCall := svc.On("ViewCert", mock.Anything, tc.token, tc.serialID).Return(tc.svcRes, tc.svcErr) - res, err := req.make() - assert.Nil(t, err, fmt.Sprintf("%s: unexpected error %s", tc.desc, err)) - var errRes respBody - err = json.NewDecoder(res.Body).Decode(&errRes) - assert.Nil(t, err, fmt.Sprintf("%s: unexpected error while decoding response body: %s", tc.desc, err)) - if errRes.Err != "" || errRes.Message != "" { - err = errors.Wrap(errors.New(errRes.Err), errors.New(errRes.Message)) - } - assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err)) - assert.Equal(t, tc.status, res.StatusCode, fmt.Sprintf("%s: expected status code %d got %d", tc.desc, tc.status, res.StatusCode)) - svcCall.Unset() - } -} - -func TestRevokeCert(t *testing.T) { - cs, svc := newCertServer() - defer cs.Close() - - cases := []struct { - desc string - token string - serialID string - status int - svcRes certs.Revoke - svcErr error - err error - }{ - { - desc: "revoke cert successfully", - token: valid, - serialID: serial, - status: http.StatusOK, - svcRes: certs.Revoke{RevocationTime: time.Now()}, - svcErr: nil, - err: nil, - }, - { - desc: "revoke with invalid token", - token: invalid, - serialID: serial, - status: http.StatusUnauthorized, - svcRes: certs.Revoke{}, - svcErr: svcerr.ErrAuthentication, - err: svcerr.ErrAuthentication, - }, - { - desc: "revoke with empty token", - token: "", - serialID: serial, - status: http.StatusUnauthorized, - svcErr: nil, - err: apiutil.ErrBearerToken, - }, - { - desc: "revoke non-existing cert", - token: valid, - serialID: invalid, - status: http.StatusNotFound, - svcRes: certs.Revoke{}, - svcErr: svcerr.ErrNotFound, - err: svcerr.ErrNotFound, - }, - } - for _, tc := range cases { - req := testRequest{ - client: cs.Client(), - method: http.MethodDelete, - url: fmt.Sprintf("%s/certs/%s", cs.URL, tc.serialID), - token: tc.token, - } - svcCall := svc.On("RevokeCert", mock.Anything, tc.token, tc.serialID).Return(tc.svcRes, tc.svcErr) - res, err := req.make() - assert.Nil(t, err, fmt.Sprintf("%s: unexpected error %s", tc.desc, err)) - var errRes respBody - err = json.NewDecoder(res.Body).Decode(&errRes) - assert.Nil(t, err, fmt.Sprintf("%s: unexpected error while decoding response body: %s", tc.desc, err)) - if errRes.Err != "" || errRes.Message != "" { - err = errors.Wrap(errors.New(errRes.Err), errors.New(errRes.Message)) - } - assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n ", tc.desc, tc.err, err)) - assert.Equal(t, tc.status, res.StatusCode, fmt.Sprintf("%s: expected status code %d got %d", tc.desc, tc.status, res.StatusCode)) - svcCall.Unset() - } -} - -func TestListSerials(t *testing.T) { - cs, svc := newCertServer() - defer cs.Close() - - cases := []struct { - desc string - token string - thingID string - offset uint64 - limit uint64 - query string - status int - svcRes certs.Page - svcErr error - err error - }{ - { - desc: "list certs successfully with default limit", - token: valid, - thingID: thingID, - offset: 0, - limit: 10, - query: "", - status: http.StatusOK, - svcRes: certs.Page{ - Total: 1, - Offset: 0, - Limit: 10, - Certs: []certs.Cert{cert}, - }, - svcErr: nil, - err: nil, - }, - { - desc: "list certs successfully with limit", - token: valid, - thingID: thingID, - offset: 0, - limit: 5, - query: "?limit=5", - status: http.StatusOK, - svcRes: certs.Page{ - Total: 1, - Offset: 0, - Limit: 5, - Certs: []certs.Cert{cert}, - }, - svcErr: nil, - err: nil, - }, - { - desc: "list certs successfully with offset", - token: valid, - thingID: thingID, - offset: 1, - limit: 10, - query: "?offset=1", - status: http.StatusOK, - svcRes: certs.Page{ - Total: 1, - Offset: 1, - Limit: 10, - Certs: []certs.Cert{}, - }, - svcErr: nil, - err: nil, - }, - { - desc: "list certs successfully with offset and limit", - token: valid, - thingID: thingID, - offset: 1, - limit: 5, - query: "?offset=1&limit=5", - status: http.StatusOK, - svcRes: certs.Page{ - Total: 1, - Offset: 1, - Limit: 5, - Certs: []certs.Cert{}, - }, - svcErr: nil, - err: nil, - }, - { - desc: "list with invalid token", - token: invalid, - thingID: thingID, - offset: 0, - limit: 10, - query: "", - status: http.StatusUnauthorized, - svcRes: certs.Page{}, - svcErr: svcerr.ErrAuthentication, - err: svcerr.ErrAuthentication, - }, - { - desc: "list with empty token", - token: "", - thingID: thingID, - offset: 0, - limit: 10, - query: "", - status: http.StatusUnauthorized, - svcRes: certs.Page{}, - svcErr: nil, - err: apiutil.ErrBearerToken, - }, - { - desc: "list with limit exceeding max limit", - token: valid, - thingID: thingID, - query: "?limit=1000", - status: http.StatusBadRequest, - svcRes: certs.Page{}, - svcErr: nil, - err: apiutil.ErrLimitSize, - }, - { - desc: "list with invalid offset", - token: valid, - thingID: thingID, - query: "?offset=invalid", - status: http.StatusBadRequest, - svcRes: certs.Page{}, - svcErr: nil, - err: apiutil.ErrValidation, - }, - { - desc: "list with invalid limit", - token: valid, - thingID: thingID, - query: "?limit=invalid", - status: http.StatusBadRequest, - svcRes: certs.Page{}, - svcErr: nil, - err: apiutil.ErrValidation, - }, - { - desc: "list with invalid thing id", - token: valid, - thingID: invalid, - offset: 0, - limit: 10, - query: "", - status: http.StatusNotFound, - svcRes: certs.Page{}, - svcErr: svcerr.ErrNotFound, - err: svcerr.ErrNotFound, - }, - } - for _, tc := range cases { - req := testRequest{ - client: cs.Client(), - method: http.MethodGet, - url: fmt.Sprintf("%s/serials/%s", cs.URL, tc.thingID) + tc.query, - token: tc.token, - } - svcCall := svc.On("ListSerials", mock.Anything, tc.token, tc.thingID, tc.offset, tc.limit).Return(tc.svcRes, tc.svcErr) - res, err := req.make() - assert.Nil(t, err, fmt.Sprintf("%s: unexpected error %s", tc.desc, err)) - var errRes respBody - err = json.NewDecoder(res.Body).Decode(&errRes) - assert.Nil(t, err, fmt.Sprintf("%s: unexpected error while decoding response body: %s", tc.desc, err)) - if errRes.Err != "" || errRes.Message != "" { - err = errors.Wrap(errors.New(errRes.Err), errors.New(errRes.Message)) - } - assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n ", tc.desc, tc.err, err)) - assert.Equal(t, tc.status, res.StatusCode, fmt.Sprintf("%s: expected status code %d got %d", tc.desc, tc.status, res.StatusCode)) - svcCall.Unset() - } -} - -type respBody struct { - Err string `json:"error"` - Message string `json:"message"` -} diff --git a/certs/api/logging.go b/certs/api/logging.go deleted file mode 100644 index 8567d65..0000000 --- a/certs/api/logging.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -//go:build !test - -package api - -import ( - "context" - "log/slog" - "time" - - "github.com/absmach/magistrala/certs" -) - -var _ certs.Service = (*loggingMiddleware)(nil) - -type loggingMiddleware struct { - logger *slog.Logger - svc certs.Service -} - -// LoggingMiddleware adds logging facilities to the bootstrap service. -func LoggingMiddleware(svc certs.Service, logger *slog.Logger) certs.Service { - return &loggingMiddleware{logger, svc} -} - -// IssueCert logs the issue_cert request. It logs the ttl, thing ID and the time it took to complete the request. -// If the request fails, it logs the error. -func (lm *loggingMiddleware) IssueCert(ctx context.Context, token, thingID, ttl string) (c certs.Cert, err error) { - defer func(begin time.Time) { - args := []any{ - slog.String("duration", time.Since(begin).String()), - slog.String("thing_id", thingID), - slog.String("ttl", ttl), - } - if err != nil { - args = append(args, slog.Any("error", err)) - lm.logger.Warn("Issue certificate failed", args...) - return - } - lm.logger.Info("Issue certificate completed successfully", args...) - }(time.Now()) - - return lm.svc.IssueCert(ctx, token, thingID, ttl) -} - -// ListCerts logs the list_certs request. It logs the thing ID and the time it took to complete the request. -func (lm *loggingMiddleware) ListCerts(ctx context.Context, token, thingID string, offset, limit uint64) (cp certs.Page, err error) { - defer func(begin time.Time) { - args := []any{ - slog.String("duration", time.Since(begin).String()), - slog.String("thing_id", thingID), - slog.Group("page", - slog.Uint64("offset", cp.Offset), - slog.Uint64("limit", cp.Limit), - slog.Uint64("total", cp.Total), - ), - } - if err != nil { - args = append(args, slog.Any("error", err)) - lm.logger.Warn("List certificates failed", args...) - return - } - lm.logger.Info("List certificates completed successfully", args...) - }(time.Now()) - - return lm.svc.ListCerts(ctx, token, thingID, offset, limit) -} - -// ListSerials logs the list_serials request. It logs the thing ID and the time it took to complete the request. -// If the request fails, it logs the error. -func (lm *loggingMiddleware) ListSerials(ctx context.Context, token, thingID string, offset, limit uint64) (cp certs.Page, err error) { - defer func(begin time.Time) { - args := []any{ - slog.String("duration", time.Since(begin).String()), - slog.String("thing_id", thingID), - slog.Group("page", - slog.Uint64("offset", cp.Offset), - slog.Uint64("limit", cp.Limit), - slog.Uint64("total", cp.Total), - ), - } - if err != nil { - args = append(args, slog.Any("error", err)) - lm.logger.Warn("List certifcates serials failed", args...) - return - } - lm.logger.Info("List certificates serials completed successfully", args...) - }(time.Now()) - - return lm.svc.ListSerials(ctx, token, thingID, offset, limit) -} - -// ViewCert logs the view_cert request. It logs the serial ID and the time it took to complete the request. -// If the request fails, it logs the error. -func (lm *loggingMiddleware) ViewCert(ctx context.Context, token, serialID string) (c certs.Cert, err error) { - defer func(begin time.Time) { - args := []any{ - slog.String("duration", time.Since(begin).String()), - slog.String("serial_id", serialID), - } - if err != nil { - args = append(args, slog.Any("error", err)) - lm.logger.Warn("View certificate failed", args...) - return - } - lm.logger.Info("View certificate completed successfully", args...) - }(time.Now()) - - return lm.svc.ViewCert(ctx, token, serialID) -} - -// RevokeCert logs the revoke_cert request. It logs the thing ID and the time it took to complete the request. -// If the request fails, it logs the error. -func (lm *loggingMiddleware) RevokeCert(ctx context.Context, token, thingID string) (c certs.Revoke, err error) { - defer func(begin time.Time) { - args := []any{ - slog.String("duration", time.Since(begin).String()), - slog.String("thing_id", thingID), - } - if err != nil { - args = append(args, slog.Any("error", err)) - lm.logger.Warn("Revoke certificate failed", args...) - return - } - lm.logger.Info("Revoke certificate completed successfully", args...) - }(time.Now()) - - return lm.svc.RevokeCert(ctx, token, thingID) -} diff --git a/certs/api/metrics.go b/certs/api/metrics.go deleted file mode 100644 index e1ab83a..0000000 --- a/certs/api/metrics.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -//go:build !test - -package api - -import ( - "context" - "time" - - "github.com/absmach/magistrala/certs" - "github.com/go-kit/kit/metrics" -) - -var _ certs.Service = (*metricsMiddleware)(nil) - -type metricsMiddleware struct { - counter metrics.Counter - latency metrics.Histogram - svc certs.Service -} - -// MetricsMiddleware instruments core service by tracking request count and latency. -func MetricsMiddleware(svc certs.Service, counter metrics.Counter, latency metrics.Histogram) certs.Service { - return &metricsMiddleware{ - counter: counter, - latency: latency, - svc: svc, - } -} - -// IssueCert instruments IssueCert method with metrics. -func (ms *metricsMiddleware) IssueCert(ctx context.Context, token, thingID, ttl string) (certs.Cert, error) { - defer func(begin time.Time) { - ms.counter.With("method", "issue_cert").Add(1) - ms.latency.With("method", "issue_cert").Observe(time.Since(begin).Seconds()) - }(time.Now()) - - return ms.svc.IssueCert(ctx, token, thingID, ttl) -} - -// ListCerts instruments ListCerts method with metrics. -func (ms *metricsMiddleware) ListCerts(ctx context.Context, token, thingID string, offset, limit uint64) (certs.Page, error) { - defer func(begin time.Time) { - ms.counter.With("method", "list_certs").Add(1) - ms.latency.With("method", "list_certs").Observe(time.Since(begin).Seconds()) - }(time.Now()) - - return ms.svc.ListCerts(ctx, token, thingID, offset, limit) -} - -// ListSerials instruments ListSerials method with metrics. -func (ms *metricsMiddleware) ListSerials(ctx context.Context, token, thingID string, offset, limit uint64) (certs.Page, error) { - defer func(begin time.Time) { - ms.counter.With("method", "list_serials").Add(1) - ms.latency.With("method", "list_serials").Observe(time.Since(begin).Seconds()) - }(time.Now()) - - return ms.svc.ListSerials(ctx, token, thingID, offset, limit) -} - -// ViewCert instruments ViewCert method with metrics. -func (ms *metricsMiddleware) ViewCert(ctx context.Context, token, serialID string) (certs.Cert, error) { - defer func(begin time.Time) { - ms.counter.With("method", "view_cert").Add(1) - ms.latency.With("method", "view_cert").Observe(time.Since(begin).Seconds()) - }(time.Now()) - - return ms.svc.ViewCert(ctx, token, serialID) -} - -// RevokeCert instruments RevokeCert method with metrics. -func (ms *metricsMiddleware) RevokeCert(ctx context.Context, token, thingID string) (certs.Revoke, error) { - defer func(begin time.Time) { - ms.counter.With("method", "revoke_cert").Add(1) - ms.latency.With("method", "revoke_cert").Observe(time.Since(begin).Seconds()) - }(time.Now()) - - return ms.svc.RevokeCert(ctx, token, thingID) -} diff --git a/certs/api/requests.go b/certs/api/requests.go deleted file mode 100644 index 78ac21d..0000000 --- a/certs/api/requests.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package api - -import ( - "time" - - "github.com/absmach/magistrala/pkg/apiutil" -) - -const maxLimitSize = 100 - -type addCertsReq struct { - token string - ThingID string `json:"thing_id"` - TTL string `json:"ttl"` -} - -func (req addCertsReq) validate() error { - if req.token == "" { - return apiutil.ErrBearerToken - } - - if req.ThingID == "" { - return apiutil.ErrMissingID - } - - if req.TTL == "" { - return apiutil.ErrMissingCertData - } - - if _, err := time.ParseDuration(req.TTL); err != nil { - return apiutil.ErrInvalidCertData - } - - return nil -} - -type listReq struct { - thingID string - token string - offset uint64 - limit uint64 -} - -func (req *listReq) validate() error { - if req.token == "" { - return apiutil.ErrBearerToken - } - if req.limit > maxLimitSize { - return apiutil.ErrLimitSize - } - return nil -} - -type viewReq struct { - serialID string - token string -} - -func (req *viewReq) validate() error { - if req.token == "" { - return apiutil.ErrBearerToken - } - if req.serialID == "" { - return apiutil.ErrMissingID - } - - return nil -} - -type revokeReq struct { - token string - certID string -} - -func (req *revokeReq) validate() error { - if req.token == "" { - return apiutil.ErrBearerToken - } - - if req.certID == "" { - return apiutil.ErrMissingID - } - - return nil -} diff --git a/certs/api/responses.go b/certs/api/responses.go deleted file mode 100644 index ce19064..0000000 --- a/certs/api/responses.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package api - -import ( - "net/http" - "time" -) - -type pageRes struct { - Total uint64 `json:"total"` - Offset uint64 `json:"offset"` - Limit uint64 `json:"limit"` -} - -type certsPageRes struct { - pageRes - Certs []certsRes `json:"certs"` -} - -type certsRes struct { - ThingID string `json:"thing_id"` - ClientCert string `json:"client_cert"` - ClientKey string `json:"client_key"` - CertSerial string `json:"cert_serial"` - Expiration time.Time `json:"expiration"` - created bool -} - -type revokeCertsRes struct { - RevocationTime time.Time `json:"revocation_time"` -} - -func (res certsPageRes) Code() int { - return http.StatusOK -} - -func (res certsPageRes) Headers() map[string]string { - return map[string]string{} -} - -func (res certsPageRes) Empty() bool { - return false -} - -func (res certsRes) Code() int { - if res.created { - return http.StatusCreated - } - - return http.StatusOK -} - -func (res certsRes) Headers() map[string]string { - return map[string]string{} -} - -func (res certsRes) Empty() bool { - return false -} - -func (res revokeCertsRes) Code() int { - return http.StatusOK -} - -func (res revokeCertsRes) Headers() map[string]string { - return map[string]string{} -} - -func (res revokeCertsRes) Empty() bool { - return false -} diff --git a/certs/api/transport.go b/certs/api/transport.go deleted file mode 100644 index c5cc717..0000000 --- a/certs/api/transport.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package api - -import ( - "context" - "encoding/json" - "log/slog" - "net/http" - - "github.com/absmach/magistrala" - "github.com/absmach/magistrala/certs" - "github.com/absmach/magistrala/pkg/apiutil" - "github.com/absmach/magistrala/pkg/errors" - "github.com/absmach/mg-contrib/pkg/api" - "github.com/go-chi/chi/v5" - kithttp "github.com/go-kit/kit/transport/http" - "github.com/prometheus/client_golang/prometheus/promhttp" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" -) - -const ( - contentType = "application/json" - offsetKey = "offset" - limitKey = "limit" - defOffset = 0 - defLimit = 10 -) - -// MakeHandler returns a HTTP handler for API endpoints. -func MakeHandler(svc certs.Service, logger *slog.Logger, instanceID string) http.Handler { - opts := []kithttp.ServerOption{ - kithttp.ServerErrorEncoder(apiutil.LoggingErrorEncoder(logger, api.EncodeError)), - } - - r := chi.NewRouter() - - r.Route("/certs", func(r chi.Router) { - r.Post("/", otelhttp.NewHandler(kithttp.NewServer( - issueCert(svc), - decodeCerts, - api.EncodeResponse, - opts..., - ), "issue").ServeHTTP) - r.Get("/{certID}", otelhttp.NewHandler(kithttp.NewServer( - viewCert(svc), - decodeViewCert, - api.EncodeResponse, - opts..., - ), "view").ServeHTTP) - r.Delete("/{certID}", otelhttp.NewHandler(kithttp.NewServer( - revokeCert(svc), - decodeRevokeCerts, - api.EncodeResponse, - opts..., - ), "revoke").ServeHTTP) - }) - r.Get("/serials/{thingID}", otelhttp.NewHandler(kithttp.NewServer( - listSerials(svc), - decodeListCerts, - api.EncodeResponse, - opts..., - ), "list_serials").ServeHTTP) - - r.Handle("/metrics", promhttp.Handler()) - r.Get("/health", magistrala.Health("certs", instanceID)) - - return r -} - -func decodeListCerts(_ context.Context, r *http.Request) (interface{}, error) { - l, err := apiutil.ReadNumQuery[uint64](r, limitKey, defLimit) - if err != nil { - return nil, errors.Wrap(apiutil.ErrValidation, err) - } - o, err := apiutil.ReadNumQuery[uint64](r, offsetKey, defOffset) - if err != nil { - return nil, errors.Wrap(apiutil.ErrValidation, err) - } - - req := listReq{ - token: apiutil.ExtractBearerToken(r), - thingID: chi.URLParam(r, "thingID"), - limit: l, - offset: o, - } - return req, nil -} - -func decodeViewCert(_ context.Context, r *http.Request) (interface{}, error) { - req := viewReq{ - token: apiutil.ExtractBearerToken(r), - serialID: chi.URLParam(r, "certID"), - } - - return req, nil -} - -func decodeCerts(_ context.Context, r *http.Request) (interface{}, error) { - if r.Header.Get("Content-Type") != contentType { - return nil, errors.Wrap(apiutil.ErrValidation, apiutil.ErrUnsupportedContentType) - } - - req := addCertsReq{token: apiutil.ExtractBearerToken(r)} - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - return nil, errors.Wrap(apiutil.ErrValidation, err) - } - - return req, nil -} - -func decodeRevokeCerts(_ context.Context, r *http.Request) (interface{}, error) { - req := revokeReq{ - token: apiutil.ExtractBearerToken(r), - certID: chi.URLParam(r, "certID"), - } - - return req, nil -} diff --git a/certs/certs.go b/certs/certs.go deleted file mode 100644 index 8bfaa60..0000000 --- a/certs/certs.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package certs - -import ( - "context" - "crypto/tls" - "crypto/x509" - "encoding/pem" - "os" - - "github.com/absmach/magistrala/pkg/errors" -) - -// ConfigsPage contains page related metadata as well as list. -type Page struct { - Total uint64 - Offset uint64 - Limit uint64 - Certs []Cert -} - -var ErrMissingCerts = errors.New("CA path or CA key path not set") - -// Repository specifies a Config persistence API. -// -//go:generate mockery --name Repository --output=./mocks --filename certs.go --quiet --note "Copyright (c) Abstract Machines" -type Repository interface { - // Save saves cert for thing into database - Save(ctx context.Context, cert Cert) (string, error) - - // RetrieveAll retrieve issued certificates for given owner ID - RetrieveAll(ctx context.Context, ownerID string, offset, limit uint64) (Page, error) - - // Remove removes certificate from DB for a given thing ID - Remove(ctx context.Context, ownerID, thingID string) error - - // RetrieveByThing retrieves issued certificates for a given thing ID - RetrieveByThing(ctx context.Context, ownerID, thingID string, offset, limit uint64) (Page, error) - - // RetrieveBySerial retrieves a certificate for a given serial ID - RetrieveBySerial(ctx context.Context, ownerID, serialID string) (Cert, error) -} - -func LoadCertificates(caPath, caKeyPath string) (tls.Certificate, *x509.Certificate, error) { - if caPath == "" || caKeyPath == "" { - return tls.Certificate{}, &x509.Certificate{}, ErrMissingCerts - } - - _, err := os.Stat(caPath) - if os.IsNotExist(err) || os.IsPermission(err) { - return tls.Certificate{}, &x509.Certificate{}, err - } - - _, err = os.Stat(caKeyPath) - if os.IsNotExist(err) || os.IsPermission(err) { - return tls.Certificate{}, &x509.Certificate{}, err - } - - tlsCert, err := tls.LoadX509KeyPair(caPath, caKeyPath) - if err != nil { - return tlsCert, &x509.Certificate{}, err - } - - b, err := os.ReadFile(caPath) - if err != nil { - return tlsCert, &x509.Certificate{}, err - } - - caCert, err := ReadCert(b) - if err != nil { - return tlsCert, &x509.Certificate{}, err - } - - return tlsCert, caCert, nil -} - -func ReadCert(b []byte) (*x509.Certificate, error) { - block, _ := pem.Decode(b) - if block == nil { - return nil, errors.New("failed to decode PEM data") - } - - return x509.ParseCertificate(block.Bytes) -} diff --git a/certs/certs_test.go b/certs/certs_test.go deleted file mode 100644 index 3ee7dc7..0000000 --- a/certs/certs_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package certs_test - -import ( - "fmt" - "testing" - - "github.com/absmach/magistrala/certs" - "github.com/absmach/magistrala/pkg/errors" - "github.com/stretchr/testify/assert" -) - -func TestLoadCertificates(t *testing.T) { - cases := []struct { - desc string - caPath string - caKeyPath string - err error - }{ - { - desc: "load valid tls certificate and valid key", - caPath: "../docker/ssl/certs/ca.crt", - caKeyPath: "../docker/ssl/certs/ca.key", - err: nil, - }, - { - desc: "load valid tls certificate and missing key", - caPath: "../docker/ssl/certs/ca.crt", - caKeyPath: "", - err: certs.ErrMissingCerts, - }, - { - desc: "load missing tls certificate and valid key", - caPath: "", - caKeyPath: "../docker/ssl/certs/ca.key", - err: certs.ErrMissingCerts, - }, - { - desc: "load empty tls certificate and empty key", - caPath: "", - caKeyPath: "", - err: certs.ErrMissingCerts, - }, - { - desc: "load valid tls certificate and invalid key", - caPath: "../docker/ssl/certs/ca.crt", - caKeyPath: "certs.go", - err: errors.New("tls: failed to find any PEM data in key input"), - }, - { - desc: "load invalid tls certificate and valid key", - caPath: "certs.go", - caKeyPath: "../docker/ssl/certs/ca.key", - err: errors.New("tls: failed to find any PEM data in certificate input"), - }, - { - desc: "load invalid tls certificate and invalid key", - caPath: "certs.go", - caKeyPath: "certs.go", - err: errors.New("tls: failed to find any PEM data in certificate input"), - }, - - { - desc: "load valid tls certificate and non-existing key", - caPath: "../docker/ssl/certs/ca.crt", - caKeyPath: "ca.key", - err: errors.New("stat ca.key: no such file or directory"), - }, - { - desc: "load non-existing tls certificate and valid key", - caPath: "ca.crt", - caKeyPath: "../docker/ssl/certs/ca.key", - err: errors.New("stat ca.crt: no such file or directory"), - }, - { - desc: "load non-existing tls certificate and non-existing key", - caPath: "ca.crt", - caKeyPath: "ca.key", - err: errors.New("stat ca.crt: no such file or directory"), - }, - } - - for _, tc := range cases { - tlsCert, caCert, err := certs.LoadCertificates(tc.caPath, tc.caKeyPath) - assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err)) - if err == nil { - assert.NotNil(t, tlsCert) - assert.NotNil(t, caCert) - } - } -} diff --git a/certs/doc.go b/certs/doc.go deleted file mode 100644 index 24a1987..0000000 --- a/certs/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package certs contains the domain concept definitions needed to support -// Magistrala certs service functionality. -package certs diff --git a/certs/mocks/certs.go b/certs/mocks/certs.go deleted file mode 100644 index ea918bb..0000000 --- a/certs/mocks/certs.go +++ /dev/null @@ -1,162 +0,0 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. - -// Copyright (c) Abstract Machines - -package mocks - -import ( - context "context" - - certs "github.com/absmach/magistrala/certs" - - mock "github.com/stretchr/testify/mock" -) - -// Repository is an autogenerated mock type for the Repository type -type Repository struct { - mock.Mock -} - -// Remove provides a mock function with given fields: ctx, ownerID, thingID -func (_m *Repository) Remove(ctx context.Context, ownerID string, thingID string) error { - ret := _m.Called(ctx, ownerID, thingID) - - if len(ret) == 0 { - panic("no return value specified for Remove") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { - r0 = rf(ctx, ownerID, thingID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// RetrieveAll provides a mock function with given fields: ctx, ownerID, offset, limit -func (_m *Repository) RetrieveAll(ctx context.Context, ownerID string, offset uint64, limit uint64) (certs.Page, error) { - ret := _m.Called(ctx, ownerID, offset, limit) - - if len(ret) == 0 { - panic("no return value specified for RetrieveAll") - } - - var r0 certs.Page - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64) (certs.Page, error)); ok { - return rf(ctx, ownerID, offset, limit) - } - if rf, ok := ret.Get(0).(func(context.Context, string, uint64, uint64) certs.Page); ok { - r0 = rf(ctx, ownerID, offset, limit) - } else { - r0 = ret.Get(0).(certs.Page) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, uint64, uint64) error); ok { - r1 = rf(ctx, ownerID, offset, limit) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// RetrieveBySerial provides a mock function with given fields: ctx, ownerID, serialID -func (_m *Repository) RetrieveBySerial(ctx context.Context, ownerID string, serialID string) (certs.Cert, error) { - ret := _m.Called(ctx, ownerID, serialID) - - if len(ret) == 0 { - panic("no return value specified for RetrieveBySerial") - } - - var r0 certs.Cert - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) (certs.Cert, error)); ok { - return rf(ctx, ownerID, serialID) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string) certs.Cert); ok { - r0 = rf(ctx, ownerID, serialID) - } else { - r0 = ret.Get(0).(certs.Cert) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { - r1 = rf(ctx, ownerID, serialID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// RetrieveByThing provides a mock function with given fields: ctx, ownerID, thingID, offset, limit -func (_m *Repository) RetrieveByThing(ctx context.Context, ownerID string, thingID string, offset uint64, limit uint64) (certs.Page, error) { - ret := _m.Called(ctx, ownerID, thingID, offset, limit) - - if len(ret) == 0 { - panic("no return value specified for RetrieveByThing") - } - - var r0 certs.Page - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, uint64, uint64) (certs.Page, error)); ok { - return rf(ctx, ownerID, thingID, offset, limit) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string, uint64, uint64) certs.Page); ok { - r0 = rf(ctx, ownerID, thingID, offset, limit) - } else { - r0 = ret.Get(0).(certs.Page) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string, uint64, uint64) error); ok { - r1 = rf(ctx, ownerID, thingID, offset, limit) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Save provides a mock function with given fields: ctx, cert -func (_m *Repository) Save(ctx context.Context, cert certs.Cert) (string, error) { - ret := _m.Called(ctx, cert) - - if len(ret) == 0 { - panic("no return value specified for Save") - } - - var r0 string - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, certs.Cert) (string, error)); ok { - return rf(ctx, cert) - } - if rf, ok := ret.Get(0).(func(context.Context, certs.Cert) string); ok { - r0 = rf(ctx, cert) - } else { - r0 = ret.Get(0).(string) - } - - if rf, ok := ret.Get(1).(func(context.Context, certs.Cert) error); ok { - r1 = rf(ctx, cert) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewRepository creates a new instance of Repository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewRepository(t interface { - mock.TestingT - Cleanup(func()) -}) *Repository { - mock := &Repository{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/certs/mocks/pki.go b/certs/mocks/pki.go deleted file mode 100644 index 47ae77b..0000000 --- a/certs/mocks/pki.go +++ /dev/null @@ -1,135 +0,0 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. - -// Copyright (c) Abstract Machines - -package mocks - -import ( - context "context" - - pki "github.com/absmach/magistrala/certs/pki" - mock "github.com/stretchr/testify/mock" - - time "time" -) - -// Agent is an autogenerated mock type for the Agent type -type Agent struct { - mock.Mock -} - -// IssueCert provides a mock function with given fields: cn, ttl -func (_m *Agent) IssueCert(cn string, ttl string) (pki.Cert, error) { - ret := _m.Called(cn, ttl) - - if len(ret) == 0 { - panic("no return value specified for IssueCert") - } - - var r0 pki.Cert - var r1 error - if rf, ok := ret.Get(0).(func(string, string) (pki.Cert, error)); ok { - return rf(cn, ttl) - } - if rf, ok := ret.Get(0).(func(string, string) pki.Cert); ok { - r0 = rf(cn, ttl) - } else { - r0 = ret.Get(0).(pki.Cert) - } - - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(cn, ttl) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// LoginAndRenew provides a mock function with given fields: ctx -func (_m *Agent) LoginAndRenew(ctx context.Context) error { - ret := _m.Called(ctx) - - if len(ret) == 0 { - panic("no return value specified for LoginAndRenew") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context) error); ok { - r0 = rf(ctx) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Read provides a mock function with given fields: serial -func (_m *Agent) Read(serial string) (pki.Cert, error) { - ret := _m.Called(serial) - - if len(ret) == 0 { - panic("no return value specified for Read") - } - - var r0 pki.Cert - var r1 error - if rf, ok := ret.Get(0).(func(string) (pki.Cert, error)); ok { - return rf(serial) - } - if rf, ok := ret.Get(0).(func(string) pki.Cert); ok { - r0 = rf(serial) - } else { - r0 = ret.Get(0).(pki.Cert) - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(serial) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Revoke provides a mock function with given fields: serial -func (_m *Agent) Revoke(serial string) (time.Time, error) { - ret := _m.Called(serial) - - if len(ret) == 0 { - panic("no return value specified for Revoke") - } - - var r0 time.Time - var r1 error - if rf, ok := ret.Get(0).(func(string) (time.Time, error)); ok { - return rf(serial) - } - if rf, ok := ret.Get(0).(func(string) time.Time); ok { - r0 = rf(serial) - } else { - r0 = ret.Get(0).(time.Time) - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(serial) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewAgent creates a new instance of Agent. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewAgent(t interface { - mock.TestingT - Cleanup(func()) -}) *Agent { - mock := &Agent{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/certs/mocks/service.go b/certs/mocks/service.go deleted file mode 100644 index 6bc257d..0000000 --- a/certs/mocks/service.go +++ /dev/null @@ -1,172 +0,0 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. - -// Copyright (c) Abstract Machines - -package mocks - -import ( - context "context" - - certs "github.com/absmach/magistrala/certs" - - mock "github.com/stretchr/testify/mock" -) - -// Service is an autogenerated mock type for the Service type -type Service struct { - mock.Mock -} - -// IssueCert provides a mock function with given fields: ctx, token, thingID, ttl -func (_m *Service) IssueCert(ctx context.Context, token string, thingID string, ttl string) (certs.Cert, error) { - ret := _m.Called(ctx, token, thingID, ttl) - - if len(ret) == 0 { - panic("no return value specified for IssueCert") - } - - var r0 certs.Cert - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, string) (certs.Cert, error)); ok { - return rf(ctx, token, thingID, ttl) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string, string) certs.Cert); ok { - r0 = rf(ctx, token, thingID, ttl) - } else { - r0 = ret.Get(0).(certs.Cert) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string, string) error); ok { - r1 = rf(ctx, token, thingID, ttl) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ListCerts provides a mock function with given fields: ctx, token, thingID, offset, limit -func (_m *Service) ListCerts(ctx context.Context, token string, thingID string, offset uint64, limit uint64) (certs.Page, error) { - ret := _m.Called(ctx, token, thingID, offset, limit) - - if len(ret) == 0 { - panic("no return value specified for ListCerts") - } - - var r0 certs.Page - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, uint64, uint64) (certs.Page, error)); ok { - return rf(ctx, token, thingID, offset, limit) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string, uint64, uint64) certs.Page); ok { - r0 = rf(ctx, token, thingID, offset, limit) - } else { - r0 = ret.Get(0).(certs.Page) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string, uint64, uint64) error); ok { - r1 = rf(ctx, token, thingID, offset, limit) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ListSerials provides a mock function with given fields: ctx, token, thingID, offset, limit -func (_m *Service) ListSerials(ctx context.Context, token string, thingID string, offset uint64, limit uint64) (certs.Page, error) { - ret := _m.Called(ctx, token, thingID, offset, limit) - - if len(ret) == 0 { - panic("no return value specified for ListSerials") - } - - var r0 certs.Page - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string, uint64, uint64) (certs.Page, error)); ok { - return rf(ctx, token, thingID, offset, limit) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string, uint64, uint64) certs.Page); ok { - r0 = rf(ctx, token, thingID, offset, limit) - } else { - r0 = ret.Get(0).(certs.Page) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string, uint64, uint64) error); ok { - r1 = rf(ctx, token, thingID, offset, limit) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// RevokeCert provides a mock function with given fields: ctx, token, serialID -func (_m *Service) RevokeCert(ctx context.Context, token string, serialID string) (certs.Revoke, error) { - ret := _m.Called(ctx, token, serialID) - - if len(ret) == 0 { - panic("no return value specified for RevokeCert") - } - - var r0 certs.Revoke - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) (certs.Revoke, error)); ok { - return rf(ctx, token, serialID) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string) certs.Revoke); ok { - r0 = rf(ctx, token, serialID) - } else { - r0 = ret.Get(0).(certs.Revoke) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { - r1 = rf(ctx, token, serialID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ViewCert provides a mock function with given fields: ctx, token, serialID -func (_m *Service) ViewCert(ctx context.Context, token string, serialID string) (certs.Cert, error) { - ret := _m.Called(ctx, token, serialID) - - if len(ret) == 0 { - panic("no return value specified for ViewCert") - } - - var r0 certs.Cert - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) (certs.Cert, error)); ok { - return rf(ctx, token, serialID) - } - if rf, ok := ret.Get(0).(func(context.Context, string, string) certs.Cert); ok { - r0 = rf(ctx, token, serialID) - } else { - r0 = ret.Get(0).(certs.Cert) - } - - if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { - r1 = rf(ctx, token, serialID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewService creates a new instance of Service. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewService(t interface { - mock.TestingT - Cleanup(func()) -}) *Service { - mock := &Service{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/certs/pki/doc.go b/certs/pki/doc.go deleted file mode 100644 index cbd2d97..0000000 --- a/certs/pki/doc.go +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package pki contains the domain concept definitions needed to -// support Magistrala Certs service functionality. -// It provides the abstraction of the PKI (Public Key Infrastructure) -// Valut service, which is used to issue and revoke certificates. -package pki diff --git a/certs/pki/vault.go b/certs/pki/vault.go deleted file mode 100644 index 69bb8bb..0000000 --- a/certs/pki/vault.go +++ /dev/null @@ -1,271 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package pki wraps vault client -package pki - -import ( - "context" - "encoding/json" - "fmt" - "log/slog" - "time" - - "github.com/absmach/magistrala/pkg/errors" - "github.com/hashicorp/vault/api" - "github.com/hashicorp/vault/api/auth/approle" - "github.com/mitchellh/mapstructure" -) - -const ( - issue = "issue" - cert = "cert" - revoke = "revoke" -) - -var ( - errFailedCertDecoding = errors.New("failed to decode response from vault service") - errFailedToLogin = errors.New("failed to login to Vault") - errFailedAppRole = errors.New("failed to create vault new app role") - errNoAuthInfo = errors.New("no auth information from Vault") - errNonRenewal = errors.New("token is not configured to be renewable") - errRenewWatcher = errors.New("unable to initialize new lifetime watcher for renewing auth token") - errFailedRenew = errors.New("failed to renew token") - errCouldNotRenew = errors.New("token can no longer be renewed") -) - -type Cert struct { - ClientCert string `json:"client_cert" mapstructure:"certificate"` - IssuingCA string `json:"issuing_ca" mapstructure:"issuing_ca"` - CAChain []string `json:"ca_chain" mapstructure:"ca_chain"` - ClientKey string `json:"client_key" mapstructure:"private_key"` - PrivateKeyType string `json:"private_key_type" mapstructure:"private_key_type"` - Serial string `json:"serial" mapstructure:"serial_number"` - Expire int64 `json:"expire" mapstructure:"expiration"` -} - -// Agent represents the Vault PKI interface. -// -//go:generate mockery --name Agent --output=../mocks --filename pki.go --quiet --note "Copyright (c) Abstract Machines" -type Agent interface { - // IssueCert issues certificate on PKI - IssueCert(cn, ttl string) (Cert, error) - - // Read retrieves certificate from PKI - Read(serial string) (Cert, error) - - // Revoke revokes certificate from PKI - Revoke(serial string) (time.Time, error) - - // Login to PKI and renews token - LoginAndRenew(ctx context.Context) error -} - -type pkiAgent struct { - appRole string - appSecret string - namespace string - path string - role string - host string - issueURL string - readURL string - revokeURL string - client *api.Client - secret *api.Secret - logger *slog.Logger -} - -type certReq struct { - CommonName string `json:"common_name"` - TTL string `json:"ttl"` -} - -type certRevokeReq struct { - SerialNumber string `json:"serial_number"` -} - -// NewVaultClient instantiates a Vault client. -func NewVaultClient(appRole, appSecret, host, namespace, path, role string, logger *slog.Logger) (Agent, error) { - conf := api.DefaultConfig() - conf.Address = host - - client, err := api.NewClient(conf) - if err != nil { - return nil, err - } - if namespace != "" { - client.SetNamespace(namespace) - } - - p := pkiAgent{ - appRole: appRole, - appSecret: appSecret, - host: host, - namespace: namespace, - role: role, - path: path, - client: client, - logger: logger, - issueURL: "/" + path + "/" + issue + "/" + role, - readURL: "/" + path + "/" + cert + "/", - revokeURL: "/" + path + "/" + revoke, - } - return &p, nil -} - -func (p *pkiAgent) IssueCert(cn, ttl string) (Cert, error) { - cReq := certReq{ - CommonName: cn, - TTL: ttl, - } - - var certIssueReq map[string]interface{} - data, err := json.Marshal(cReq) - if err != nil { - return Cert{}, err - } - if err := json.Unmarshal(data, &certIssueReq); err != nil { - return Cert{}, nil - } - - s, err := p.client.Logical().Write(p.issueURL, certIssueReq) - if err != nil { - return Cert{}, err - } - - cert := Cert{} - if err = mapstructure.Decode(s.Data, &cert); err != nil { - return Cert{}, errors.Wrap(errFailedCertDecoding, err) - } - - return cert, nil -} - -func (p *pkiAgent) Read(serial string) (Cert, error) { - s, err := p.client.Logical().Read(p.readURL + serial) - if err != nil { - return Cert{}, err - } - cert := Cert{} - if err = mapstructure.Decode(s.Data, &cert); err != nil { - return Cert{}, errors.Wrap(errFailedCertDecoding, err) - } - return cert, nil -} - -func (p *pkiAgent) Revoke(serial string) (time.Time, error) { - cReq := certRevokeReq{ - SerialNumber: serial, - } - - var certRevokeReq map[string]interface{} - data, err := json.Marshal(cReq) - if err != nil { - return time.Time{}, err - } - if err := json.Unmarshal(data, &certRevokeReq); err != nil { - return time.Time{}, nil - } - - s, err := p.client.Logical().Write(p.revokeURL, certRevokeReq) - if err != nil { - return time.Time{}, err - } - - // Vault will return a response without errors but with a warning if the certificate is expired. - // The response will not have "revocation_time" in such cases. - if revokeTime, ok := s.Data["revocation_time"]; ok { - switch v := revokeTime.(type) { - case json.Number: - rev, err := v.Float64() - if err != nil { - return time.Time{}, err - } - return time.Unix(0, int64(rev)*int64(time.Second)), nil - - default: - return time.Time{}, fmt.Errorf("unsupported type for revocation_time: %T", v) - } - } - - return time.Time{}, nil -} - -func (p *pkiAgent) LoginAndRenew(ctx context.Context) error { - for { - select { - case <-ctx.Done(): - p.logger.Info("pki login and renew function stopping") - return nil - default: - err := p.login(ctx) - if err != nil { - p.logger.Info("unable to authenticate to Vault", slog.Any("error", err)) - time.Sleep(5 * time.Second) - break - } - tokenErr := p.manageTokenLifecycle() - if tokenErr != nil { - p.logger.Info("unable to start managing token lifecycle", slog.Any("error", tokenErr)) - time.Sleep(5 * time.Second) - } - } - } -} - -func (p *pkiAgent) login(ctx context.Context) error { - secretID := &approle.SecretID{FromString: p.appSecret} - - authMethod, err := approle.NewAppRoleAuth( - p.appRole, - secretID, - ) - if err != nil { - return errors.Wrap(errFailedAppRole, err) - } - if p.namespace != "" { - p.client.SetNamespace(p.namespace) - } - secret, err := p.client.Auth().Login(ctx, authMethod) - if err != nil { - return errors.Wrap(errFailedToLogin, err) - } - if secret == nil { - return errNoAuthInfo - } - p.secret = secret - return nil -} - -func (p *pkiAgent) manageTokenLifecycle() error { - renew := p.secret.Auth.Renewable - if !renew { - return errNonRenewal - } - - watcher, err := p.client.NewLifetimeWatcher(&api.LifetimeWatcherInput{ - Secret: p.secret, - Increment: 3600, // Requesting token for 3600s = 1h, If this is more than token_max_ttl, then response token will have token_max_ttl - }) - if err != nil { - return errors.Wrap(errRenewWatcher, err) - } - - go watcher.Start() - defer watcher.Stop() - - for { - select { - case err := <-watcher.DoneCh(): - if err != nil { - return errors.Wrap(errFailedRenew, err) - } - // This occurs once the token has reached max TTL or if token is disabled for renewal. - return errCouldNotRenew - - case renewal := <-watcher.RenewCh(): - p.logger.Info("Successfully renewed token", slog.Any("renewed_at", renewal.RenewedAt)) - } - } -} diff --git a/certs/postgres/certs.go b/certs/postgres/certs.go deleted file mode 100644 index 8f581ea..0000000 --- a/certs/postgres/certs.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package postgres - -import ( - "context" - "database/sql" - "fmt" - "log/slog" - "time" - - "github.com/absmach/magistrala/certs" - "github.com/absmach/magistrala/pkg/errors" - repoerr "github.com/absmach/magistrala/pkg/errors/repository" - "github.com/absmach/magistrala/pkg/postgres" - "github.com/jackc/pgerrcode" - "github.com/jackc/pgx/v5/pgconn" - "github.com/jmoiron/sqlx" -) - -var _ certs.Repository = (*certsRepository)(nil) - -// Cert holds info on expiration date for specific cert issued for specific Thing. -type Cert struct { - ThingID string - Serial string - Expire time.Time -} - -type certsRepository struct { - db postgres.Database - log *slog.Logger -} - -// NewRepository instantiates a PostgreSQL implementation of certs -// repository. -func NewRepository(db postgres.Database, log *slog.Logger) certs.Repository { - return &certsRepository{db: db, log: log} -} - -func (cr certsRepository) RetrieveAll(ctx context.Context, ownerID string, offset, limit uint64) (certs.Page, error) { - q := `SELECT thing_id, owner_id, serial, expire FROM certs WHERE owner_id = $1 ORDER BY expire LIMIT $2 OFFSET $3;` - rows, err := cr.db.QueryContext(ctx, q, ownerID, limit, offset) - if err != nil { - cr.log.Error(fmt.Sprintf("Failed to retrieve configs due to %s", err)) - return certs.Page{}, err - } - defer rows.Close() - - certificates := []certs.Cert{} - for rows.Next() { - c := certs.Cert{} - if err := rows.Scan(&c.ThingID, &c.OwnerID, &c.Serial, &c.Expire); err != nil { - cr.log.Error(fmt.Sprintf("Failed to read retrieved config due to %s", err)) - return certs.Page{}, err - } - certificates = append(certificates, c) - } - - q = `SELECT COUNT(*) FROM certs WHERE owner_id = $1` - var total uint64 - if err := cr.db.QueryRowxContext(ctx, q, ownerID).Scan(&total); err != nil { - cr.log.Error(fmt.Sprintf("Failed to count certs due to %s", err)) - return certs.Page{}, err - } - - return certs.Page{ - Total: total, - Limit: limit, - Offset: offset, - Certs: certificates, - }, nil -} - -func (cr certsRepository) Save(ctx context.Context, cert certs.Cert) (string, error) { - q := `INSERT INTO certs (thing_id, owner_id, serial, expire) VALUES (:thing_id, :owner_id, :serial, :expire)` - - tx, err := cr.db.BeginTxx(ctx, nil) - if err != nil { - return "", errors.Wrap(repoerr.ErrCreateEntity, err) - } - - dbcrt := toDBCert(cert) - - if _, err := tx.NamedExec(q, dbcrt); err != nil { - e := err - if pgErr, ok := err.(*pgconn.PgError); ok && pgErr.Code == pgerrcode.UniqueViolation { - e = errors.New("error conflict") - } - - cr.rollback("Failed to insert a Cert", tx, err) - - return "", errors.Wrap(repoerr.ErrCreateEntity, e) - } - - if err := tx.Commit(); err != nil { - cr.rollback("Failed to commit Config save", tx, err) - } - - return cert.Serial, nil -} - -func (cr certsRepository) Remove(ctx context.Context, ownerID, serial string) error { - if _, err := cr.RetrieveBySerial(ctx, ownerID, serial); err != nil { - return errors.Wrap(repoerr.ErrRemoveEntity, err) - } - q := `DELETE FROM certs WHERE serial = :serial` - var c certs.Cert - c.Serial = serial - dbcrt := toDBCert(c) - if _, err := cr.db.NamedExecContext(ctx, q, dbcrt); err != nil { - return errors.Wrap(repoerr.ErrRemoveEntity, err) - } - return nil -} - -func (cr certsRepository) RetrieveByThing(ctx context.Context, ownerID, thingID string, offset, limit uint64) (certs.Page, error) { - q := `SELECT thing_id, owner_id, serial, expire FROM certs WHERE owner_id = $1 AND thing_id = $2 ORDER BY expire LIMIT $3 OFFSET $4;` - rows, err := cr.db.QueryContext(ctx, q, ownerID, thingID, limit, offset) - if err != nil { - cr.log.Error(fmt.Sprintf("Failed to retrieve configs due to %s", err)) - return certs.Page{}, err - } - defer rows.Close() - - certificates := []certs.Cert{} - for rows.Next() { - c := certs.Cert{} - if err := rows.Scan(&c.ThingID, &c.OwnerID, &c.Serial, &c.Expire); err != nil { - cr.log.Error(fmt.Sprintf("Failed to read retrieved config due to %s", err)) - return certs.Page{}, err - } - certificates = append(certificates, c) - } - - q = `SELECT COUNT(*) FROM certs WHERE owner_id = $1 AND thing_id = $2` - var total uint64 - if err := cr.db.QueryRowxContext(ctx, q, ownerID, thingID).Scan(&total); err != nil { - cr.log.Error(fmt.Sprintf("Failed to count certs due to %s", err)) - return certs.Page{}, err - } - - return certs.Page{ - Total: total, - Limit: limit, - Offset: offset, - Certs: certificates, - }, nil -} - -func (cr certsRepository) RetrieveBySerial(ctx context.Context, ownerID, serialID string) (certs.Cert, error) { - q := `SELECT thing_id, owner_id, serial, expire FROM certs WHERE owner_id = $1 AND serial = $2` - var dbcrt dbCert - var c certs.Cert - - if err := cr.db.QueryRowxContext(ctx, q, ownerID, serialID).StructScan(&dbcrt); err != nil { - pqErr, ok := err.(*pgconn.PgError) - if err == sql.ErrNoRows || ok && pgerrcode.InvalidTextRepresentation == pqErr.Code { - return c, errors.Wrap(repoerr.ErrNotFound, err) - } - - return c, errors.Wrap(repoerr.ErrViewEntity, err) - } - c = toCert(dbcrt) - - return c, nil -} - -func (cr certsRepository) rollback(content string, tx *sqlx.Tx, err error) { - cr.log.Error(fmt.Sprintf("%s %s", content, err)) - - if err := tx.Rollback(); err != nil { - cr.log.Error(fmt.Sprintf("Failed to rollback due to %s", err)) - } -} - -type dbCert struct { - ThingID string `db:"thing_id"` - Serial string `db:"serial"` - Expire time.Time `db:"expire"` - OwnerID string `db:"owner_id"` -} - -func toDBCert(c certs.Cert) dbCert { - return dbCert{ - ThingID: c.ThingID, - OwnerID: c.OwnerID, - Serial: c.Serial, - Expire: c.Expire, - } -} - -func toCert(cdb dbCert) certs.Cert { - var c certs.Cert - c.OwnerID = cdb.OwnerID - c.ThingID = cdb.ThingID - c.Serial = cdb.Serial - c.Expire = cdb.Expire - return c -} diff --git a/certs/postgres/doc.go b/certs/postgres/doc.go deleted file mode 100644 index 73a6784..0000000 --- a/certs/postgres/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package postgres contains repository implementations using PostgreSQL as -// the underlying database. -package postgres diff --git a/certs/postgres/init.go b/certs/postgres/init.go deleted file mode 100644 index a1f1eda..0000000 --- a/certs/postgres/init.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package postgres - -import migrate "github.com/rubenv/sql-migrate" - -// Migration of Certs service. -func Migration() *migrate.MemoryMigrationSource { - return &migrate.MemoryMigrationSource{ - Migrations: []*migrate.Migration{ - { - Id: "certs_1", - Up: []string{ - `CREATE TABLE IF NOT EXISTS certs ( - thing_id TEXT NOT NULL, - owner_id TEXT NOT NULL, - expire TIMESTAMPTZ NOT NULL, - serial TEXT NOT NULL, - PRIMARY KEY (thing_id, owner_id, serial) - );`, - }, - Down: []string{ - "DROP TABLE IF EXISTS certs;", - }, - }, - }, - } -} diff --git a/certs/postgres/setup_test.go b/certs/postgres/setup_test.go deleted file mode 100644 index 1281e08..0000000 --- a/certs/postgres/setup_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package postgres_test - -import ( - "database/sql" - "fmt" - "log" - "os" - "testing" - - "github.com/absmach/magistrala/certs/postgres" - mglog "github.com/absmach/magistrala/logger" - pgclient "github.com/absmach/magistrala/pkg/postgres" - "github.com/jmoiron/sqlx" - "github.com/ory/dockertest/v3" - "github.com/ory/dockertest/v3/docker" -) - -var ( - testLog, _ = mglog.New(os.Stdout, "info") - db *sqlx.DB -) - -func TestMain(m *testing.M) { - pool, err := dockertest.NewPool("") - if err != nil { - testLog.Error(fmt.Sprintf("Could not connect to docker: %s", err)) - return - } - - container, err := pool.RunWithOptions(&dockertest.RunOptions{ - Repository: "postgres", - Tag: "16.2-alpine", - Env: []string{ - "POSTGRES_USER=test", - "POSTGRES_PASSWORD=test", - "POSTGRES_DB=test", - "listen_addresses = '*'", - }, - }, func(config *docker.HostConfig) { - config.AutoRemove = true - config.RestartPolicy = docker.RestartPolicy{Name: "no"} - }) - if err != nil { - log.Fatalf("Could not start container: %s", err) - } - - port := container.GetPort("5432/tcp") - - if err := pool.Retry(func() error { - url := fmt.Sprintf("host=localhost port=%s user=test dbname=test password=test sslmode=disable", port) - db, err := sql.Open("pgx", url) - if err != nil { - return err - } - return db.Ping() - }); err != nil { - testLog.Error(fmt.Sprintf("Could not connect to docker: %s", err)) - } - - dbConfig := pgclient.Config{ - Host: "localhost", - Port: port, - User: "test", - Pass: "test", - Name: "test", - SSLMode: "disable", - SSLCert: "", - SSLKey: "", - SSLRootCert: "", - } - - if db, err = pgclient.Setup(dbConfig, *postgres.Migration()); err != nil { - testLog.Error(fmt.Sprintf("Could not setup test DB connection: %s", err)) - } - - code := m.Run() - - // Defers will not be run when using os.Exit - db.Close() - if err := pool.Purge(container); err != nil { - testLog.Error(fmt.Sprintf("Could not purge container: %s", err)) - } - - os.Exit(code) -} diff --git a/certs/service.go b/certs/service.go deleted file mode 100644 index 191b328..0000000 --- a/certs/service.go +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package certs - -import ( - "context" - "time" - - "github.com/absmach/magistrala" - "github.com/absmach/magistrala/certs/pki" - "github.com/absmach/magistrala/pkg/errors" - svcerr "github.com/absmach/magistrala/pkg/errors/service" - mgsdk "github.com/absmach/magistrala/pkg/sdk/go" -) - -var ( - // ErrFailedCertCreation failed to create certificate. - ErrFailedCertCreation = errors.New("failed to create client certificate") - - // ErrFailedCertRevocation failed to revoke certificate. - ErrFailedCertRevocation = errors.New("failed to revoke certificate") - - ErrFailedToRemoveCertFromDB = errors.New("failed to remove cert serial from db") - - ErrFailedReadFromPKI = errors.New("failed to read certificate from PKI") -) - -var _ Service = (*certsService)(nil) - -// Service specifies an API that must be fulfilled by the domain service -// implementation, and all of its decorators (e.g. logging & metrics). -// -//go:generate mockery --name Service --output=./mocks --filename service.go --quiet --note "Copyright (c) Abstract Machines" -type Service interface { - // IssueCert issues certificate for given thing id if access is granted with token - IssueCert(ctx context.Context, token, thingID, ttl string) (Cert, error) - - // ListCerts lists certificates issued for a given thing ID - ListCerts(ctx context.Context, token, thingID string, offset, limit uint64) (Page, error) - - // ListSerials lists certificate serial IDs issued for a given thing ID - ListSerials(ctx context.Context, token, thingID string, offset, limit uint64) (Page, error) - - // ViewCert retrieves the certificate issued for a given serial ID - ViewCert(ctx context.Context, token, serialID string) (Cert, error) - - // RevokeCert revokes a certificate for a given serial ID - RevokeCert(ctx context.Context, token, serialID string) (Revoke, error) -} - -type certsService struct { - auth magistrala.AuthServiceClient - certsRepo Repository - sdk mgsdk.SDK - pki pki.Agent -} - -// New returns new Certs service. -func New(auth magistrala.AuthServiceClient, certs Repository, sdk mgsdk.SDK, pkiAgent pki.Agent) Service { - return &certsService{ - certsRepo: certs, - sdk: sdk, - auth: auth, - pki: pkiAgent, - } -} - -// Revoke defines the conditions to revoke a certificate. -type Revoke struct { - RevocationTime time.Time `mapstructure:"revocation_time"` -} - -// Cert defines the certificate paremeters. -type Cert struct { - OwnerID string `json:"owner_id" mapstructure:"owner_id"` - ThingID string `json:"thing_id" mapstructure:"thing_id"` - ClientCert string `json:"client_cert" mapstructure:"certificate"` - IssuingCA string `json:"issuing_ca" mapstructure:"issuing_ca"` - CAChain []string `json:"ca_chain" mapstructure:"ca_chain"` - ClientKey string `json:"client_key" mapstructure:"private_key"` - PrivateKeyType string `json:"private_key_type" mapstructure:"private_key_type"` - Serial string `json:"serial" mapstructure:"serial_number"` - Expire time.Time `json:"expire" mapstructure:"-"` -} - -func (cs *certsService) IssueCert(ctx context.Context, token, thingID, ttl string) (Cert, error) { - owner, err := cs.auth.Identify(ctx, &magistrala.IdentityReq{Token: token}) - if err != nil { - return Cert{}, errors.Wrap(svcerr.ErrAuthentication, err) - } - - thing, err := cs.sdk.Thing(thingID, token) - if err != nil { - return Cert{}, errors.Wrap(ErrFailedCertCreation, err) - } - - cert, err := cs.pki.IssueCert(thing.Credentials.Secret, ttl) - if err != nil { - return Cert{}, errors.Wrap(ErrFailedCertCreation, err) - } - - c := Cert{ - ThingID: thingID, - OwnerID: owner.GetId(), - ClientCert: cert.ClientCert, - IssuingCA: cert.IssuingCA, - CAChain: cert.CAChain, - ClientKey: cert.ClientKey, - PrivateKeyType: cert.PrivateKeyType, - Serial: cert.Serial, - Expire: time.Unix(0, int64(cert.Expire)*int64(time.Second)), - } - - _, err = cs.certsRepo.Save(ctx, c) - return c, err -} - -func (cs *certsService) RevokeCert(ctx context.Context, token, thingID string) (Revoke, error) { - var revoke Revoke - u, err := cs.auth.Identify(ctx, &magistrala.IdentityReq{Token: token}) - if err != nil { - return revoke, errors.Wrap(svcerr.ErrAuthentication, err) - } - thing, err := cs.sdk.Thing(thingID, token) - if err != nil { - return revoke, errors.Wrap(ErrFailedCertRevocation, err) - } - - offset, limit := uint64(0), uint64(10000) - cp, err := cs.certsRepo.RetrieveByThing(ctx, u.GetId(), thing.ID, offset, limit) - if err != nil { - return revoke, errors.Wrap(ErrFailedCertRevocation, err) - } - - for _, c := range cp.Certs { - revTime, err := cs.pki.Revoke(c.Serial) - if err != nil { - return revoke, errors.Wrap(ErrFailedCertRevocation, err) - } - revoke.RevocationTime = revTime - if err = cs.certsRepo.Remove(ctx, u.GetId(), c.Serial); err != nil { - return revoke, errors.Wrap(ErrFailedToRemoveCertFromDB, err) - } - } - - return revoke, nil -} - -func (cs *certsService) ListCerts(ctx context.Context, token, thingID string, offset, limit uint64) (Page, error) { - u, err := cs.auth.Identify(ctx, &magistrala.IdentityReq{Token: token}) - if err != nil { - return Page{}, errors.Wrap(svcerr.ErrAuthentication, err) - } - - cp, err := cs.certsRepo.RetrieveByThing(ctx, u.GetId(), thingID, offset, limit) - if err != nil { - return Page{}, errors.Wrap(svcerr.ErrViewEntity, err) - } - - for i, cert := range cp.Certs { - vcert, err := cs.pki.Read(cert.Serial) - if err != nil { - return Page{}, errors.Wrap(ErrFailedReadFromPKI, err) - } - cp.Certs[i].ClientCert = vcert.ClientCert - cp.Certs[i].ClientKey = vcert.ClientKey - } - - return cp, nil -} - -func (cs *certsService) ListSerials(ctx context.Context, token, thingID string, offset, limit uint64) (Page, error) { - u, err := cs.auth.Identify(ctx, &magistrala.IdentityReq{Token: token}) - if err != nil { - return Page{}, errors.Wrap(svcerr.ErrAuthentication, err) - } - - return cs.certsRepo.RetrieveByThing(ctx, u.GetId(), thingID, offset, limit) -} - -func (cs *certsService) ViewCert(ctx context.Context, token, serialID string) (Cert, error) { - u, err := cs.auth.Identify(ctx, &magistrala.IdentityReq{Token: token}) - if err != nil { - return Cert{}, errors.Wrap(svcerr.ErrAuthentication, err) - } - - cert, err := cs.certsRepo.RetrieveBySerial(ctx, u.GetId(), serialID) - if err != nil { - return Cert{}, errors.Wrap(svcerr.ErrViewEntity, err) - } - - vcert, err := cs.pki.Read(serialID) - if err != nil { - return Cert{}, errors.Wrap(ErrFailedReadFromPKI, err) - } - - c := Cert{ - ThingID: cert.ThingID, - ClientCert: vcert.ClientCert, - Serial: cert.Serial, - Expire: cert.Expire, - } - - return c, nil -} diff --git a/certs/service_test.go b/certs/service_test.go deleted file mode 100644 index 49043eb..0000000 --- a/certs/service_test.go +++ /dev/null @@ -1,448 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package certs_test - -import ( - "context" - "fmt" - "strings" - "testing" - "time" - - "github.com/absmach/magistrala" - authmocks "github.com/absmach/magistrala/auth/mocks" - "github.com/absmach/magistrala/certs" - "github.com/absmach/magistrala/certs/mocks" - "github.com/absmach/magistrala/certs/pki" - "github.com/absmach/magistrala/pkg/errors" - repoerr "github.com/absmach/magistrala/pkg/errors/repository" - svcerr "github.com/absmach/magistrala/pkg/errors/service" - mgsdk "github.com/absmach/magistrala/pkg/sdk/go" - sdkmocks "github.com/absmach/magistrala/pkg/sdk/mocks" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -const ( - invalid = "invalid" - email = "user@example.com" - token = "token" - thingsNum = 1 - thingKey = "thingKey" - thingID = "1" - ttl = "1h" - certNum = 10 - validID = "d4ebb847-5d0e-4e46-bdd9-b6aceaaa3a22" -) - -func newService(_ *testing.T) (certs.Service, *mocks.Repository, *mocks.Agent, *authmocks.AuthClient, *sdkmocks.SDK) { - repo := new(mocks.Repository) - agent := new(mocks.Agent) - auth := new(authmocks.AuthClient) - sdk := new(sdkmocks.SDK) - - return certs.New(auth, repo, sdk, agent), repo, agent, auth, sdk -} - -var cert = certs.Cert{ - OwnerID: validID, - ThingID: thingID, - Serial: "", - Expire: time.Time{}, -} - -func TestIssueCert(t *testing.T) { - svc, repo, agent, auth, sdk := newService(t) - cases := []struct { - token string - desc string - thingID string - ttl string - key string - pki pki.Cert - identifyRes *magistrala.IdentityRes - identifyErr error - thingErr errors.SDKError - issueCertErr error - repoErr error - err error - }{ - { - desc: "issue new cert", - token: token, - thingID: thingID, - ttl: ttl, - pki: pki.Cert{ - ClientCert: "", - IssuingCA: "", - CAChain: []string{}, - ClientKey: "", - PrivateKeyType: "", - Serial: "", - Expire: 0, - }, - identifyRes: &magistrala.IdentityRes{Id: validID}, - }, - { - desc: "issue new cert for non existing thing id", - token: token, - thingID: "2", - ttl: ttl, - pki: pki.Cert{ - ClientCert: "", - IssuingCA: "", - CAChain: []string{}, - ClientKey: "", - PrivateKeyType: "", - Serial: "", - Expire: 0, - }, - identifyRes: &magistrala.IdentityRes{Id: validID}, - thingErr: errors.NewSDKError(errors.ErrMalformedEntity), - err: certs.ErrFailedCertCreation, - }, - { - desc: "issue new cert for invalid token", - token: invalid, - thingID: thingID, - ttl: ttl, - pki: pki.Cert{ - ClientCert: "", - IssuingCA: "", - CAChain: []string{}, - ClientKey: "", - PrivateKeyType: "", - Serial: "", - Expire: 0, - }, - identifyRes: &magistrala.IdentityRes{Id: validID}, - identifyErr: svcerr.ErrAuthentication, - err: svcerr.ErrAuthentication, - }, - } - - for _, tc := range cases { - authCall := auth.On("Identify", context.Background(), &magistrala.IdentityReq{Token: tc.token}).Return(tc.identifyRes, tc.identifyErr) - sdkCall := sdk.On("Thing", tc.thingID, tc.token).Return(mgsdk.Thing{ID: tc.thingID, Credentials: mgsdk.Credentials{Secret: thingKey}}, tc.thingErr) - agentCall := agent.On("IssueCert", thingKey, tc.ttl).Return(tc.pki, tc.issueCertErr) - repoCall := repo.On("Save", context.Background(), mock.Anything).Return("", tc.repoErr) - - c, err := svc.IssueCert(context.Background(), tc.token, tc.thingID, tc.ttl) - assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err)) - cert, _ := certs.ReadCert([]byte(c.ClientCert)) - if cert != nil { - assert.True(t, strings.Contains(cert.Subject.CommonName, thingKey), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, thingKey, cert.Subject.CommonName)) - } - authCall.Unset() - sdkCall.Unset() - agentCall.Unset() - repoCall.Unset() - } -} - -func TestRevokeCert(t *testing.T) { - svc, repo, _, auth, sdk := newService(t) - cases := []struct { - token string - desc string - thingID string - page certs.Page - identifyRes *magistrala.IdentityRes - identifyErr error - authErr error - thingErr errors.SDKError - repoErr error - err error - }{ - { - desc: "revoke cert", - token: token, - thingID: thingID, - page: certs.Page{Limit: 10000, Offset: 0, Total: 1, Certs: []certs.Cert{cert}}, - identifyRes: &magistrala.IdentityRes{Id: validID}, - }, - { - desc: "revoke cert for invalid token", - token: invalid, - thingID: thingID, - page: certs.Page{}, - identifyRes: &magistrala.IdentityRes{Id: validID}, - identifyErr: svcerr.ErrAuthentication, - err: svcerr.ErrAuthentication, - }, - { - desc: "revoke cert for invalid thing id", - token: token, - thingID: "2", - page: certs.Page{}, - identifyRes: &magistrala.IdentityRes{Id: validID}, - thingErr: errors.NewSDKError(certs.ErrFailedCertCreation), - err: certs.ErrFailedCertRevocation, - }, - } - - for _, tc := range cases { - authCall := auth.On("Identify", context.Background(), &magistrala.IdentityReq{Token: tc.token}).Return(tc.identifyRes, tc.identifyErr) - authCall1 := auth.On("Authorize", context.Background(), mock.Anything).Return(&magistrala.AuthorizeRes{Authorized: true}, tc.authErr) - sdkCall := sdk.On("Thing", tc.thingID, tc.token).Return(mgsdk.Thing{ID: tc.thingID, Credentials: mgsdk.Credentials{Secret: thingKey}}, tc.thingErr) - repoCall := repo.On("RetrieveByThing", context.Background(), validID, tc.thingID, tc.page.Offset, tc.page.Limit).Return(certs.Page{}, tc.repoErr) - - _, err := svc.RevokeCert(context.Background(), tc.token, tc.thingID) - assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err)) - authCall.Unset() - authCall1.Unset() - sdkCall.Unset() - repoCall.Unset() - } -} - -func TestListCerts(t *testing.T) { - svc, repo, agent, auth, _ := newService(t) - var mycerts []certs.Cert - for i := 0; i < certNum; i++ { - c := certs.Cert{ - OwnerID: validID, - ThingID: thingID, - Serial: fmt.Sprintf("%d", i), - Expire: time.Now().Add(time.Hour), - } - mycerts = append(mycerts, c) - } - - for i := 0; i < certNum; i++ { - agent.On("Read", fmt.Sprintf("%d", i)).Return(pki.Cert{}, nil) - } - - cases := []struct { - token string - desc string - thingID string - page certs.Page - cert certs.Cert - identifyRes *magistrala.IdentityRes - identifyErr error - repoErr error - err error - }{ - { - desc: "list all certs with valid token", - token: token, - thingID: thingID, - page: certs.Page{Limit: certNum, Offset: 0, Total: certNum, Certs: mycerts}, - cert: certs.Cert{ - OwnerID: validID, - ThingID: thingID, - Serial: "0", - Expire: time.Now().Add(time.Hour), - }, - identifyRes: &magistrala.IdentityRes{Id: validID}, - }, - { - desc: "list all certs with invalid token", - token: invalid, - thingID: thingID, - page: certs.Page{}, - cert: certs.Cert{ - OwnerID: validID, - ThingID: thingID, - Serial: fmt.Sprintf("%d", certNum-1), - Expire: time.Now().Add(time.Hour), - }, - identifyRes: &magistrala.IdentityRes{Id: validID}, - identifyErr: svcerr.ErrAuthentication, - err: svcerr.ErrAuthentication, - }, - { - desc: "list half certs with valid token", - token: token, - thingID: thingID, - page: certs.Page{Limit: certNum, Offset: certNum / 2, Total: certNum / 2, Certs: mycerts[certNum/2:]}, - cert: certs.Cert{ - OwnerID: validID, - ThingID: thingID, - Serial: fmt.Sprintf("%d", certNum/2), - Expire: time.Now().Add(time.Hour), - }, - identifyRes: &magistrala.IdentityRes{Id: validID}, - }, - { - desc: "list last cert with valid token", - token: token, - thingID: thingID, - page: certs.Page{Limit: certNum, Offset: certNum - 1, Total: 1, Certs: []certs.Cert{mycerts[certNum-1]}}, - cert: certs.Cert{ - OwnerID: validID, - ThingID: thingID, - Serial: fmt.Sprintf("%d", certNum-1), - Expire: time.Now().Add(time.Hour), - }, - identifyRes: &magistrala.IdentityRes{Id: validID}, - }, - } - - for _, tc := range cases { - authCall := auth.On("Identify", context.Background(), &magistrala.IdentityReq{Token: tc.token}).Return(tc.identifyRes, tc.identifyErr) - repoCall := repo.On("RetrieveByThing", context.Background(), validID, thingID, tc.page.Offset, tc.page.Limit).Return(tc.page, tc.repoErr) - - page, err := svc.ListCerts(context.Background(), tc.token, tc.thingID, tc.page.Offset, tc.page.Limit) - size := uint64(len(page.Certs)) - assert.Equal(t, tc.page.Total, size, fmt.Sprintf("%s: expected %d got %d\n", tc.desc, tc.page.Total, size)) - assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err)) - authCall.Unset() - repoCall.Unset() - } -} - -func TestListSerials(t *testing.T) { - svc, repo, _, auth, _ := newService(t) - - var issuedCerts []certs.Cert - for i := 0; i < certNum; i++ { - crt := certs.Cert{ - OwnerID: cert.OwnerID, - ThingID: cert.ThingID, - Serial: cert.Serial, - Expire: cert.Expire, - } - issuedCerts = append(issuedCerts, crt) - } - - cases := []struct { - token string - desc string - thingID string - offset uint64 - limit uint64 - certs []certs.Cert - identifyRes *magistrala.IdentityRes - identifyErr error - repoErr error - err error - }{ - { - desc: "list all certs with valid token", - token: token, - thingID: thingID, - offset: 0, - limit: certNum, - certs: issuedCerts, - identifyRes: &magistrala.IdentityRes{Id: validID}, - }, - { - desc: "list all certs with invalid token", - token: invalid, - thingID: thingID, - offset: 0, - limit: certNum, - certs: nil, - identifyRes: &magistrala.IdentityRes{Id: validID}, - identifyErr: svcerr.ErrAuthentication, - err: svcerr.ErrAuthentication, - }, - { - desc: "list half certs with valid token", - token: token, - thingID: thingID, - offset: certNum / 2, - limit: certNum, - certs: issuedCerts[certNum/2:], - identifyRes: &magistrala.IdentityRes{Id: validID}, - }, - { - desc: "list last cert with valid token", - token: token, - thingID: thingID, - offset: certNum - 1, - limit: certNum, - certs: []certs.Cert{issuedCerts[certNum-1]}, - identifyRes: &magistrala.IdentityRes{Id: validID}, - }, - } - - for _, tc := range cases { - authCall := auth.On("Identify", context.Background(), &magistrala.IdentityReq{Token: tc.token}).Return(tc.identifyRes, tc.identifyErr) - repoCall := repo.On("RetrieveByThing", context.Background(), mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(certs.Page{Limit: tc.limit, Offset: tc.offset, Total: certNum, Certs: tc.certs}, tc.repoErr) - - page, err := svc.ListSerials(context.Background(), tc.token, tc.thingID, tc.offset, tc.limit) - assert.Equal(t, tc.certs, page.Certs, fmt.Sprintf("%s: expected %v got %v\n", tc.desc, tc.certs, page.Certs)) - assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err)) - authCall.Unset() - repoCall.Unset() - } -} - -func TestViewCert(t *testing.T) { - svc, repo, agent, auth, sdk := newService(t) - - authCall := auth.On("Identify", context.Background(), &magistrala.IdentityReq{Token: token}).Return(&magistrala.IdentityRes{Id: validID}, nil) - sdkCall := sdk.On("Thing", thingID, token).Return(mgsdk.Thing{ID: thingID, Credentials: mgsdk.Credentials{Secret: thingKey}}, nil) - agentCall := agent.On("IssueCert", thingKey, ttl).Return(pki.Cert{}, nil) - repoCall := repo.On("Save", context.Background(), mock.Anything).Return("", nil) - - ic, err := svc.IssueCert(context.Background(), token, thingID, ttl) - require.Nil(t, err, fmt.Sprintf("unexpected cert creation error: %s\n", err)) - authCall.Unset() - sdkCall.Unset() - agentCall.Unset() - repoCall.Unset() - - cert := certs.Cert{ - ThingID: thingID, - ClientCert: ic.ClientCert, - Serial: ic.Serial, - Expire: ic.Expire, - } - - cases := []struct { - token string - desc string - serialID string - cert certs.Cert - identifyRes *magistrala.IdentityRes - identifyErr error - repoErr error - agentErr error - err error - }{ - { - desc: "list cert with valid token and serial", - token: token, - serialID: cert.Serial, - cert: cert, - identifyRes: &magistrala.IdentityRes{Id: validID}, - }, - { - desc: "list cert with invalid token", - token: invalid, - serialID: cert.Serial, - cert: certs.Cert{}, - identifyRes: &magistrala.IdentityRes{Id: validID}, - identifyErr: svcerr.ErrAuthentication, - err: svcerr.ErrAuthentication, - }, - { - desc: "list cert with invalid serial", - token: token, - serialID: invalid, - cert: certs.Cert{}, - identifyRes: &magistrala.IdentityRes{Id: validID}, - repoErr: repoerr.ErrNotFound, - err: svcerr.ErrNotFound, - }, - } - - for _, tc := range cases { - authCall := auth.On("Identify", context.Background(), &magistrala.IdentityReq{Token: tc.token}).Return(tc.identifyRes, tc.identifyErr) - repoCall := repo.On("RetrieveBySerial", context.Background(), validID, tc.serialID).Return(tc.cert, tc.repoErr) - agentCall := agent.On("Read", tc.serialID).Return(pki.Cert{}, tc.agentErr) - - cert, err := svc.ViewCert(context.Background(), tc.token, tc.serialID) - assert.Equal(t, tc.cert, cert, fmt.Sprintf("%s: expected %v got %v\n", tc.desc, tc.cert, cert)) - assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s\n", tc.desc, tc.err, err)) - authCall.Unset() - repoCall.Unset() - agentCall.Unset() - } -} diff --git a/certs/tracing/doc.go b/certs/tracing/doc.go deleted file mode 100644 index 6a419f3..0000000 --- a/certs/tracing/doc.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package tracing provides tracing instrumentation for Magistrala Users Groups service. -// -// This package provides tracing middleware for Magistrala Users Groups service. -// It can be used to trace incoming requests and add tracing capabilities to -// Magistrala Users Groups service. -// -// For more details about tracing instrumentation for Magistrala messaging refer -// to the documentation at https://docs.magistrala.abstractmachines.fr/tracing/. -package tracing diff --git a/certs/tracing/tracing.go b/certs/tracing/tracing.go deleted file mode 100644 index e42614e..0000000 --- a/certs/tracing/tracing.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package tracing - -import ( - "context" - - "github.com/absmach/magistrala/certs" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -var _ certs.Service = (*tracingMiddleware)(nil) - -type tracingMiddleware struct { - tracer trace.Tracer - svc certs.Service -} - -// New returns a new certs service with tracing capabilities. -func New(svc certs.Service, tracer trace.Tracer) certs.Service { - return &tracingMiddleware{tracer, svc} -} - -// IssueCert traces the "IssueCert" operation of the wrapped certs.Service. -func (tm *tracingMiddleware) IssueCert(ctx context.Context, token, thingID, ttl string) (certs.Cert, error) { - ctx, span := tm.tracer.Start(ctx, "svc_create_group", trace.WithAttributes( - attribute.String("thing_id", thingID), - attribute.String("ttl", ttl), - )) - defer span.End() - - return tm.svc.IssueCert(ctx, token, thingID, ttl) -} - -// ListCerts traces the "ListCerts" operation of the wrapped certs.Service. -func (tm *tracingMiddleware) ListCerts(ctx context.Context, token, thingID string, offset, limit uint64) (certs.Page, error) { - ctx, span := tm.tracer.Start(ctx, "svc_list_certs", trace.WithAttributes( - attribute.String("thing_id", thingID), - attribute.Int64("offset", int64(offset)), - attribute.Int64("limit", int64(limit)), - )) - defer span.End() - - return tm.svc.ListCerts(ctx, token, thingID, offset, limit) -} - -// ListSerials traces the "ListSerials" operation of the wrapped certs.Service. -func (tm *tracingMiddleware) ListSerials(ctx context.Context, token, thingID string, offset, limit uint64) (certs.Page, error) { - ctx, span := tm.tracer.Start(ctx, "svc_list_serials", trace.WithAttributes( - attribute.String("thing_id", thingID), - attribute.Int64("offset", int64(offset)), - attribute.Int64("limit", int64(limit)), - )) - defer span.End() - - return tm.svc.ListSerials(ctx, token, thingID, offset, limit) -} - -// ViewCert traces the "ViewCert" operation of the wrapped certs.Service. -func (tm *tracingMiddleware) ViewCert(ctx context.Context, token, serialID string) (certs.Cert, error) { - ctx, span := tm.tracer.Start(ctx, "svc_view_cert", trace.WithAttributes( - attribute.String("serial_id", serialID), - )) - defer span.End() - - return tm.svc.ViewCert(ctx, token, serialID) -} - -// RevokeCert traces the "RevokeCert" operation of the wrapped certs.Service. -func (tm *tracingMiddleware) RevokeCert(ctx context.Context, token, serialID string) (certs.Revoke, error) { - ctx, span := tm.tracer.Start(ctx, "svc_revoke_cert", trace.WithAttributes( - attribute.String("serial_id", serialID), - )) - defer span.End() - - return tm.svc.RevokeCert(ctx, token, serialID) -} diff --git a/cmd/certs/main.go b/cmd/certs/main.go deleted file mode 100644 index a0c32e7..0000000 --- a/cmd/certs/main.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package main contains certs main function to start the certs service. -package main - -import ( - "context" - "fmt" - "log" - "log/slog" - "net/url" - "os" - - chclient "github.com/absmach/callhome/pkg/client" - "github.com/absmach/magistrala" - "github.com/absmach/magistrala/certs" - "github.com/absmach/magistrala/certs/api" - vault "github.com/absmach/magistrala/certs/pki" - certspg "github.com/absmach/magistrala/certs/postgres" - "github.com/absmach/magistrala/certs/tracing" - mglog "github.com/absmach/magistrala/logger" - "github.com/absmach/magistrala/pkg/auth" - jaegerclient "github.com/absmach/magistrala/pkg/jaeger" - "github.com/absmach/magistrala/pkg/postgres" - pgclient "github.com/absmach/magistrala/pkg/postgres" - "github.com/absmach/magistrala/pkg/prometheus" - mgsdk "github.com/absmach/magistrala/pkg/sdk/go" - "github.com/absmach/magistrala/pkg/server" - httpserver "github.com/absmach/magistrala/pkg/server/http" - "github.com/absmach/magistrala/pkg/uuid" - "github.com/caarlos0/env/v10" - "github.com/jmoiron/sqlx" - "go.opentelemetry.io/otel/trace" - "golang.org/x/sync/errgroup" -) - -const ( - svcName = "certs" - envPrefixDB = "MG_CERTS_DB_" - envPrefixHTTP = "MG_CERTS_HTTP_" - envPrefixAuth = "MG_AUTH_GRPC_" - defDB = "certs" - defSvcHTTPPort = "9019" -) - -type config struct { - LogLevel string `env:"MG_CERTS_LOG_LEVEL" envDefault:"info"` - ThingsURL string `env:"MG_THINGS_URL" envDefault:"http://localhost:9000"` - JaegerURL url.URL `env:"MG_JAEGER_URL" envDefault:"http://localhost:14268/api/traces"` - SendTelemetry bool `env:"MG_SEND_TELEMETRY" envDefault:"true"` - InstanceID string `env:"MG_CERTS_INSTANCE_ID" envDefault:""` - TraceRatio float64 `env:"MG_JAEGER_TRACE_RATIO" envDefault:"1.0"` - - // Sign and issue certificates without 3rd party PKI - SignCAPath string `env:"MG_CERTS_SIGN_CA_PATH" envDefault:"ca.crt"` - SignCAKeyPath string `env:"MG_CERTS_SIGN_CA_KEY_PATH" envDefault:"ca.key"` - - // 3rd party PKI API access settings - PkiHost string `env:"MG_CERTS_VAULT_HOST" envDefault:""` - PkiAppRoleID string `env:"MG_CERTS_VAULT_APPROLE_ROLEID" envDefault:""` - PkiAppSecret string `env:"MG_CERTS_VAULT_APPROLE_SECRET" envDefault:""` - PkiNamespace string `env:"MG_CERTS_VAULT_NAMESPACE" envDefault:""` - PkiPath string `env:"MG_CERTS_VAULT_THINGS_CERTS_PKI_PATH" envDefault:"pki_int"` - PkiRole string `env:"MG_CERTS_VAULT_THINGS_CERTS_PKI_ROLE_NAME" envDefault:"magistrala"` -} - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - g, ctx := errgroup.WithContext(ctx) - - cfg := config{} - if err := env.Parse(&cfg); err != nil { - log.Fatalf("failed to load %s configuration : %s", svcName, err) - } - - logger, err := mglog.New(os.Stdout, cfg.LogLevel) - if err != nil { - log.Fatalf("failed to init logger: %s", err.Error()) - } - - var exitCode int - defer mglog.ExitWithError(&exitCode) - - if cfg.InstanceID == "" { - if cfg.InstanceID, err = uuid.New().ID(); err != nil { - logger.Error(fmt.Sprintf("failed to generate instanceID: %s", err)) - exitCode = 1 - return - } - } - - if cfg.PkiHost == "" { - logger.Error("No host specified for PKI engine") - exitCode = 1 - return - } - - pkiclient, err := vault.NewVaultClient(cfg.PkiAppRoleID, cfg.PkiAppSecret, cfg.PkiHost, cfg.PkiNamespace, cfg.PkiPath, cfg.PkiRole, logger) - if err != nil { - logger.Error("failed to configure client for PKI engine") - exitCode = 1 - return - } - - g.Go(func() error { - return pkiclient.LoginAndRenew(ctx) - }) - - dbConfig := pgclient.Config{Name: defDB} - if err := env.ParseWithOptions(&dbConfig, env.Options{Prefix: envPrefixDB}); err != nil { - logger.Error(err.Error()) - } - db, err := pgclient.Setup(dbConfig, *certspg.Migration()) - if err != nil { - logger.Error(err.Error()) - exitCode = 1 - return - } - defer db.Close() - - authConfig := auth.Config{} - if err := env.ParseWithOptions(&authConfig, env.Options{Prefix: envPrefixAuth}); err != nil { - logger.Error(fmt.Sprintf("failed to load %s auth configuration : %s", svcName, err)) - exitCode = 1 - return - } - - authClient, authHandler, err := auth.Setup(ctx, authConfig) - if err != nil { - logger.Error(err.Error()) - exitCode = 1 - return - } - defer authHandler.Close() - - logger.Info("Successfully connected to auth grpc server " + authHandler.Secure()) - - tp, err := jaegerclient.NewProvider(ctx, svcName, cfg.JaegerURL, cfg.InstanceID, cfg.TraceRatio) - if err != nil { - logger.Error(fmt.Sprintf("failed to init Jaeger: %s", err)) - exitCode = 1 - return - } - defer func() { - if err := tp.Shutdown(ctx); err != nil { - logger.Error(fmt.Sprintf("error shutting down tracer provider: %v", err)) - } - }() - tracer := tp.Tracer(svcName) - - svc := newService(authClient, db, tracer, logger, cfg, dbConfig, pkiclient) - - httpServerConfig := server.Config{Port: defSvcHTTPPort} - if err := env.ParseWithOptions(&httpServerConfig, env.Options{Prefix: envPrefixHTTP}); err != nil { - logger.Error(fmt.Sprintf("failed to load %s HTTP server configuration : %s", svcName, err)) - exitCode = 1 - return - } - hs := httpserver.NewServer(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svc, logger, cfg.InstanceID), logger) - - if cfg.SendTelemetry { - chc := chclient.New(svcName, magistrala.Version, logger, cancel) - go chc.CallHome(ctx) - } - - g.Go(func() error { - return hs.Start() - }) - - g.Go(func() error { - return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Certs service terminated: %s", err)) - } -} - -func newService(authClient magistrala.AuthServiceClient, db *sqlx.DB, tracer trace.Tracer, logger *slog.Logger, cfg config, dbConfig pgclient.Config, pkiAgent vault.Agent) certs.Service { - database := postgres.NewDatabase(db, dbConfig, tracer) - certsRepo := certspg.NewRepository(database, logger) - config := mgsdk.Config{ - ThingsURL: cfg.ThingsURL, - } - sdk := mgsdk.NewSDK(config) - svc := certs.New(authClient, certsRepo, sdk, pkiAgent) - svc = api.LoggingMiddleware(svc, logger) - counter, latency := prometheus.MakeMetrics(svcName, "api") - svc = api.MetricsMiddleware(svc, counter, latency) - svc = tracing.New(svc, tracer) - - return svc -} diff --git a/cmd/provision/main.go b/cmd/provision/main.go deleted file mode 100644 index e6e5b8a..0000000 --- a/cmd/provision/main.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package main contains provision main function to start the provision service. -package main - -import ( - "context" - "encoding/json" - "fmt" - "log" - "os" - "reflect" - - chclient "github.com/absmach/callhome/pkg/client" - "github.com/absmach/magistrala" - mglog "github.com/absmach/magistrala/logger" - mgclients "github.com/absmach/magistrala/pkg/clients" - "github.com/absmach/magistrala/pkg/errors" - mggroups "github.com/absmach/magistrala/pkg/groups" - mgsdk "github.com/absmach/magistrala/pkg/sdk/go" - "github.com/absmach/magistrala/pkg/server" - httpserver "github.com/absmach/magistrala/pkg/server/http" - "github.com/absmach/magistrala/pkg/uuid" - "github.com/absmach/mg-contrib/provision" - "github.com/absmach/mg-contrib/provision/api" - "github.com/caarlos0/env/v10" - "golang.org/x/sync/errgroup" -) - -const ( - svcName = "provision" - contentType = "application/json" -) - -var ( - errMissingConfigFile = errors.New("missing config file setting") - errFailLoadingConfigFile = errors.New("failed to load config from file") - errFailedToReadBootstrapContent = errors.New("failed to read bootstrap content from envs") -) - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - g, ctx := errgroup.WithContext(ctx) - - cfg, err := loadConfig() - if err != nil { - log.Fatalf(fmt.Sprintf("failed to load %s configuration : %s", svcName, err)) - } - - logger, err := mglog.New(os.Stdout, cfg.Server.LogLevel) - if err != nil { - log.Fatalf("failed to init logger: %s", err.Error()) - } - - var exitCode int - defer mglog.ExitWithError(&exitCode) - - if cfg.InstanceID == "" { - if cfg.InstanceID, err = uuid.New().ID(); err != nil { - logger.Error(fmt.Sprintf("failed to generate instanceID: %s", err)) - exitCode = 1 - return - } - } - - if cfgFromFile, err := loadConfigFromFile(cfg.File); err != nil { - logger.Warn(fmt.Sprintf("Continue with settings from env, failed to load from: %s: %s", cfg.File, err)) - } else { - // Merge environment variables and file settings. - mergeConfigs(&cfgFromFile, &cfg) - cfg = cfgFromFile - logger.Info("Continue with settings from file: " + cfg.File) - } - - SDKCfg := mgsdk.Config{ - UsersURL: cfg.Server.UsersURL, - ThingsURL: cfg.Server.ThingsURL, - BootstrapURL: cfg.Server.MgBSURL, - CertsURL: cfg.Server.MgCertsURL, - MsgContentType: contentType, - TLSVerification: cfg.Server.TLS, - } - SDK := mgsdk.NewSDK(SDKCfg) - - svc := provision.New(cfg, SDK, logger) - svc = api.NewLoggingMiddleware(svc, logger) - - httpServerConfig := server.Config{Host: "", Port: cfg.Server.HTTPPort, KeyFile: cfg.Server.ServerKey, CertFile: cfg.Server.ServerCert} - hs := httpserver.NewServer(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svc, logger, cfg.InstanceID), logger) - - if cfg.SendTelemetry { - chc := chclient.New(svcName, magistrala.Version, logger, cancel) - go chc.CallHome(ctx) - } - - g.Go(func() error { - return hs.Start() - }) - - g.Go(func() error { - return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Provision service terminated: %s", err)) - } -} - -func loadConfigFromFile(file string) (provision.Config, error) { - _, err := os.Stat(file) - if os.IsNotExist(err) { - return provision.Config{}, errors.Wrap(errMissingConfigFile, err) - } - c, err := provision.Read(file) - if err != nil { - return provision.Config{}, errors.Wrap(errFailLoadingConfigFile, err) - } - return c, nil -} - -func loadConfig() (provision.Config, error) { - cfg := provision.Config{} - if err := env.Parse(&cfg); err != nil { - return provision.Config{}, err - } - - if cfg.Bootstrap.AutoWhiteList && !cfg.Bootstrap.Provision { - return provision.Config{}, errors.New("Can't auto whitelist if auto config save is off") - } - - var content map[string]interface{} - if cfg.BSContent != "" { - if err := json.Unmarshal([]byte(cfg.BSContent), &content); err != nil { - return provision.Config{}, errFailedToReadBootstrapContent - } - } - - cfg.Bootstrap.Content = content - // This is default conf for provision if there is no config file - cfg.Channels = []mggroups.Group{ - { - Name: "control-channel", - Metadata: map[string]interface{}{"type": "control"}, - }, { - Name: "data-channel", - Metadata: map[string]interface{}{"type": "data"}, - }, - } - cfg.Things = []mgclients.Client{ - { - Name: "thing", - Metadata: map[string]interface{}{"external_id": "xxxxxx"}, - }, - } - - return cfg, nil -} - -func mergeConfigs(dst, src interface{}) interface{} { - d := reflect.ValueOf(dst).Elem() - s := reflect.ValueOf(src).Elem() - - for i := 0; i < d.NumField(); i++ { - dField := d.Field(i) - sField := s.Field(i) - switch dField.Kind() { - case reflect.Struct: - dst := dField.Addr().Interface() - src := sField.Addr().Interface() - m := mergeConfigs(dst, src) - val := reflect.ValueOf(m).Elem().Interface() - dField.Set(reflect.ValueOf(val)) - case reflect.Slice: - case reflect.Bool: - if dField.Interface() == false { - dField.Set(reflect.ValueOf(sField.Interface())) - } - case reflect.Int: - if dField.Interface() == 0 { - dField.Set(reflect.ValueOf(sField.Interface())) - } - case reflect.String: - if dField.Interface() == "" { - dField.Set(reflect.ValueOf(sField.Interface())) - } - } - } - return dst -} diff --git a/cmd/smtp-notifier/main.go b/cmd/smtp-notifier/main.go index ca0821c..fe09f9d 100644 --- a/cmd/smtp-notifier/main.go +++ b/cmd/smtp-notifier/main.go @@ -31,7 +31,7 @@ import ( "github.com/absmach/magistrala/pkg/ulid" "github.com/absmach/magistrala/pkg/uuid" "github.com/absmach/mg-contrib/consumers/notifiers/smtp" - "github.com/absmach/mg-contrib/pkg/email" + email "github.com/absmach/mg-contrib/pkg/email" "github.com/caarlos0/env/v10" "github.com/jmoiron/sqlx" "go.opentelemetry.io/otel/trace" diff --git a/cmd/timescale-reader/main.go b/cmd/timescale-reader/main.go deleted file mode 100644 index 31dbd84..0000000 --- a/cmd/timescale-reader/main.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package main contains timescale-reader main function to start the timescale-reader service. -package main - -import ( - "context" - "fmt" - "log" - "log/slog" - "os" - - chclient "github.com/absmach/callhome/pkg/client" - "github.com/absmach/magistrala" - mglog "github.com/absmach/magistrala/logger" - "github.com/absmach/magistrala/pkg/auth" - pgclient "github.com/absmach/magistrala/pkg/postgres" - "github.com/absmach/magistrala/pkg/prometheus" - "github.com/absmach/magistrala/pkg/server" - httpserver "github.com/absmach/magistrala/pkg/server/http" - "github.com/absmach/magistrala/pkg/uuid" - "github.com/absmach/magistrala/readers" - "github.com/absmach/magistrala/readers/api" - "github.com/absmach/mg-contrib/readers/timescale" - "github.com/caarlos0/env/v10" - "github.com/jmoiron/sqlx" - "golang.org/x/sync/errgroup" -) - -const ( - svcName = "timescaledb-reader" - envPrefixDB = "MG_TIMESCALE_" - envPrefixHTTP = "MG_TIMESCALE_READER_HTTP_" - envPrefixAuth = "MG_AUTH_GRPC_" - envPrefixAuthz = "MG_THINGS_AUTH_GRPC_" - defDB = "messages" - defSvcHTTPPort = "9011" -) - -type config struct { - LogLevel string `env:"MG_TIMESCALE_READER_LOG_LEVEL" envDefault:"info"` - SendTelemetry bool `env:"MG_SEND_TELEMETRY" envDefault:"true"` - InstanceID string `env:"MG_TIMESCALE_READER_INSTANCE_ID" envDefault:""` -} - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - g, ctx := errgroup.WithContext(ctx) - - cfg := config{} - if err := env.Parse(&cfg); err != nil { - log.Fatalf("failed to load %s configuration : %s", svcName, err) - } - - logger, err := mglog.New(os.Stdout, cfg.LogLevel) - if err != nil { - log.Fatalf("failed to init logger: %s", err.Error()) - } - - var exitCode int - defer mglog.ExitWithError(&exitCode) - - if cfg.InstanceID == "" { - if cfg.InstanceID, err = uuid.New().ID(); err != nil { - logger.Error(fmt.Sprintf("failed to generate instanceID: %s", err)) - exitCode = 1 - return - } - } - - dbConfig := pgclient.Config{Name: defDB} - if err := env.ParseWithOptions(&dbConfig, env.Options{Prefix: envPrefixDB}); err != nil { - logger.Error(err.Error()) - exitCode = 1 - return - } - db, err := pgclient.Connect(dbConfig) - if err != nil { - logger.Error(err.Error()) - } - defer db.Close() - - repo := newService(db, logger) - - authConfig := auth.Config{} - if err := env.ParseWithOptions(&authConfig, env.Options{Prefix: envPrefixAuth}); err != nil { - logger.Error(fmt.Sprintf("failed to load %s auth configuration : %s", svcName, err)) - exitCode = 1 - return - } - - ac, acHandler, err := auth.Setup(ctx, authConfig) - if err != nil { - logger.Error(err.Error()) - exitCode = 1 - return - } - defer acHandler.Close() - - logger.Info("Successfully connected to auth grpc server " + acHandler.Secure()) - - authConfig = auth.Config{} - if err := env.ParseWithOptions(&authConfig, env.Options{Prefix: envPrefixAuthz}); err != nil { - logger.Error(fmt.Sprintf("failed to load %s auth configuration : %s", svcName, err)) - exitCode = 1 - return - } - - tc, tcHandler, err := auth.SetupAuthz(ctx, authConfig) - if err != nil { - logger.Error(err.Error()) - exitCode = 1 - return - } - defer tcHandler.Close() - - logger.Info("Successfully connected to things grpc server " + tcHandler.Secure()) - - httpServerConfig := server.Config{Port: defSvcHTTPPort} - if err := env.ParseWithOptions(&httpServerConfig, env.Options{Prefix: envPrefixHTTP}); err != nil { - logger.Error(fmt.Sprintf("failed to load %s HTTP server configuration : %s", svcName, err)) - exitCode = 1 - return - } - hs := httpserver.NewServer(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(repo, ac, tc, svcName, cfg.InstanceID), logger) - - if cfg.SendTelemetry { - chc := chclient.New(svcName, magistrala.Version, logger, cancel) - go chc.CallHome(ctx) - } - - g.Go(func() error { - return hs.Start() - }) - - g.Go(func() error { - return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Timescale reader service terminated: %s", err)) - } -} - -func newService(db *sqlx.DB, logger *slog.Logger) readers.MessageRepository { - svc := timescale.New(db) - svc = api.LoggingMiddleware(svc, logger) - counter, latency := prometheus.MakeMetrics("timescale", "message_reader") - svc = api.MetricsMiddleware(svc, counter, latency) - - return svc -} diff --git a/cmd/timescale-writer/main.go b/cmd/timescale-writer/main.go deleted file mode 100644 index 943a51f..0000000 --- a/cmd/timescale-writer/main.go +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package main contains timescale-writer main function to start the timescale-writer service. -package main - -import ( - "context" - "fmt" - "log" - "log/slog" - "net/url" - "os" - - chclient "github.com/absmach/callhome/pkg/client" - "github.com/absmach/magistrala" - "github.com/absmach/magistrala/consumers" - consumertracing "github.com/absmach/magistrala/consumers/tracing" - "github.com/absmach/magistrala/consumers/writers/api" - mglog "github.com/absmach/magistrala/logger" - jaegerclient "github.com/absmach/magistrala/pkg/jaeger" - "github.com/absmach/magistrala/pkg/messaging/brokers" - brokerstracing "github.com/absmach/magistrala/pkg/messaging/brokers/tracing" - pgclient "github.com/absmach/magistrala/pkg/postgres" - "github.com/absmach/magistrala/pkg/prometheus" - "github.com/absmach/magistrala/pkg/server" - httpserver "github.com/absmach/magistrala/pkg/server/http" - "github.com/absmach/magistrala/pkg/uuid" - "github.com/absmach/mg-contrib/consumers/writers/timescale" - "github.com/caarlos0/env/v10" - "github.com/jmoiron/sqlx" - "golang.org/x/sync/errgroup" -) - -const ( - svcName = "timescaledb-writer" - envPrefixDB = "MG_TIMESCALE_" - envPrefixHTTP = "MG_TIMESCALE_WRITER_HTTP_" - defDB = "messages" - defSvcHTTPPort = "9012" -) - -type config struct { - LogLevel string `env:"MG_TIMESCALE_WRITER_LOG_LEVEL" envDefault:"info"` - ConfigPath string `env:"MG_TIMESCALE_WRITER_CONFIG_PATH" envDefault:"/config.toml"` - BrokerURL string `env:"MG_MESSAGE_BROKER_URL" envDefault:"nats://localhost:4222"` - JaegerURL url.URL `env:"MG_JAEGER_URL" envDefault:"http://jaeger:14268/api/traces"` - SendTelemetry bool `env:"MG_SEND_TELEMETRY" envDefault:"true"` - InstanceID string `env:"MG_TIMESCALE_WRITER_INSTANCE_ID" envDefault:""` - TraceRatio float64 `env:"MG_JAEGER_TRACE_RATIO" envDefault:"1.0"` -} - -func main() { - ctx, cancel := context.WithCancel(context.Background()) - g, ctx := errgroup.WithContext(ctx) - - cfg := config{} - if err := env.Parse(&cfg); err != nil { - log.Fatalf("failed to load %s service configuration : %s", svcName, err) - } - - logger, err := mglog.New(os.Stdout, cfg.LogLevel) - if err != nil { - log.Fatalf("failed to init logger: %s", err.Error()) - } - - var exitCode int - defer mglog.ExitWithError(&exitCode) - - if cfg.InstanceID == "" { - if cfg.InstanceID, err = uuid.New().ID(); err != nil { - logger.Error(fmt.Sprintf("failed to generate instanceID: %s", err)) - exitCode = 1 - return - } - } - - httpServerConfig := server.Config{Port: defSvcHTTPPort} - if err := env.ParseWithOptions(&httpServerConfig, env.Options{Prefix: envPrefixHTTP}); err != nil { - logger.Error(fmt.Sprintf("failed to load %s HTTP server configuration : %s", svcName, err)) - exitCode = 1 - return - } - - dbConfig := pgclient.Config{Name: defDB} - if err := env.ParseWithOptions(&dbConfig, env.Options{Prefix: envPrefixDB}); err != nil { - logger.Error(fmt.Sprintf("failed to load %s Postgres configuration : %s", svcName, err)) - exitCode = 1 - return - } - db, err := pgclient.Setup(dbConfig, *timescale.Migration()) - if err != nil { - logger.Error(err.Error()) - exitCode = 1 - return - } - defer db.Close() - - tp, err := jaegerclient.NewProvider(ctx, svcName, cfg.JaegerURL, cfg.InstanceID, cfg.TraceRatio) - if err != nil { - logger.Error(fmt.Sprintf("Failed to init Jaeger: %s", err)) - exitCode = 1 - return - } - defer func() { - if err := tp.Shutdown(ctx); err != nil { - logger.Error(fmt.Sprintf("Error shutting down tracer provider: %v", err)) - } - }() - tracer := tp.Tracer(svcName) - - repo := newService(db, logger) - repo = consumertracing.NewBlocking(tracer, repo, httpServerConfig) - - pubSub, err := brokers.NewPubSub(ctx, cfg.BrokerURL, logger) - if err != nil { - logger.Error(fmt.Sprintf("failed to connect to message broker: %s", err)) - exitCode = 1 - return - } - defer pubSub.Close() - pubSub = brokerstracing.NewPubSub(httpServerConfig, tracer, pubSub) - - if err = consumers.Start(ctx, svcName, pubSub, repo, cfg.ConfigPath, logger); err != nil { - logger.Error(fmt.Sprintf("failed to create Timescale writer: %s", err)) - exitCode = 1 - return - } - - hs := httpserver.NewServer(ctx, cancel, svcName, httpServerConfig, api.MakeHandler(svcName, cfg.InstanceID), logger) - - if cfg.SendTelemetry { - chc := chclient.New(svcName, magistrala.Version, logger, cancel) - go chc.CallHome(ctx) - } - - g.Go(func() error { - return hs.Start() - }) - - g.Go(func() error { - return server.StopSignalHandler(ctx, cancel, logger, svcName, hs) - }) - - if err := g.Wait(); err != nil { - logger.Error(fmt.Sprintf("Timescale writer service terminated: %s", err)) - } -} - -func newService(db *sqlx.DB, logger *slog.Logger) consumers.BlockingConsumer { - svc := timescale.New(db) - svc = api.LoggingMiddleware(svc, logger) - counter, latency := prometheus.MakeMetrics("timescale", "message_writer") - svc = api.MetricsMiddleware(svc, counter, latency) - return svc -} diff --git a/consumers/consumer.go b/consumers/consumer.go deleted file mode 100644 index 403f9a3..0000000 --- a/consumers/consumer.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package consumers - -import "context" - -// AsyncConsumer specifies a non-blocking message-consuming API, -// which can be used for writing data to the DB, publishing messages -// to broker, sending notifications, or any other asynchronous job. -type AsyncConsumer interface { - // ConsumeAsync method is used to asynchronously consume received messages. - ConsumeAsync(ctx context.Context, messages interface{}) - - // Errors method returns a channel for reading errors which occur during async writes. - // Must be called before performing any writes for errors to be collected. - // The channel is buffered(1) so it allows only 1 error without blocking if not drained. - // The channel may receive nil error to indicate success. - Errors() <-chan error -} - -// BlockingConsumer specifies a blocking message-consuming API, -// which can be used for writing data to the DB, publishing messages -// to broker, sending notifications... BlockingConsumer implementations -// might also support concurrent use, but consult implementation for more details. -type BlockingConsumer interface { - // ConsumeBlocking method is used to consume received messages synchronously. - // A non-nil error is returned to indicate operation failure. - ConsumeBlocking(ctx context.Context, messages interface{}) error -} diff --git a/consumers/messages.go b/consumers/messages.go deleted file mode 100644 index 0d25edf..0000000 --- a/consumers/messages.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package consumers - -import ( - "context" - "fmt" - "log/slog" - "os" - "strings" - - "github.com/absmach/magistrala/pkg/apiutil" - "github.com/absmach/magistrala/pkg/errors" - "github.com/absmach/magistrala/pkg/messaging" - "github.com/absmach/magistrala/pkg/messaging/brokers" - "github.com/absmach/magistrala/pkg/transformers" - "github.com/absmach/magistrala/pkg/transformers/json" - "github.com/absmach/magistrala/pkg/transformers/senml" - "github.com/pelletier/go-toml" -) - -const ( - defContentType = "application/senml+json" - defFormat = "senml" -) - -var ( - errOpenConfFile = errors.New("unable to open configuration file") - errParseConfFile = errors.New("unable to parse configuration file") -) - -// Start method starts consuming messages received from Message broker. -// This method transforms messages to SenML format before -// using MessageRepository to store them. -func Start(ctx context.Context, id string, sub messaging.Subscriber, consumer interface{}, configPath string, logger *slog.Logger) error { - cfg, err := loadConfig(configPath) - if err != nil { - logger.Warn(fmt.Sprintf("Failed to load consumer config: %s", err)) - } - - transformer := makeTransformer(cfg.TransformerCfg, logger) - - for _, subject := range cfg.SubscriberCfg.Subjects { - subCfg := messaging.SubscriberConfig{ - ID: id, - Topic: subject, - DeliveryPolicy: messaging.DeliverAllPolicy, - } - switch c := consumer.(type) { - case AsyncConsumer: - subCfg.Handler = handleAsync(ctx, transformer, c) - if err := sub.Subscribe(ctx, subCfg); err != nil { - return err - } - case BlockingConsumer: - subCfg.Handler = handleSync(ctx, transformer, c) - if err := sub.Subscribe(ctx, subCfg); err != nil { - return err - } - default: - return apiutil.ErrInvalidQueryParams - } - } - return nil -} - -func handleSync(ctx context.Context, t transformers.Transformer, sc BlockingConsumer) handleFunc { - return func(msg *messaging.Message) error { - m := interface{}(msg) - var err error - if t != nil { - m, err = t.Transform(msg) - if err != nil { - return err - } - } - return sc.ConsumeBlocking(ctx, m) - } -} - -func handleAsync(ctx context.Context, t transformers.Transformer, ac AsyncConsumer) handleFunc { - return func(msg *messaging.Message) error { - m := interface{}(msg) - var err error - if t != nil { - m, err = t.Transform(msg) - if err != nil { - return err - } - } - - ac.ConsumeAsync(ctx, m) - return nil - } -} - -type handleFunc func(msg *messaging.Message) error - -func (h handleFunc) Handle(msg *messaging.Message) error { - return h(msg) -} - -func (h handleFunc) Cancel() error { - return nil -} - -type subscriberConfig struct { - Subjects []string `toml:"subjects"` -} - -type transformerConfig struct { - Format string `toml:"format"` - ContentType string `toml:"content_type"` - TimeFields []json.TimeField `toml:"time_fields"` -} - -type config struct { - SubscriberCfg subscriberConfig `toml:"subscriber"` - TransformerCfg transformerConfig `toml:"transformer"` -} - -func loadConfig(configPath string) (config, error) { - cfg := config{ - SubscriberCfg: subscriberConfig{ - Subjects: []string{brokers.SubjectAllChannels}, - }, - TransformerCfg: transformerConfig{ - Format: defFormat, - ContentType: defContentType, - }, - } - - data, err := os.ReadFile(configPath) - if err != nil { - return cfg, errors.Wrap(errOpenConfFile, err) - } - - if err := toml.Unmarshal(data, &cfg); err != nil { - return cfg, errors.Wrap(errParseConfFile, err) - } - - return cfg, nil -} - -func makeTransformer(cfg transformerConfig, logger *slog.Logger) transformers.Transformer { - switch strings.ToUpper(cfg.Format) { - case "SENML": - logger.Info("Using SenML transformer") - return senml.New(cfg.ContentType) - case "JSON": - logger.Info("Using JSON transformer") - return json.New(cfg.TimeFields) - default: - logger.Error(fmt.Sprintf("Can't create transformer: unknown transformer type %s", cfg.Format)) - os.Exit(1) - return nil - } -} diff --git a/consumers/writers/api/doc.go b/consumers/writers/api/doc.go deleted file mode 100644 index 2424852..0000000 --- a/consumers/writers/api/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package api contains API-related concerns: endpoint definitions, middlewares -// and all resource representations. -package api diff --git a/consumers/writers/api/logging.go b/consumers/writers/api/logging.go deleted file mode 100644 index 77e5f91..0000000 --- a/consumers/writers/api/logging.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -//go:build !test - -package api - -import ( - "context" - "log/slog" - "time" - - "github.com/absmach/magistrala/consumers" -) - -var _ consumers.BlockingConsumer = (*loggingMiddleware)(nil) - -type loggingMiddleware struct { - logger *slog.Logger - consumer consumers.BlockingConsumer -} - -// LoggingMiddleware adds logging facilities to the adapter. -func LoggingMiddleware(consumer consumers.BlockingConsumer, logger *slog.Logger) consumers.BlockingConsumer { - return &loggingMiddleware{ - logger: logger, - consumer: consumer, - } -} - -// ConsumeBlocking logs the consume request. It logs the time it took to complete the request. -// If the request fails, it logs the error. -func (lm *loggingMiddleware) ConsumeBlocking(ctx context.Context, msgs interface{}) (err error) { - defer func(begin time.Time) { - args := []any{ - slog.String("duration", time.Since(begin).String()), - } - if err != nil { - args = append(args, slog.Any("error", err)) - lm.logger.Warn("Blocking consumer failed to consume messages successfully", args...) - return - } - lm.logger.Info("Blocking consumer consumed messages successfully", args...) - }(time.Now()) - - return lm.consumer.ConsumeBlocking(ctx, msgs) -} diff --git a/consumers/writers/api/metrics.go b/consumers/writers/api/metrics.go deleted file mode 100644 index 29dfb2f..0000000 --- a/consumers/writers/api/metrics.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -//go:build !test - -package api - -import ( - "context" - "time" - - "github.com/absmach/magistrala/consumers" - "github.com/go-kit/kit/metrics" -) - -var _ consumers.BlockingConsumer = (*metricsMiddleware)(nil) - -type metricsMiddleware struct { - counter metrics.Counter - latency metrics.Histogram - consumer consumers.BlockingConsumer -} - -// MetricsMiddleware returns new message repository -// with Save method wrapped to expose metrics. -func MetricsMiddleware(consumer consumers.BlockingConsumer, counter metrics.Counter, latency metrics.Histogram) consumers.BlockingConsumer { - return &metricsMiddleware{ - counter: counter, - latency: latency, - consumer: consumer, - } -} - -// ConsumeBlocking instruments ConsumeBlocking method with metrics. -func (mm *metricsMiddleware) ConsumeBlocking(ctx context.Context, msgs interface{}) error { - defer func(begin time.Time) { - mm.counter.With("method", "consume").Add(1) - mm.latency.With("method", "consume").Observe(time.Since(begin).Seconds()) - }(time.Now()) - return mm.consumer.ConsumeBlocking(ctx, msgs) -} diff --git a/consumers/writers/api/transport.go b/consumers/writers/api/transport.go deleted file mode 100644 index 3c2fa5d..0000000 --- a/consumers/writers/api/transport.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package api - -import ( - "net/http" - - "github.com/absmach/magistrala" - "github.com/go-chi/chi/v5" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -// MakeHandler returns a HTTP API handler with health check and metrics. -func MakeHandler(svcName, instanceID string) http.Handler { - r := chi.NewRouter() - r.Get("/health", magistrala.Health(svcName, instanceID)) - r.Handle("/metrics", promhttp.Handler()) - - return r -} diff --git a/go.mod b/go.mod index e9784bc..24f438a 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22.5 require ( github.com/0x6flab/namegenerator v1.4.0 github.com/absmach/callhome v0.14.0 - github.com/absmach/magistrala v0.14.1-0.20240704203022-083e655bde82 + github.com/absmach/magistrala v0.14.1-0.20240709113739-04c359462746 github.com/absmach/senml v1.0.5 github.com/caarlos0/env/v10 v10.0.0 github.com/eclipse/paho.mqtt.golang v1.4.3 @@ -15,25 +15,28 @@ require ( github.com/go-redis/redis/v8 v8.11.5 github.com/gocql/gocql v1.6.0 github.com/gofrs/uuid v4.4.0+incompatible + github.com/gookit/color v1.5.4 github.com/gopcua/opcua v0.1.6 - github.com/hashicorp/vault/api v1.14.0 - github.com/hashicorp/vault/api/auth/approle v0.7.0 + github.com/gorilla/websocket v1.5.3 github.com/influxdata/influxdb-client-go/v2 v2.13.0 + github.com/ivanpirog/coloredcobra v1.0.1 github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 github.com/jackc/pgx/v5 v5.6.0 github.com/jmoiron/sqlx v1.4.0 - github.com/mitchellh/mapstructure v1.5.0 github.com/ory/dockertest/v3 v3.10.0 github.com/pelletier/go-toml v1.9.5 github.com/prometheus/client_golang v1.19.1 github.com/rubenv/sql-migrate v1.6.1 + github.com/spf13/cobra v1.8.1 + github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 go.mongodb.org/mongo-driver v1.15.0 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 - go.opentelemetry.io/otel v1.27.0 - go.opentelemetry.io/otel/trace v1.27.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 + go.opentelemetry.io/otel v1.28.0 + go.opentelemetry.io/otel/trace v1.28.0 golang.org/x/sync v0.7.0 - google.golang.org/grpc v1.64.0 + gonum.org/v1/gonum v0.15.0 + google.golang.org/grpc v1.65.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df ) @@ -44,7 +47,6 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/continuity v0.4.3 // indirect @@ -54,31 +56,24 @@ require ( github.com/docker/docker v26.0.2+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/fatih/color v1.17.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.6.0 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect - github.com/go-jose/go-jose/v4 v4.0.1 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect - github.com/gorilla/websocket v1.5.3 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-retryablehttp v0.7.7 // indirect - github.com/hashicorp/go-rootcerts v1.0.2 // indirect - github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 // indirect - github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect - github.com/hashicorp/go-sockaddr v1.0.6 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -87,7 +82,10 @@ require ( github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/klauspost/compress v1.17.8 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/term v0.5.0 // indirect github.com/montanaflynn/stats v0.7.1 // indirect @@ -99,16 +97,23 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runc v1.1.12 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.52.2 // indirect github.com/prometheus/procfs v0.13.0 // indirect github.com/rabbitmq/amqp091-go v1.10.0 // indirect - github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/smarty/assertions v1.16.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect @@ -116,25 +121,28 @@ require ( github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect go.opentelemetry.io/proto/otlp v1.2.0 // indirect - golang.org/x/crypto v0.24.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/mod v0.18.0 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect - golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.22.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240709173604-40e1e62336c5 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect moul.io/http2curl v1.0.0 // indirect diff --git a/go.sum b/go.sum index 96cb44a..3ae24c9 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrd github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/absmach/callhome v0.14.0 h1:zB4tIZJ1YUmZ1VGHFPfMA/Lo6/Mv19y2dvoOiXj2BWs= github.com/absmach/callhome v0.14.0/go.mod h1:l12UJOfibK4Muvg/AbupHuquNV9qSz/ROdTEPg7f2Vk= -github.com/absmach/magistrala v0.14.1-0.20240704203022-083e655bde82 h1:8EQqSe0oj88smyZcHwndRw1Ye4v6xQs35Nkvs5Mi9lI= -github.com/absmach/magistrala v0.14.1-0.20240704203022-083e655bde82/go.mod h1:Nk3rTAEyI8S83eibXnGEfnRrTT5dyb+v2nJgXfeMGLU= +github.com/absmach/magistrala v0.14.1-0.20240709113739-04c359462746 h1:Tj567KeGVygjTsSCxn4++skKiz9GkPugM1KMdIFxvfw= +github.com/absmach/magistrala v0.14.1-0.20240709113739-04c359462746/go.mod h1:CIx3OsPFc4doJZmBWSA6LNWefcznKv9c3cLOxNxL4q4= github.com/absmach/mproxy v0.4.3-0.20240430090627-27dad4c91c6c h1:wGtfVk3knDUsrUoyOxfyDPK3lJB6Yc6BMePf62UaTOo= github.com/absmach/mproxy v0.4.3-0.20240430090627-27dad4c91c6c/go.mod h1:Nevip6o8u5Zx7l3LTtN8BwlCI5h5KpsnI9YnAxF5RT8= github.com/absmach/senml v1.0.5 h1:zNPRYpGr2Wsb8brAusz8DIfFqemy1a2dNbmMnegY3GE= @@ -49,6 +49,8 @@ github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -70,12 +72,15 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eclipse/paho.mqtt.golang v1.4.3 h1:2kwcUGn8seMUfWndX0hGbvH8r7crgcJguQNCyp70xik= github.com/eclipse/paho.mqtt.golang v1.4.3/go.mod h1:CSYvoAlsMkhYOXh/oKyxa8EcBci6dVkLCbo5tTC1RIE= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fiorix/go-smpp v0.0.0-20210403173735-2894b96e70ba h1:vBqABUa2HUSc6tj22Tw+ZMVGHuBzKtljM38kbRanmrM= github.com/fiorix/go-smpp v0.0.0-20210403173735-2894b96e70ba/go.mod h1:VfKFK7fGeCP81xEhbrOqUEh45n73Yy6jaPWwTVbxprI= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA= @@ -95,8 +100,8 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= @@ -104,8 +109,6 @@ github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= -github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= @@ -125,6 +128,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= +github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/gopcua/opcua v0.1.6 h1:B9SVRKQGzcWcwP2QPYN93Uku32+3wL+v5cgzBxE6V5I= github.com/gopcua/opcua v0.1.6/go.mod h1:INwnDoRxmNWAt7+tzqxuGqQkSF2c1C69VAL0c2q6AcY= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -135,13 +140,10 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1 github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= -github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= @@ -160,10 +162,15 @@ github.com/hashicorp/vault/api v1.14.0 h1:Ah3CFLixD5jmjusOgm8grfN9M0d+Y8fVR2SW0K github.com/hashicorp/vault/api v1.14.0/go.mod h1:pV9YLxBGSz+cItFDd8Ii4G17waWOQ32zVjMWHe/cOqk= github.com/hashicorp/vault/api/auth/approle v0.7.0 h1:R5IRVuFA5JSdG3UdGVcGysi0StrL1lPmyJnrawiV0Ss= github.com/hashicorp/vault/api/auth/approle v0.7.0/go.mod h1:B+WaC6VR+aSXiUxykpaPUoFiiZAhic53tDLbGjWZmRA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb-client-go/v2 v2.13.0 h1:ioBbLmR5NMbAjP4UVA5r9b5xGjpABD7j65pI8kFphDM= github.com/influxdata/influxdb-client-go/v2 v2.13.0/go.mod h1:k+spCbt9hcvqvUiz0sr5D8LolXHqAAOfPw9v/RIRHl4= github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU= github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/ivanpirog/coloredcobra v1.0.1 h1:aURSdEmlR90/tSiWS0dMjdwOvCVUeYLfltLfbgNxrN4= +github.com/ivanpirog/coloredcobra v1.0.1/go.mod h1:iho4nEKcnwZFiniGSdcgdvRgZNjxm+h20acv8vqmN6Q= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= @@ -258,13 +265,18 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= @@ -308,6 +320,8 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -335,8 +349,13 @@ github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThC github.com/rubenv/sql-migrate v1.6.1 h1:bo6/sjsan9HaXAsNxYP/jCEDUGibHp8JmOBw7NTGRos= github.com/rubenv/sql-migrate v1.6.1/go.mod h1:tPzespupJS0jacLfhbwto/UjSX+8h2FdWB7ar+QlHa0= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= @@ -351,6 +370,19 @@ github.com/smarty/assertions v1.16.0 h1:EvHNkdRA4QHMrn75NZSoUQ/mAUXAYWfatfB01yTC github.com/smarty/assertions v1.16.0/go.mod h1:duaaFdCS0K9dnoM50iyek/eYINOZ64gbh1Xlf6LG7AI= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -367,8 +399,11 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= @@ -385,6 +420,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -393,22 +430,22 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 h1:vS1Ao/R55RNV4O7TA2Qopok8yN+X0LIP6RVWLFkprck= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0/go.mod h1:BMsdeOxN04K0L5FNUBfjFdvwWGNe/rkmSwH4Aelu/X0= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= -go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= -go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= -go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= -go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= -go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= -go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -420,6 +457,8 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -438,8 +477,10 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -460,8 +501,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -486,14 +527,17 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -536,12 +580,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 h1:P8OJ/WCl/Xo4E4zoe4/bifHpSmmKwARqyqE4nW6J2GQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:RGnPtTG7r4i8sPlNyDeikXF99hMM+hN6QMm4ooG9g2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 h1:Q2RxlXqh1cgzzUgV261vBO2jI5R/3DD1J2pM0nI4NhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= +gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240709173604-40e1e62336c5 h1:SbSDUWW1PAO24TNpLdeheoYPd7kllICcLU52x6eD4kQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240709173604-40e1e62336c5/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= @@ -556,6 +602,8 @@ gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkp gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/provision/README.md b/provision/README.md deleted file mode 100644 index 73f6c86..0000000 --- a/provision/README.md +++ /dev/null @@ -1,194 +0,0 @@ -# Provision service - -Provision service provides an HTTP API to interact with [Magistrala][magistrala]. -Provision service is used to setup initial applications configuration i.e. things, channels, connections and certificates that will be required for the specific use case especially useful for gateway provision. - -For gateways to communicate with [Magistrala][magistrala] configuration is required (mqtt host, thing, channels, certificates...). To get the configuration gateway will send a request to [Bootstrap][bootstrap] service providing `` and `` in request. To make a request to [Bootstrap][bootstrap] service you can use [Agent][agent] service on a gateway. - -To create bootstrap configuration you can use [Bootstrap][bootstrap] or `Provision` service. [Magistrala UI][mgxui] uses [Bootstrap][bootstrap] service for creating gateway configurations. `Provision` service should provide an easy way of provisioning your gateways i.e creating bootstrap configuration and as many things and channels that your setup requires. - -Also you may use provision service to create certificates for each thing. Each service running on gateway may require more than one thing and channel for communication. Let's say that you are using services [Agent][agent] and [Export][export] on a gateway you will need two channels for `Agent` (`data` and `control`) and one for `Export` and one thing. Additionally if you enabled mtls each service will need its own thing and certificate for access to [Magistrala][magistrala]. Your setup could require any number of things and channels this kind of setup we can call `provision layout`. - -Provision service provides a way of specifying this `provision layout` and creating a setup according to that layout by serving requests on `/mapping` endpoint. Provision layout is configured in [config.toml](configs/config.toml). - -## Configuration - -The service is configured using the environment variables presented in the -following table. Note that any unset variables will be replaced with their -default values. - -| Variable | Description | Default | -| ----------------------------------- | ------------------------------------------------- | ------------------------------------ | -| MG_PROVISION_LOG_LEVEL | Service log level | debug | -| MG_PROVISION_USER | User (email) for accessing Magistrala | | -| MG_PROVISION_PASS | Magistrala password | user123 | -| MG_PROVISION_API_KEY | Magistrala authentication token | | -| MG_PROVISION_CONFIG_FILE | Provision config file | config.toml | -| MG_PROVISION_HTTP_PORT | Provision service listening port | 9016 | -| MG_PROVISION_ENV_CLIENTS_TLS | Magistrala SDK TLS verification | false | -| MG_PROVISION_SERVER_CERT | Magistrala gRPC secure server cert | | -| MG_PROVISION_SERVER_KEY | Magistrala gRPC secure server key | | -| MG_PROVISION_USERS_LOCATION | Users service URL | | -| MG_PROVISION_THINGS_LOCATION | Things service URL | | -| MG_PROVISION_BS_SVC_URL | Magistrala Bootstrap service URL | | -| MG_PROVISION_CERTS_SVC_URL | Certificates service URL | | -| MG_PROVISION_X509_PROVISIONING | Should X509 client cert be provisioned | false | -| MG_PROVISION_BS_CONFIG_PROVISIONING | Should thing config be saved in Bootstrap service | true | -| MG_PROVISION_BS_AUTO_WHITELIST | Should thing be auto whitelisted | true | -| MG_PROVISION_BS_CONTENT | Bootstrap service configs content, JSON format | {} | -| MG_PROVISION_CERTS_RSA_BITS | Certificate RSA bits parameter | 4096 | -| MG_PROVISION_CERTS_HOURS_VALID | Number of hours that certificate is valid | "2400h" | -| MG_SEND_TELEMETRY | Send telemetry to magistrala call home server | true | - -By default, call to `/mapping` endpoint will create one thing and two channels (`control` and `data`) and connect it. If there is a requirement for different provision layout we can use [config](docker/configs/config.toml) file in addition to environment variables. - -For the purposes of running provision as an add-on in docker composition environment variables seems more suitable. Environment variables are set in [.env](.env). - -Configuration can be specified in [config.toml](configs/config.toml). Config file can specify all the settings that environment variables can configure and in addition -`/mapping` endpoint provision layout can be configured. - -In `config.toml` we can enlist array of things and channels that we want to create and make connections between them which we call provision layout. - -Metadata can be whatever suits your needs except that at least one thing needs to have `external_id` (which is populated with value from [request](#example)). Thing that has `external_id` will be used for creating bootstrap configuration which can be fetched with [Agent][agent]. -For channels metadata `type` is reserved for `control` and `data` which we use with [Agent][agent]. - -Example of provision layout below - -```toml -[[things]] - name = "thing" - - [things.metadata] - external_id = "xxxxxx" - - -[[channels]] - name = "control-channel" - - [channels.metadata] - type = "control" - -[[channels]] - name = "data-channel" - - [channels.metadata] - type = "data" - -[[channels]] - name = "export-channel" - - [channels.metadata] - type = "data" -``` - -## Authentication - -In order to create necessary entities provision service needs to authenticate against Magistrala. To provide authentication credentials to the provision service you can pass it in an environment variable or in a config file as Magistrala user and password or as API token that can be issued on `/users/tokens/issue`. - -Additionally users or API token can be passed in Authorization header, this authentication takes precedence over others. - -- `username`, `password` - (`MG_PROVISION_USER`, `MG_PROVISION_PASSWORD` in [.env](../.env), `mg_user`, `mg_pass` in [config.toml](../docker/addons/provision/configs/config.toml)) -- API Key - (`MG_PROVISION_API_KEY` in [.env](../.env) or [config.toml](../docker/addons/provision/configs/config.toml)) -- `Authorization: Bearer Token` - request authorization header containing either users token. - -## Running - -Provision service can be run as a standalone or in docker composition as addon to the core docker composition. - -Standalone: - -```bash -MG_PROVISION_BS_SVC_URL=http://localhost:9013 \ -MG_PROVISION_THINGS_LOCATION=http://localhost:9000 \ -MG_PROVISION_USERS_LOCATION=http://localhost:9002 \ -MG_PROVISION_CONFIG_FILE=docker/addons/provision/configs/config.toml \ -build/magistrala-provision -``` - -Docker composition: - -```bash -docker compose -f docker/addons/provision/docker-compose.yml up -``` - -For the case that credentials or API token is passed in configuration file or environment variables, call to `/mapping` endpoint doesn't require `Authentication` header: - -```bash -curl -s -S -X POST http://localhost:/mapping -H 'Content-Type: application/json' -d '{"external_id": "33:52:77:99:43", "external_key": "223334fw2"}' -``` - -In the case that provision service is not deployed with credentials or API key or you want to use user other than one being set in environment (or config file): - -```bash -curl -s -S -X POST http://localhost:/mapping -H "Authorization: Bearer " -H 'Content-Type: application/json' -d '{"external_id": "", "external_key": ""}' -``` - -Or if you want to specify a name for thing different than in `config.toml` you can specify post data as: - -```json -{ - "name": "", - "external_id": "", - "external_key": "" -} -``` - -Response contains created things, channels and certificates if any: - -```json -{ - "things": [ - { - "id": "c22b0c0f-8c03-40da-a06b-37ed3a72c8d1", - "name": "thing", - "key": "007cce56-e0eb-40d6-b2b9-ed348a97d1eb", - "metadata": { - "external_id": "33:52:79:C3:43" - } - } - ], - "channels": [ - { - "id": "064c680e-181b-4b58-975e-6983313a5170", - "name": "control-channel", - "metadata": { - "type": "control" - } - }, - { - "id": "579da92d-6078-4801-a18a-dd1cfa2aa44f", - "name": "data-channel", - "metadata": { - "type": "data" - } - } - ], - "whitelisted": { - "c22b0c0f-8c03-40da-a06b-37ed3a72c8d1": true - } -} -``` - -## Certificates - -Provision service has `/certs` endpoint that can be used to generate certificates for things when mTLS is required: - -- `users_token` - users authentication token or API token -- `thing_id` - id of the thing for which certificate is going to be generated - -```bash -curl -s -X POST http://localhost:8190/certs -H "Authorization: Bearer " -H 'Content-Type: application/json' -d '{"thing_id": "", "ttl":"2400h" }' -``` - -```json -{ - "thing_cert": "-----BEGIN CERTIFICATE-----\nMIIEmDCCA4CgAwIBAgIQCZ0NOq2oKLo+XftbAu0TfzANBgkqhkiG9w0BAQsFADBX\nMRIwEAYDVQQDDAlsb2NhbGhvc3QxETAPBgNVBAoMCE1haW5mbHV4MQwwCgYDVQQL\nDANJb1QxIDAeBgkqhkiG9w0BCQEWEWluZm9AbWFpbmZsdXguY29tMB4XDTIwMDYw\nNTEyMzc1M1oXDTIwMDkxMzEyMzc1M1owVTERMA8GA1UEChMITWFpbmZsdXgxETAP\nBgNVBAsTCG1haW5mbHV4MS0wKwYDVQQDEyQyYmZlYmZmMC05ODZhLTQ3ZTAtOGQ3\nYS00YTRiN2UyYjU3OGUwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCn\nWvTuOIdhqOLEREcEJqfQAtDoYu3rUDijOffXuWFZgNqfZTGmoD5ZqJXxwbZ4tCST\npdSteHtyr7JXnPJQN1dsslU+q3haKjFoZRc39/7u4/8XCTwlqbMl9YVcwqS+FLkM\niLSyyqzryP7Y8H8cidTKg56p5JALaEKfzZS6Km3G+CCinR6hNNW9ckWsy29a0/9E\nMAUtM+Lsk5OjsHzOnWruuqHsCx4ODI5aJQaMC1qntkbXkht0WDiwAt9SDQ3uLWru\nAoSJDK9a6EgR3a0Jf7ZiVPiwlZNjrB/I5OQyFDGqcmSAl2rdJqPkmaDXKKFyL1cG\nMIyHv62QzJoMdRoXu20lxyGxAvEjQNVHux4LA3dbf/85nEVTI2uP8crMf2Jnzbg5\n9zF+iTMJGpUlatCyK2RJS/mvHbbUIf5Ro3VbcPHbgFroJ7qMFz0Fc5kYY8IdwXjG\nlyG9MobKEO2CfBGRjPmCuTQq2HcuOy7F6KfQf3HToI8MmC5hBtCmTNbV8I3GIjWA\n/xJQLm2pVZ41QhrnNGtuqAYoe3Zt6OldxGRcoAj7KlIpYcPZ55PJ6mWcV6dB9Fnl\n5mYOwQL8jtfybbGWvqJldhTxUqm7/EbAaF0Qjmh4oOHMl2xADrmYzJHvf0llwr6g\noRQuzqxPi0aW3tkFNsm63NX1Ab5BXFQhMSj5+82blwIDAQABo2IwYDAOBgNVHQ8B\nAf8EBAMCB4AwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMA4GA1UdDgQH\nBAUBAgMEBjAfBgNVHSMEGDAWgBRs4xR91qEjNRGmw391xS7x6Tc+8jANBgkqhkiG\n9w0BAQsFAAOCAQEAphLT8PjawRRWswU1B5oWnnqeTllnvGB88sjDPLAG0UiBlDLX\nwoPiBVPWuYV+MMJuaREgheYF1Ahx4Jrfy9stFDU7B99ON1T58oM1aKEq4rKc+/Ke\nyxrAFTonclC0LNaaOvpZZjsPFWr2muTQO8XHiS8icw3BLxEzoF+5aJ8ihtxRtfKL\nUvtHDqC6IPAbSUcvqyjrFh3RrTUAyGOzW12IEWSXP9DLwoiLPwJ6kCVoXdG/asjz\nUpk/jj7AUn9oJNF8nUbyhdOnmeJ2z0x1ylgYrIAxvGzm8zs+NEVN67CrBYKwstlN\nvw7DRQsCvGJjZzWj28VV3FGLtXFgu52bFZNBww==\n-----END CERTIFICATE-----\n", - "thing_cert_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIJJwIBAAKCAgEAp1r07jiHYajixERHBCan0ALQ6GLt61A4ozn317lhWYDan2Ux\npqA+WaiV8cG2eLQkk6XUrXh7cq+yV5zyUDdXbLJVPqt4WioxaGUXN/f+7uP/Fwk8\nJamzJfWFXMKkvhS5DIi0ssqs68j+2PB/HInUyoOeqeSQC2hCn82Uuiptxvggop0e\noTTVvXJFrMtvWtP/RDAFLTPi7JOTo7B8zp1q7rqh7AseDgyOWiUGjAtap7ZG15Ib\ndFg4sALfUg0N7i1q7gKEiQyvWuhIEd2tCX+2YlT4sJWTY6wfyOTkMhQxqnJkgJdq\n3Saj5Jmg1yihci9XBjCMh7+tkMyaDHUaF7ttJcchsQLxI0DVR7seCwN3W3//OZxF\nUyNrj/HKzH9iZ824OfcxfokzCRqVJWrQsitkSUv5rx221CH+UaN1W3Dx24Ba6Ce6\njBc9BXOZGGPCHcF4xpchvTKGyhDtgnwRkYz5grk0Kth3Ljsuxein0H9x06CPDJgu\nYQbQpkzW1fCNxiI1gP8SUC5tqVWeNUIa5zRrbqgGKHt2bejpXcRkXKAI+ypSKWHD\n2eeTyeplnFenQfRZ5eZmDsEC/I7X8m2xlr6iZXYU8VKpu/xGwGhdEI5oeKDhzJds\nQA65mMyR739JZcK+oKEULs6sT4tGlt7ZBTbJutzV9QG+QVxUITEo+fvNm5cCAwEA\nAQKCAgAmCIfNc89gpG8Ux6eUC+zrWxh7F7CWX97fSZdH0XuMSbplqyvDgHtrCOM6\n1BlSCS6e13skCVOU1tUjECoJjOoza7vvyCxL4XblEMRcFeI8DFi2tYST0qNCJzAt\nypaCFFeRv6fBUkpGM6GnT9Czfad8drkiRy1tSj6J7sC0JlxYcZ+JFUgWvtksesHW\n6UzfSXqj1n32reoOdeOBueRDWIcqxgNyj3w/GR9o4S1BunrZzpT+/Nd8c2g+qAh0\nrz7ROEUq3iucseNQN6XZWZWvqPScGE+EYhni9wUqNMqfjvNSlzi7+K1yoQtyMm/Z\nNgSq3JNcdsAZQbiCRd1ko2BQsGm3ZBnbsAJ1Dxcn+i9nF5DT/ddWjUWin6LYWuUM\n/0Bqfv3etlrFuP6yxc8bPEMX0ucJg4yVxdkDrm1tYlJ+ANEQoOlZqhngvjz0f8uO\nOtEcDLmiG5VG6Yl72UtWIw+ALnKc5U7ib43Qve0bDAKR5zlHODcRetN9BCMvpekY\nOA4hohkllTP25xmMzLokBqY9n38zEt74kJOp67VKMvhoF7QkrLOfKWCRJjFL7/9I\nHDa6jb31INA9Wu+p/2LIa6I1SUYnMvCUqISgF2hBG9Q9S9TZvKnYUvfurhFS9jZv\n18sxW7IFYWmQyioo+gsAmfKLolJtLl9hCmTfYi7oqCh/EtZdIQKCAQEA0Umkp0Uu\nimVilLjgYGTWLcg8T3NWaELQzb2HYRXSzEq/M8GOtEr7TR7noJBm8fcgl55HEnPl\ni4cEJrr+VprzGbdMtXjHbCD+I945GA6vv3khg7mbqS9a1Uw6gjrQEZgZQU+/IVCu\n9Pbvx8Af32xaBWuN2cFzC7Z6iB815LPc2O5qyZ3+3nEUPah+Z+a9WEeTR6M0hy5c\nkkaRqhehugHDgqMRWGt8GfsFOmaR13kvfFfKadPRPkaGkftCSKBMWjrU4uX7aulm\nD7k4VDbnXIBMhI039+0znSkhZdcV1zk6qwBYn9TtZ11PTlspFPjtPxqS5M6IGflw\nsXkZGv4rZ5CkiQKCAQEAzLVdw2qw/8rWGsCV39EKp7hXLvp7+FuodPvX1L55lWB0\nvmSOldGcNvb2ZsK3RNvgteb8VfKRgaY6waeN5Qm1UXazsOX4F+GThPGHstdNuzkt\nJofRQQHQVR3npZbCngSkSZdahQ9SjiLIDKn8baPN8I8HfpJ4oHLUvkayavbch1kJ\nYWUfGtVKxHGX5m/nnxLdgbJEx9Q+3Qa7DDHuxTqsEqhkk0R0Ganred34HjpDNMs6\nV95HFNolW3yKfuHETKA1bLhej+XdMa11Ts5hBVGCMnnT07WcGhxtyK2dSa656SyT\ngT9+Hd1VWZ/KPpAkQmH9boOr2ihE+oAXiZ4D1t53HwKCAQAD0cA7fTu4Mtl1tVoC\n6FQwSbMwD/7HsFB3MLpDv041hDexDhs4lxW29pVrjLcUO1pQ6gaKA6twvGoK+uah\nVfqRwZKYzTd2dbOtm+SW183FRMSjzsNUdxTFR7rZnZEmgQwU8Quf5AUNW2RM1Oi/\n/w41gxz3mFwtHotl6IvnPJEPNGqme0enb5Da/zQvWTqjXcsGR6gxv1rZIIiP/hZp\nepbCz48FehCtuLMDudN3hzKipkd/Xuo2pLrX9ynigWpjSyePbHsGHHRMXSj2AHqA\naab71EftMlr6x0FgxmgToWu8qyjy4cPjWwSTfX5mb5SEzktX+ZzqPG8eDgOzRmgs\nX6thAoIBADL3kQG/hZQaL1Z3zpjsFggOKH7E1KrQP0/pCCKqzeC4JDjnFm0MxCUX\nNd/96N1XFUqU2QyZGUs7VPO0QOrekOtYb4LCrxNbEXyPGicX3f2YTbqDJEFYL0OR\n74PV1ly7cR/1dA8e8oH6/O3SQMwXdYXIRqhn1Wq1TGyXc4KYNe3o6CH8qFLo+fWR\nBq3T/MopS0coWGGcYY5sR5PQts8aPY9jp67W40UkfkFYV5dHEEaLttn7uJzjd1ug\n1Waj1VjypnqMKNcQ9xKQSl21mohVc+IXXPsgA16o51iIiVm4DAeXFp6ebUsIOWDY\nHOWYw75XYV7rn5TwY8Qusi2MTw5nUycCggEAB/45U0LW7ZGpks/aF/BeGaSWiLIG\nodBWUjRQ4w+Le/pTC8Ci9fiidxuCDH6TQbsUTGKOk7GsfncWHTQJogaMyO26IJ1N\nmYGgK2JJvs7PKyIkocPDVD/Yh0gIzQIE92ZdyXUT21pIYKDUB9e3p0fy/+E0pyeI\nsmsV8oaLr4tZRY1cMogI+pvtUUferbLQmZHhFd9X3m3RslR43Dl1qpYQyzE3x/a3\nWA2NJZbJhh+LiAKzqk7swXOqrTrmXuzLcjMG+T/3lizrbLLuKjQrf+eehlpw0db0\nHVVvkMLOP5ZH/ImkmvOZJY7xxup89VV7LD7TfMKwXafOrjMDdvTAYPtgxw==\n-----END RSA PRIVATE KEY-----\n" -} -``` - -[magistrala]: https://github.com/absmach/magistrala -[bootstrap]: https://github.com/absmach/magistrala/tree/master/bootstrap -[export]: https://github.com/absmach/export -[agent]: https://github.com/absmach/agent -[mgxui]: https://github.com/absmach/magistrala/ui diff --git a/provision/api/doc.go b/provision/api/doc.go deleted file mode 100644 index 2424852..0000000 --- a/provision/api/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package api contains API-related concerns: endpoint definitions, middlewares -// and all resource representations. -package api diff --git a/provision/api/endpoint.go b/provision/api/endpoint.go deleted file mode 100644 index db9ac6d..0000000 --- a/provision/api/endpoint.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package api - -import ( - "context" - - "github.com/absmach/magistrala/pkg/apiutil" - "github.com/absmach/magistrala/pkg/errors" - "github.com/absmach/mg-contrib/provision" - "github.com/go-kit/kit/endpoint" -) - -func doProvision(svc provision.Service) endpoint.Endpoint { - return func(_ context.Context, request interface{}) (interface{}, error) { - req := request.(provisionReq) - if err := req.validate(); err != nil { - return nil, errors.Wrap(apiutil.ErrValidation, err) - } - - res, err := svc.Provision(req.token, req.Name, req.ExternalID, req.ExternalKey) - if err != nil { - return nil, err - } - - provisionResponse := provisionRes{ - Things: res.Things, - Channels: res.Channels, - ClientCert: res.ClientCert, - ClientKey: res.ClientKey, - CACert: res.CACert, - Whitelisted: res.Whitelisted, - } - - return provisionResponse, nil - } -} - -func getMapping(svc provision.Service) endpoint.Endpoint { - return func(_ context.Context, request interface{}) (interface{}, error) { - req := request.(mappingReq) - if err := req.validate(); err != nil { - return nil, errors.Wrap(apiutil.ErrValidation, err) - } - - res, err := svc.Mapping(req.token) - if err != nil { - return nil, err - } - - return mappingRes{Data: res}, nil - } -} diff --git a/provision/api/endpoint_test.go b/provision/api/endpoint_test.go deleted file mode 100644 index 4bcf310..0000000 --- a/provision/api/endpoint_test.go +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package api_test - -import ( - "fmt" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - mglog "github.com/absmach/magistrala/logger" - "github.com/absmach/magistrala/pkg/apiutil" - svcerr "github.com/absmach/magistrala/pkg/errors/service" - "github.com/absmach/mg-contrib/pkg/testsutil" - "github.com/absmach/mg-contrib/provision" - "github.com/absmach/mg-contrib/provision/api" - "github.com/absmach/mg-contrib/provision/mocks" - "github.com/stretchr/testify/assert" -) - -var ( - validToken = "valid" - validContenType = "application/json" - validID = testsutil.GenerateUUID(&testing.T{}) -) - -type testRequest struct { - client *http.Client - method string - url string - token string - contentType string - body io.Reader -} - -func (tr testRequest) make() (*http.Response, error) { - req, err := http.NewRequest(tr.method, tr.url, tr.body) - if err != nil { - return nil, err - } - - if tr.token != "" { - req.Header.Set("Authorization", apiutil.BearerPrefix+tr.token) - } - - if tr.contentType != "" { - req.Header.Set("Content-Type", tr.contentType) - } - - return tr.client.Do(req) -} - -func newProvisionServer() (*httptest.Server, *mocks.Service) { - svc := new(mocks.Service) - - logger := mglog.NewMock() - mux := api.MakeHandler(svc, logger, "test") - return httptest.NewServer(mux), svc -} - -func TestProvision(t *testing.T) { - is, svc := newProvisionServer() - - cases := []struct { - desc string - token string - data string - contentType string - status int - svcErr error - }{ - { - desc: "valid request", - token: validToken, - data: fmt.Sprintf(`{"name": "test", "external_id": "%s", "external_key": "%s"}`, validID, validID), - status: http.StatusCreated, - contentType: validContenType, - svcErr: nil, - }, - { - desc: "request with empty external id", - token: validToken, - data: fmt.Sprintf(`{"name": "test", "external_key": "%s"}`, validID), - status: http.StatusBadRequest, - contentType: validContenType, - svcErr: nil, - }, - { - desc: "request with empty external key", - token: validToken, - data: fmt.Sprintf(`{"name": "test", "external_id": "%s"}`, validID), - status: http.StatusBadRequest, - contentType: validContenType, - svcErr: nil, - }, - { - desc: "empty token", - token: "", - data: fmt.Sprintf(`{"name": "test", "external_id": "%s", "external_key": "%s"}`, validID, validID), - status: http.StatusCreated, - contentType: validContenType, - svcErr: nil, - }, - { - desc: "invalid content type", - token: validToken, - data: fmt.Sprintf(`{"name": "test", "external_id": "%s", "external_key": "%s"}`, validID, validID), - status: http.StatusUnsupportedMediaType, - contentType: "text/plain", - svcErr: nil, - }, - { - desc: "invalid request", - token: validToken, - data: `data`, - status: http.StatusBadRequest, - contentType: validContenType, - svcErr: nil, - }, - { - desc: "service error", - token: validToken, - data: fmt.Sprintf(`{"name": "test", "external_id": "%s", "external_key": "%s"}`, validID, validID), - status: http.StatusForbidden, - contentType: validContenType, - svcErr: svcerr.ErrAuthorization, - }, - } - - for _, tc := range cases { - t.Run(tc.desc, func(t *testing.T) { - repocall := svc.On("Provision", tc.token, "test", validID, validID).Return(provision.Result{}, tc.svcErr) - req := testRequest{ - client: is.Client(), - method: http.MethodPost, - url: is.URL + "/mapping", - token: tc.token, - contentType: tc.contentType, - body: strings.NewReader(tc.data), - } - - resp, err := req.make() - assert.Nil(t, err, tc.desc) - assert.Equal(t, tc.status, resp.StatusCode, tc.desc) - repocall.Unset() - }) - } -} - -func TestMapping(t *testing.T) { - is, svc := newProvisionServer() - - cases := []struct { - desc string - token string - contentType string - status int - svcErr error - }{ - { - desc: "valid request", - token: validToken, - status: http.StatusOK, - contentType: validContenType, - svcErr: nil, - }, - { - desc: "empty token", - token: "", - status: http.StatusUnauthorized, - contentType: validContenType, - svcErr: nil, - }, - { - desc: "invalid content type", - token: validToken, - status: http.StatusUnsupportedMediaType, - contentType: "text/plain", - svcErr: nil, - }, - { - desc: "service error", - token: validToken, - status: http.StatusForbidden, - contentType: validContenType, - svcErr: svcerr.ErrAuthorization, - }, - } - - for _, tc := range cases { - t.Run(tc.desc, func(t *testing.T) { - repocall := svc.On("Mapping", tc.token).Return(map[string]interface{}{}, tc.svcErr) - req := testRequest{ - client: is.Client(), - method: http.MethodGet, - url: is.URL + "/mapping", - token: tc.token, - contentType: tc.contentType, - } - - resp, err := req.make() - assert.Nil(t, err, tc.desc) - assert.Equal(t, tc.status, resp.StatusCode, tc.desc) - repocall.Unset() - }) - } -} diff --git a/provision/api/logging.go b/provision/api/logging.go deleted file mode 100644 index eea62a9..0000000 --- a/provision/api/logging.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -//go:build !test - -package api - -import ( - "log/slog" - "time" - - "github.com/absmach/mg-contrib/provision" -) - -var _ provision.Service = (*loggingMiddleware)(nil) - -type loggingMiddleware struct { - logger *slog.Logger - svc provision.Service -} - -// NewLoggingMiddleware adds logging facilities to the core service. -func NewLoggingMiddleware(svc provision.Service, logger *slog.Logger) provision.Service { - return &loggingMiddleware{logger, svc} -} - -func (lm *loggingMiddleware) Provision(token, name, externalID, externalKey string) (res provision.Result, err error) { - defer func(begin time.Time) { - args := []any{ - slog.String("duration", time.Since(begin).String()), - slog.String("name", name), - slog.String("external_id", externalID), - } - if err != nil { - args = append(args, slog.Any("error", err)) - lm.logger.Warn("Provision failed", args...) - return - } - lm.logger.Info("Provision completed successfully", args...) - }(time.Now()) - - return lm.svc.Provision(token, name, externalID, externalKey) -} - -func (lm *loggingMiddleware) Cert(token, thingID, duration string) (cert, key string, err error) { - defer func(begin time.Time) { - args := []any{ - slog.String("duration", time.Since(begin).String()), - slog.String("thing_id", thingID), - slog.String("ttl", duration), - } - if err != nil { - args = append(args, slog.Any("error", err)) - lm.logger.Warn("Thing certificate failed to create successfully", args...) - return - } - lm.logger.Info("Thing certificate created successfully", args...) - }(time.Now()) - - return lm.svc.Cert(token, thingID, duration) -} - -func (lm *loggingMiddleware) Mapping(token string) (res map[string]interface{}, err error) { - defer func(begin time.Time) { - args := []any{ - slog.String("duration", time.Since(begin).String()), - } - if err != nil { - args = append(args, slog.Any("error", err)) - lm.logger.Warn("Mapping failed", args...) - return - } - lm.logger.Info("Mapping completed successfully", args...) - }(time.Now()) - - return lm.svc.Mapping(token) -} diff --git a/provision/api/requests.go b/provision/api/requests.go deleted file mode 100644 index 323b98e..0000000 --- a/provision/api/requests.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package api - -import "github.com/absmach/magistrala/pkg/apiutil" - -type provisionReq struct { - token string - Name string `json:"name"` - ExternalID string `json:"external_id"` - ExternalKey string `json:"external_key"` -} - -func (req provisionReq) validate() error { - if req.ExternalID == "" { - return apiutil.ErrMissingID - } - - if req.ExternalKey == "" { - return apiutil.ErrBearerKey - } - - if req.Name == "" { - return apiutil.ErrMissingName - } - - return nil -} - -type mappingReq struct { - token string -} - -func (req mappingReq) validate() error { - if req.token == "" { - return apiutil.ErrBearerToken - } - return nil -} diff --git a/provision/api/requests_test.go b/provision/api/requests_test.go deleted file mode 100644 index c7f12c9..0000000 --- a/provision/api/requests_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package api - -import ( - "fmt" - "testing" - - "github.com/absmach/magistrala/pkg/apiutil" - "github.com/absmach/magistrala/pkg/errors" - "github.com/absmach/mg-contrib/pkg/testsutil" - "github.com/stretchr/testify/assert" -) - -func TestProvisioReq(t *testing.T) { - cases := []struct { - desc string - req provisionReq - err error - }{ - { - desc: "valid request", - req: provisionReq{ - token: "token", - Name: "name", - ExternalID: testsutil.GenerateUUID(t), - ExternalKey: testsutil.GenerateUUID(t), - }, - err: nil, - }, - { - desc: "empty external id", - req: provisionReq{ - token: "token", - Name: "name", - ExternalID: "", - ExternalKey: testsutil.GenerateUUID(t), - }, - err: apiutil.ErrMissingID, - }, - { - desc: "empty external key", - req: provisionReq{ - token: "token", - Name: "name", - ExternalID: testsutil.GenerateUUID(t), - ExternalKey: "", - }, - err: apiutil.ErrBearerKey, - }, - } - - for _, tc := range cases { - err := tc.req.validate() - assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected `%v` got `%v`", tc.desc, tc.err, err)) - } -} - -func TestMappingReq(t *testing.T) { - cases := []struct { - desc string - req mappingReq - err error - }{ - { - desc: "valid request", - req: mappingReq{ - token: "token", - }, - err: nil, - }, - { - desc: "empty token", - req: mappingReq{ - token: "", - }, - err: apiutil.ErrBearerToken, - }, - } - - for _, tc := range cases { - err := tc.req.validate() - assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected `%v` got `%v`", tc.desc, tc.err, err)) - } -} diff --git a/provision/api/responses.go b/provision/api/responses.go deleted file mode 100644 index 87c1052..0000000 --- a/provision/api/responses.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package api - -import ( - "encoding/json" - "net/http" - - "github.com/absmach/magistrala" - sdk "github.com/absmach/magistrala/pkg/sdk/go" -) - -var _ magistrala.Response = (*provisionRes)(nil) - -type provisionRes struct { - Things []sdk.Thing `json:"things"` - Channels []sdk.Channel `json:"channels"` - ClientCert map[string]string `json:"client_cert,omitempty"` - ClientKey map[string]string `json:"client_key,omitempty"` - CACert string `json:"ca_cert,omitempty"` - Whitelisted map[string]bool `json:"whitelisted,omitempty"` -} - -func (res provisionRes) Code() int { - return http.StatusCreated -} - -func (res provisionRes) Headers() map[string]string { - return map[string]string{} -} - -func (res provisionRes) Empty() bool { - return false -} - -type mappingRes struct { - Data interface{} -} - -func (res mappingRes) Code() int { - return http.StatusOK -} - -func (res mappingRes) Headers() map[string]string { - return map[string]string{} -} - -func (res mappingRes) Empty() bool { - return false -} - -func (res mappingRes) MarshalJSON() ([]byte, error) { - return json.Marshal(res.Data) -} diff --git a/provision/api/transport.go b/provision/api/transport.go deleted file mode 100644 index 4485d87..0000000 --- a/provision/api/transport.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package api - -import ( - "context" - "encoding/json" - "log/slog" - "net/http" - - "github.com/absmach/magistrala" - "github.com/absmach/magistrala/pkg/apiutil" - "github.com/absmach/magistrala/pkg/errors" - "github.com/absmach/mg-contrib/pkg/api" - "github.com/absmach/mg-contrib/provision" - "github.com/go-chi/chi/v5" - kithttp "github.com/go-kit/kit/transport/http" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -const ( - contentType = "application/json" -) - -// MakeHandler returns a HTTP handler for API endpoints. -func MakeHandler(svc provision.Service, logger *slog.Logger, instanceID string) http.Handler { - opts := []kithttp.ServerOption{ - kithttp.ServerErrorEncoder(apiutil.LoggingErrorEncoder(logger, api.EncodeError)), - } - - r := chi.NewRouter() - - r.Route("/mapping", func(r chi.Router) { - r.Post("/", kithttp.NewServer( - doProvision(svc), - decodeProvisionRequest, - api.EncodeResponse, - opts..., - ).ServeHTTP) - r.Get("/", kithttp.NewServer( - getMapping(svc), - decodeMappingRequest, - api.EncodeResponse, - opts..., - ).ServeHTTP) - }) - - r.Handle("/metrics", promhttp.Handler()) - r.Get("/health", magistrala.Health("provision", instanceID)) - - return r -} - -func decodeProvisionRequest(_ context.Context, r *http.Request) (interface{}, error) { - if r.Header.Get("Content-Type") != contentType { - return nil, errors.Wrap(apiutil.ErrValidation, apiutil.ErrUnsupportedContentType) - } - - req := provisionReq{token: apiutil.ExtractBearerToken(r)} - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - return nil, errors.Wrap(apiutil.ErrValidation, errors.Wrap(err, errors.ErrMalformedEntity)) - } - - return req, nil -} - -func decodeMappingRequest(_ context.Context, r *http.Request) (interface{}, error) { - if r.Header.Get("Content-Type") != contentType { - return nil, errors.Wrap(apiutil.ErrValidation, apiutil.ErrUnsupportedContentType) - } - - req := mappingReq{token: apiutil.ExtractBearerToken(r)} - - return req, nil -} diff --git a/provision/config.go b/provision/config.go deleted file mode 100644 index d0f6683..0000000 --- a/provision/config.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package provision - -import ( - "fmt" - "os" - - mgclients "github.com/absmach/magistrala/pkg/clients" - "github.com/absmach/magistrala/pkg/errors" - "github.com/absmach/magistrala/pkg/groups" - "github.com/pelletier/go-toml" -) - -var errFailedToReadConfig = errors.New("failed to read config file") - -// ServiceConf represents service config. -type ServiceConf struct { - Port string `toml:"port" env:"MG_PROVISION_HTTP_PORT" envDefault:"9016"` - LogLevel string `toml:"log_level" env:"MG_PROVISION_LOG_LEVEL" envDefault:"info"` - TLS bool `toml:"tls" env:"MG_PROVISION_ENV_CLIENTS_TLS" envDefault:"false"` - ServerCert string `toml:"server_cert" env:"MG_PROVISION_SERVER_CERT" envDefault:""` - ServerKey string `toml:"server_key" env:"MG_PROVISION_SERVER_KEY" envDefault:""` - ThingsURL string `toml:"things_url" env:"MG_PROVISION_THINGS_LOCATION" envDefault:"http://localhost"` - UsersURL string `toml:"users_url" env:"MG_PROVISION_USERS_LOCATION" envDefault:"http://localhost"` - HTTPPort string `toml:"http_port" env:"MG_PROVISION_HTTP_PORT" envDefault:"9016"` - MgUser string `toml:"mg_user" env:"MG_PROVISION_USER" envDefault:"test@example.com"` - MgPass string `toml:"mg_pass" env:"MG_PROVISION_PASS" envDefault:"test"` - MgDomainID string `toml:"mg_domain_id" env:"MG_PROVISION_DOMAIN_ID" envDefault:""` - MgAPIKey string `toml:"mg_api_key" env:"MG_PROVISION_API_KEY" envDefault:""` - MgBSURL string `toml:"mg_bs_url" env:"MG_PROVISION_BS_SVC_URL" envDefault:"http://localhost:9000"` - MgCertsURL string `toml:"mg_certs_url" env:"MG_PROVISION_CERTS_SVC_URL" envDefault:"http://localhost:9019"` -} - -// Bootstrap represetns the Bootstrap config. -type Bootstrap struct { - X509Provision bool `toml:"x509_provision" env:"MG_PROVISION_X509_PROVISIONING" envDefault:"false"` - Provision bool `toml:"provision" env:"MG_PROVISION_BS_CONFIG_PROVISIONING" envDefault:"true"` - AutoWhiteList bool `toml:"autowhite_list" env:"MG_PROVISION_BS_AUTO_WHITELIST" envDefault:"true"` - Content map[string]interface{} `toml:"content"` -} - -// Gateway represetns the Gateway config. -type Gateway struct { - Type string `toml:"type" json:"type"` - ExternalID string `toml:"external_id" json:"external_id"` - ExternalKey string `toml:"external_key" json:"external_key"` - CtrlChannelID string `toml:"ctrl_channel_id" json:"ctrl_channel_id"` - DataChannelID string `toml:"data_channel_id" json:"data_channel_id"` - ExportChannelID string `toml:"export_channel_id" json:"export_channel_id"` - CfgID string `toml:"cfg_id" json:"cfg_id"` -} - -// Cert represetns the certificate config. -type Cert struct { - TTL string `json:"ttl" toml:"ttl" env:"MG_PROVISION_CERTS_HOURS_VALID" envDefault:"2400h"` -} - -// Config struct of Provision. -type Config struct { - File string `toml:"file" env:"MG_PROVISION_CONFIG_FILE" envDefault:"config.toml"` - Server ServiceConf `toml:"server" mapstructure:"server"` - Bootstrap Bootstrap `toml:"bootstrap" mapstructure:"bootstrap"` - Things []mgclients.Client `toml:"things" mapstructure:"things"` - Channels []groups.Group `toml:"channels" mapstructure:"channels"` - Cert Cert `toml:"cert" mapstructure:"cert"` - BSContent string `env:"MG_PROVISION_BS_CONTENT" envDefault:""` - SendTelemetry bool `env:"MG_SEND_TELEMETRY" envDefault:"true"` - InstanceID string `env:"MG_MQTT_ADAPTER_INSTANCE_ID" envDefault:""` -} - -// Save - store config in a file. -func Save(c Config, file string) error { - if file == "" { - return errors.ErrEmptyPath - } - - b, err := toml.Marshal(c) - if err != nil { - return errors.Wrap(errFailedToReadConfig, err) - } - if err := os.WriteFile(file, b, 0o644); err != nil { - return fmt.Errorf("Error writing toml: %w", err) - } - - return nil -} - -// Read - retrieve config from a file. -func Read(file string) (Config, error) { - data, err := os.ReadFile(file) - if err != nil { - return Config{}, errors.Wrap(errFailedToReadConfig, err) - } - - var c Config - if err := toml.Unmarshal(data, &c); err != nil { - return Config{}, fmt.Errorf("Error unmarshaling toml: %w", err) - } - - return c, nil -} diff --git a/provision/config_test.go b/provision/config_test.go deleted file mode 100644 index 515fe01..0000000 --- a/provision/config_test.go +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package provision_test - -import ( - "fmt" - "os" - "testing" - - mgclients "github.com/absmach/magistrala/pkg/clients" - "github.com/absmach/magistrala/pkg/errors" - "github.com/absmach/magistrala/pkg/groups" - "github.com/absmach/mg-contrib/provision" - "github.com/pelletier/go-toml" - "github.com/stretchr/testify/assert" -) - -var ( - validConfig = provision.Config{ - Server: provision.ServiceConf{ - Port: "9016", - LogLevel: "info", - TLS: false, - }, - Bootstrap: provision.Bootstrap{ - X509Provision: true, - Provision: true, - AutoWhiteList: true, - Content: map[string]interface{}{ - "test": "test", - }, - }, - Things: []mgclients.Client{ - { - ID: "1234567890", - Name: "test", - Tags: []string{"test"}, - Metadata: map[string]interface{}{ - "test": "test", - }, - Permissions: []string{"test"}, - }, - }, - Channels: []groups.Group{ - { - ID: "1234567890", - Name: "test", - Metadata: map[string]interface{}{ - "test": "test", - }, - Permissions: []string{"test"}, - }, - }, - Cert: provision.Cert{}, - SendTelemetry: true, - InstanceID: "1234567890", - } - validConfigFile = "./config.toml" - invalidConfig = provision.Config{ - Bootstrap: provision.Bootstrap{ - Content: map[string]interface{}{ - "invalid": make(chan int), - }, - }, - } - invalidConfigFile = "./invalid.toml" -) - -func createInvalidConfigFile() error { - config := map[string]interface{}{ - "invalid": "invalid", - } - b, err := toml.Marshal(config) - if err != nil { - return err - } - - f, err := os.Create(invalidConfigFile) - if err != nil { - return err - } - - if _, err = f.Write(b); err != nil { - return err - } - - return nil -} - -func createValidConfigFile() error { - b, err := toml.Marshal(validConfig) - if err != nil { - return err - } - - f, err := os.Create(validConfigFile) - if err != nil { - return err - } - - if _, err = f.Write(b); err != nil { - return err - } - - return nil -} - -func TestSave(t *testing.T) { - cases := []struct { - desc string - cfg provision.Config - file string - err error - }{ - { - desc: "save valid config", - cfg: validConfig, - file: validConfigFile, - err: nil, - }, - { - desc: "save valid config with empty file name", - cfg: validConfig, - file: "", - err: errors.ErrEmptyPath, - }, - { - desc: "save empty config with valid config file", - cfg: provision.Config{}, - file: validConfigFile, - err: nil, - }, - { - desc: "save empty config with empty file name", - cfg: provision.Config{}, - file: "", - err: errors.ErrEmptyPath, - }, - { - desc: "save invalid config", - cfg: invalidConfig, - file: invalidConfigFile, - err: errors.New("failed to read config file"), - }, - } - - for _, c := range cases { - t.Run(c.desc, func(t *testing.T) { - err := provision.Save(c.cfg, c.file) - assert.True(t, errors.Contains(err, c.err), fmt.Sprintf("expected: %v, got: %v", c.err, err)) - - if err == nil { - defer func() { - if c.file != "" { - err := os.Remove(c.file) - assert.NoError(t, err) - } - }() - - cfg, err := provision.Read(c.file) - if c.cfg.Bootstrap.Content == nil { - c.cfg.Bootstrap.Content = map[string]interface{}{} - } - assert.Equal(t, c.err, err) - assert.Equal(t, c.cfg, cfg) - } - }) - } -} - -func TestRead(t *testing.T) { - err := createInvalidConfigFile() - assert.NoError(t, err) - - err = createValidConfigFile() - assert.NoError(t, err) - - t.Cleanup(func() { - err := os.Remove(invalidConfigFile) - assert.NoError(t, err) - err = os.Remove(validConfigFile) - assert.NoError(t, err) - }) - - cases := []struct { - desc string - file string - cfg provision.Config - err error - }{ - { - desc: "read valid config", - file: validConfigFile, - cfg: validConfig, - err: nil, - }, - { - desc: "read invalid config", - file: invalidConfigFile, - cfg: invalidConfig, - err: nil, - }, - { - desc: "read empty config", - file: "", - cfg: provision.Config{}, - err: errors.New("failed to read config file"), - }, - } - - for _, c := range cases { - t.Run(c.desc, func(t *testing.T) { - cfg, err := provision.Read(c.file) - if c.desc == "read invalid config" { - c.cfg.Bootstrap.Content = nil - } - assert.True(t, errors.Contains(err, c.err), fmt.Sprintf("expected: %v, got: %v", c.err, err)) - assert.Equal(t, c.cfg, cfg) - }) - } -} diff --git a/provision/configs/config.toml b/provision/configs/config.toml deleted file mode 100644 index 38455eb..0000000 --- a/provision/configs/config.toml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) Abstract Machines -# SPDX-License-Identifier: Apache-2.0 - -file = "config.toml" - -[bootstrap] - autowhite_list = true - content = "" - provision = true - x509_provision = false - - -[server] - LogLevel = "info" - ca_certs = "" - http_port = "8190" - mg_api_key = "" - mg_bs_url = "http://localhost:9013" - mg_certs_url = "http://localhost:9019" - mg_pass = "" - mg_user = "" - mqtt_url = "" - port = "" - server_cert = "" - server_key = "" - things_location = "http://localhost:9000" - tls = true - users_location = "" - -[[things]] - name = "thing" - - [things.metadata] - external_id = "xxxxxx" - - -[[channels]] - name = "control-channel" - - [channels.metadata] - type = "control" - -[[channels]] - name = "data-channel" - - [channels.metadata] - type = "data" diff --git a/provision/doc.go b/provision/doc.go deleted file mode 100644 index e9b8552..0000000 --- a/provision/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -// Package provision contains domain concept definitions needed to support -// Provision service feature, i.e. automate provision process. -package provision diff --git a/provision/mocks/service.go b/provision/mocks/service.go deleted file mode 100644 index efa02eb..0000000 --- a/provision/mocks/service.go +++ /dev/null @@ -1,122 +0,0 @@ -// Code generated by mockery v2.43.2. DO NOT EDIT. - -// Copyright (c) Abstract Machines - -package mocks - -import ( - provision "github.com/absmach/mg-contrib/provision" - mock "github.com/stretchr/testify/mock" -) - -// Service is an autogenerated mock type for the Service type -type Service struct { - mock.Mock -} - -// Cert provides a mock function with given fields: token, thingID, duration -func (_m *Service) Cert(token string, thingID string, duration string) (string, string, error) { - ret := _m.Called(token, thingID, duration) - - if len(ret) == 0 { - panic("no return value specified for Cert") - } - - var r0 string - var r1 string - var r2 error - if rf, ok := ret.Get(0).(func(string, string, string) (string, string, error)); ok { - return rf(token, thingID, duration) - } - if rf, ok := ret.Get(0).(func(string, string, string) string); ok { - r0 = rf(token, thingID, duration) - } else { - r0 = ret.Get(0).(string) - } - - if rf, ok := ret.Get(1).(func(string, string, string) string); ok { - r1 = rf(token, thingID, duration) - } else { - r1 = ret.Get(1).(string) - } - - if rf, ok := ret.Get(2).(func(string, string, string) error); ok { - r2 = rf(token, thingID, duration) - } else { - r2 = ret.Error(2) - } - - return r0, r1, r2 -} - -// Mapping provides a mock function with given fields: token -func (_m *Service) Mapping(token string) (map[string]interface{}, error) { - ret := _m.Called(token) - - if len(ret) == 0 { - panic("no return value specified for Mapping") - } - - var r0 map[string]interface{} - var r1 error - if rf, ok := ret.Get(0).(func(string) (map[string]interface{}, error)); ok { - return rf(token) - } - if rf, ok := ret.Get(0).(func(string) map[string]interface{}); ok { - r0 = rf(token) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string]interface{}) - } - } - - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(token) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Provision provides a mock function with given fields: token, name, externalID, externalKey -func (_m *Service) Provision(token string, name string, externalID string, externalKey string) (provision.Result, error) { - ret := _m.Called(token, name, externalID, externalKey) - - if len(ret) == 0 { - panic("no return value specified for Provision") - } - - var r0 provision.Result - var r1 error - if rf, ok := ret.Get(0).(func(string, string, string, string) (provision.Result, error)); ok { - return rf(token, name, externalID, externalKey) - } - if rf, ok := ret.Get(0).(func(string, string, string, string) provision.Result); ok { - r0 = rf(token, name, externalID, externalKey) - } else { - r0 = ret.Get(0).(provision.Result) - } - - if rf, ok := ret.Get(1).(func(string, string, string, string) error); ok { - r1 = rf(token, name, externalID, externalKey) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewService creates a new instance of Service. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewService(t interface { - mock.TestingT - Cleanup(func()) -}) *Service { - mock := &Service{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/provision/service.go b/provision/service.go deleted file mode 100644 index 6e49c98..0000000 --- a/provision/service.go +++ /dev/null @@ -1,414 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package provision - -import ( - "encoding/json" - "fmt" - "log/slog" - - "github.com/absmach/magistrala/pkg/errors" - sdk "github.com/absmach/magistrala/pkg/sdk/go" -) - -const ( - externalIDKey = "external_id" - gateway = "gateway" - Active = 1 - - control = "control" - data = "data" - export = "export" -) - -var ( - ErrUnauthorized = errors.New("unauthorized access") - ErrFailedToCreateToken = errors.New("failed to create access token") - ErrEmptyThingsList = errors.New("things list in configuration empty") - ErrThingUpdate = errors.New("failed to update thing") - ErrEmptyChannelsList = errors.New("channels list in configuration is empty") - ErrFailedChannelCreation = errors.New("failed to create channel") - ErrFailedChannelRetrieval = errors.New("failed to retrieve channel") - ErrFailedThingCreation = errors.New("failed to create thing") - ErrFailedThingRetrieval = errors.New("failed to retrieve thing") - ErrMissingCredentials = errors.New("missing credentials") - ErrFailedBootstrapRetrieval = errors.New("failed to retrieve bootstrap") - ErrFailedCertCreation = errors.New("failed to create certificates") - ErrFailedBootstrap = errors.New("failed to create bootstrap config") - ErrFailedBootstrapValidate = errors.New("failed to validate bootstrap config creation") - ErrGatewayUpdate = errors.New("failed to updated gateway metadata") - - limit uint = 10 - offset uint = 0 -) - -var _ Service = (*provisionService)(nil) - -// Service specifies Provision service API. -// -//go:generate mockery --name Service --output=./mocks --filename service.go --quiet --note "Copyright (c) Abstract Machines" -type Service interface { - // Provision is the only method this API specifies. Depending on the configuration, - // the following actions will can be executed: - // - create a Thing based on external_id (eg. MAC address) - // - create multiple Channels - // - create Bootstrap configuration - // - whitelist Thing in Bootstrap configuration == connect Thing to Channels - Provision(token, name, externalID, externalKey string) (Result, error) - - // Mapping returns current configuration used for provision - // useful for using in ui to create configuration that matches - // one created with Provision method. - Mapping(token string) (map[string]interface{}, error) - - // Certs creates certificate for things that communicate over mTLS - // A duration string is a possibly signed sequence of decimal numbers, - // each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". - // Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - Cert(token, thingID, duration string) (string, string, error) -} - -type provisionService struct { - logger *slog.Logger - sdk sdk.SDK - conf Config -} - -// Result represent what is created with additional info. -type Result struct { - Things []sdk.Thing `json:"things,omitempty"` - Channels []sdk.Channel `json:"channels,omitempty"` - ClientCert map[string]string `json:"client_cert,omitempty"` - ClientKey map[string]string `json:"client_key,omitempty"` - CACert string `json:"ca_cert,omitempty"` - Whitelisted map[string]bool `json:"whitelisted,omitempty"` - Error string `json:"error,omitempty"` -} - -// New returns new provision service. -func New(cfg Config, mgsdk sdk.SDK, logger *slog.Logger) Service { - return &provisionService{ - logger: logger, - conf: cfg, - sdk: mgsdk, - } -} - -// Mapping retrieves current configuration. -func (ps *provisionService) Mapping(token string) (map[string]interface{}, error) { - pm := sdk.PageMetadata{ - Offset: uint64(offset), - Limit: uint64(limit), - } - - if _, err := ps.sdk.Users(pm, token); err != nil { - return map[string]interface{}{}, errors.Wrap(ErrUnauthorized, err) - } - - return ps.conf.Bootstrap.Content, nil -} - -// Provision is provision method for creating setup according to -// provision layout specified in config.toml. -func (ps *provisionService) Provision(token, name, externalID, externalKey string) (res Result, err error) { - var channels []sdk.Channel - var things []sdk.Thing - defer ps.recover(&err, &things, &channels, &token) - - token, err = ps.createTokenIfEmpty(token) - if err != nil { - return res, errors.Wrap(ErrFailedToCreateToken, err) - } - - if len(ps.conf.Things) == 0 { - return res, ErrEmptyThingsList - } - if len(ps.conf.Channels) == 0 { - return res, ErrEmptyChannelsList - } - for _, thing := range ps.conf.Things { - // If thing in configs contains metadata with external_id - // set value for it from the provision request - if _, ok := thing.Metadata[externalIDKey]; ok { - thing.Metadata[externalIDKey] = externalID - } - - th := sdk.Thing{ - Metadata: thing.Metadata, - } - if name == "" { - name = thing.Name - } - th.Name = name - th, err := ps.sdk.CreateThing(th, token) - if err != nil { - res.Error = err.Error() - return res, errors.Wrap(ErrFailedThingCreation, err) - } - - // Get newly created thing (in order to get the key). - th, err = ps.sdk.Thing(th.ID, token) - if err != nil { - e := errors.Wrap(err, fmt.Errorf("thing id: %s", th.ID)) - return res, errors.Wrap(ErrFailedThingRetrieval, e) - } - things = append(things, th) - } - - for _, channel := range ps.conf.Channels { - ch := sdk.Channel{ - Name: name + "_" + channel.Name, - Metadata: sdk.Metadata(channel.Metadata), - } - ch, err := ps.sdk.CreateChannel(ch, token) - if err != nil { - return res, errors.Wrap(ErrFailedChannelCreation, err) - } - ch, err = ps.sdk.Channel(ch.ID, token) - if err != nil { - e := errors.Wrap(err, fmt.Errorf("channel id: %s", ch.ID)) - return res, errors.Wrap(ErrFailedChannelRetrieval, e) - } - channels = append(channels, ch) - } - - res = Result{ - Things: things, - Channels: channels, - Whitelisted: map[string]bool{}, - ClientCert: map[string]string{}, - ClientKey: map[string]string{}, - } - - var cert sdk.Cert - var bsConfig sdk.BootstrapConfig - for _, thing := range things { - var chanIDs []string - - for _, ch := range channels { - chanIDs = append(chanIDs, ch.ID) - } - content, err := json.Marshal(ps.conf.Bootstrap.Content) - if err != nil { - return Result{}, errors.Wrap(ErrFailedBootstrap, err) - } - - if ps.conf.Bootstrap.Provision && needsBootstrap(thing) { - bsReq := sdk.BootstrapConfig{ - ThingID: thing.ID, - ExternalID: externalID, - ExternalKey: externalKey, - Channels: chanIDs, - CACert: res.CACert, - ClientCert: cert.ClientCert, - ClientKey: cert.ClientKey, - Content: string(content), - } - bsid, err := ps.sdk.AddBootstrap(bsReq, token) - if err != nil { - return Result{}, errors.Wrap(ErrFailedBootstrap, err) - } - - bsConfig, err = ps.sdk.ViewBootstrap(bsid, token) - if err != nil { - return Result{}, errors.Wrap(ErrFailedBootstrapValidate, err) - } - } - - if ps.conf.Bootstrap.X509Provision { - var cert sdk.Cert - - cert, err = ps.sdk.IssueCert(thing.ID, ps.conf.Cert.TTL, token) - if err != nil { - e := errors.Wrap(err, fmt.Errorf("thing id: %s", thing.ID)) - return res, errors.Wrap(ErrFailedCertCreation, e) - } - - res.ClientCert[thing.ID] = cert.ClientCert - res.ClientKey[thing.ID] = cert.ClientKey - res.CACert = "" - - if needsBootstrap(thing) { - if _, err = ps.sdk.UpdateBootstrapCerts(bsConfig.ThingID, cert.ClientCert, cert.ClientKey, "", token); err != nil { - return Result{}, errors.Wrap(ErrFailedCertCreation, err) - } - } - } - - if ps.conf.Bootstrap.AutoWhiteList { - if err := ps.sdk.Whitelist(thing.ID, Active, token); err != nil { - res.Error = err.Error() - return res, ErrThingUpdate - } - res.Whitelisted[thing.ID] = true - } - } - - if err = ps.updateGateway(token, bsConfig, channels); err != nil { - return res, err - } - return res, nil -} - -func (ps *provisionService) Cert(token, thingID, ttl string) (string, string, error) { - token, err := ps.createTokenIfEmpty(token) - if err != nil { - return "", "", errors.Wrap(ErrFailedToCreateToken, err) - } - - th, err := ps.sdk.Thing(thingID, token) - if err != nil { - return "", "", errors.Wrap(ErrUnauthorized, err) - } - cert, err := ps.sdk.IssueCert(th.ID, ps.conf.Cert.TTL, token) - return cert.ClientCert, cert.ClientKey, err -} - -func (ps *provisionService) createTokenIfEmpty(token string) (string, error) { - if token != "" { - return token, nil - } - - // If no token in request is provided - // use API key provided in config file or env - if ps.conf.Server.MgAPIKey != "" { - return ps.conf.Server.MgAPIKey, nil - } - - // If no API key use username and password provided to create access token. - if ps.conf.Server.MgUser == "" || ps.conf.Server.MgPass == "" { - return token, ErrMissingCredentials - } - - u := sdk.Login{ - Identity: ps.conf.Server.MgUser, - Secret: ps.conf.Server.MgPass, - DomainID: ps.conf.Server.MgDomainID, - } - tkn, err := ps.sdk.CreateToken(u) - if err != nil { - return token, errors.Wrap(ErrFailedToCreateToken, err) - } - - return tkn.AccessToken, nil -} - -func (ps *provisionService) updateGateway(token string, bs sdk.BootstrapConfig, channels []sdk.Channel) error { - var gw Gateway - for _, ch := range channels { - switch ch.Metadata["type"] { - case control: - gw.CtrlChannelID = ch.ID - case data: - gw.DataChannelID = ch.ID - case export: - gw.ExportChannelID = ch.ID - } - } - gw.ExternalID = bs.ExternalID - gw.ExternalKey = bs.ExternalKey - gw.CfgID = bs.ThingID - gw.Type = gateway - - th, sdkerr := ps.sdk.Thing(bs.ThingID, token) - if sdkerr != nil { - return errors.Wrap(ErrGatewayUpdate, sdkerr) - } - b, err := json.Marshal(gw) - if err != nil { - return errors.Wrap(ErrGatewayUpdate, err) - } - if err := json.Unmarshal(b, &th.Metadata); err != nil { - return errors.Wrap(ErrGatewayUpdate, err) - } - if _, err := ps.sdk.UpdateThing(th, token); err != nil { - return errors.Wrap(ErrGatewayUpdate, err) - } - return nil -} - -func (ps *provisionService) errLog(err error) { - if err != nil { - ps.logger.Error(fmt.Sprintf("Error recovering: %s", err)) - } -} - -func clean(ps *provisionService, things []sdk.Thing, channels []sdk.Channel, token string) { - for _, t := range things { - err := ps.sdk.DeleteThing(t.ID, token) - ps.errLog(err) - } - for _, c := range channels { - err := ps.sdk.DeleteChannel(c.ID, token) - ps.errLog(err) - } -} - -func (ps *provisionService) recover(e *error, ths *[]sdk.Thing, chs *[]sdk.Channel, tkn *string) { - if e == nil { - return - } - things, channels, token, err := *ths, *chs, *tkn, *e - - if errors.Contains(err, ErrFailedThingRetrieval) || errors.Contains(err, ErrFailedChannelCreation) { - for _, th := range things { - err := ps.sdk.DeleteThing(th.ID, token) - ps.errLog(err) - } - return - } - - if errors.Contains(err, ErrFailedBootstrap) || errors.Contains(err, ErrFailedChannelRetrieval) { - clean(ps, things, channels, token) - return - } - - if errors.Contains(err, ErrFailedBootstrapValidate) || errors.Contains(err, ErrFailedCertCreation) { - clean(ps, things, channels, token) - for _, th := range things { - if needsBootstrap(th) { - ps.errLog(ps.sdk.RemoveBootstrap(th.ID, token)) - } - } - return - } - - if errors.Contains(err, ErrFailedBootstrapValidate) || errors.Contains(err, ErrFailedCertCreation) { - clean(ps, things, channels, token) - for _, th := range things { - if needsBootstrap(th) { - bs, err := ps.sdk.ViewBootstrap(th.ID, token) - ps.errLog(errors.Wrap(ErrFailedBootstrapRetrieval, err)) - ps.errLog(ps.sdk.RemoveBootstrap(bs.ThingID, token)) - } - } - } - - if errors.Contains(err, ErrThingUpdate) || errors.Contains(err, ErrGatewayUpdate) { - clean(ps, things, channels, token) - for _, th := range things { - if ps.conf.Bootstrap.X509Provision && needsBootstrap(th) { - _, err := ps.sdk.RevokeCert(th.ID, token) - ps.errLog(err) - } - if needsBootstrap(th) { - bs, err := ps.sdk.ViewBootstrap(th.ID, token) - ps.errLog(errors.Wrap(ErrFailedBootstrapRetrieval, err)) - ps.errLog(ps.sdk.RemoveBootstrap(bs.ThingID, token)) - } - } - return - } -} - -func needsBootstrap(th sdk.Thing) bool { - if th.Metadata == nil { - return false - } - - if _, ok := th.Metadata[externalIDKey]; ok { - return true - } - return false -} diff --git a/provision/service_test.go b/provision/service_test.go deleted file mode 100644 index c903e00..0000000 --- a/provision/service_test.go +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package provision_test - -import ( - "fmt" - "testing" - - mglog "github.com/absmach/magistrala/logger" - "github.com/absmach/magistrala/pkg/errors" - repoerr "github.com/absmach/magistrala/pkg/errors/repository" - svcerr "github.com/absmach/magistrala/pkg/errors/service" - sdk "github.com/absmach/magistrala/pkg/sdk/go" - sdkmocks "github.com/absmach/magistrala/pkg/sdk/mocks" - "github.com/absmach/mg-contrib/pkg/testsutil" - "github.com/absmach/mg-contrib/provision" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -var validToken = "valid" - -func TestMapping(t *testing.T) { - mgsdk := new(sdkmocks.SDK) - svc := provision.New(validConfig, mgsdk, mglog.NewMock()) - - cases := []struct { - desc string - token string - content map[string]interface{} - sdkerr error - err error - }{ - { - desc: "valid token", - token: validToken, - content: validConfig.Bootstrap.Content, - sdkerr: nil, - err: nil, - }, - { - desc: "invalid token", - token: "invalid", - content: map[string]interface{}{}, - sdkerr: errors.NewSDKErrorWithStatus(svcerr.ErrAuthentication, 401), - err: provision.ErrUnauthorized, - }, - } - - for _, c := range cases { - t.Run(c.desc, func(t *testing.T) { - pm := sdk.PageMetadata{Offset: uint64(0), Limit: uint64(10)} - repocall := mgsdk.On("Users", pm, c.token).Return(sdk.UsersPage{}, c.sdkerr) - content, err := svc.Mapping(c.token) - assert.True(t, errors.Contains(err, c.err), fmt.Sprintf("expected error %v, got %v", c.err, err)) - assert.Equal(t, c.content, content) - repocall.Unset() - }) - } -} - -func TestCert(t *testing.T) { - cases := []struct { - desc string - config provision.Config - token string - thingID string - ttl string - cert string - key string - sdkThingErr error - sdkCertErr error - sdkTokenErr error - err error - }{ - { - desc: "valid", - config: validConfig, - token: validToken, - thingID: testsutil.GenerateUUID(t), - ttl: "1h", - cert: "cert", - key: "key", - sdkThingErr: nil, - sdkCertErr: nil, - sdkTokenErr: nil, - err: nil, - }, - { - desc: "empty token with config API key", - config: provision.Config{ - Server: provision.ServiceConf{MgAPIKey: "key"}, - Cert: provision.Cert{TTL: "1h"}, - }, - token: "", - thingID: testsutil.GenerateUUID(t), - ttl: "1h", - cert: "cert", - key: "key", - sdkThingErr: nil, - sdkCertErr: nil, - sdkTokenErr: nil, - err: nil, - }, - { - desc: "empty token with username and password", - config: provision.Config{ - Server: provision.ServiceConf{ - MgUser: "test@example.com", - MgPass: "12345678", - MgDomainID: testsutil.GenerateUUID(t), - }, - Cert: provision.Cert{TTL: "1h"}, - }, - token: "", - thingID: testsutil.GenerateUUID(t), - ttl: "1h", - cert: "cert", - key: "key", - sdkThingErr: nil, - sdkCertErr: nil, - sdkTokenErr: nil, - err: nil, - }, - { - desc: "empty token with username and invalid password", - config: provision.Config{ - Server: provision.ServiceConf{ - MgUser: "test@example.com", - MgPass: "12345678", - MgDomainID: testsutil.GenerateUUID(t), - }, - Cert: provision.Cert{TTL: "1h"}, - }, - token: "", - thingID: testsutil.GenerateUUID(t), - ttl: "1h", - cert: "", - key: "", - sdkThingErr: nil, - sdkCertErr: nil, - sdkTokenErr: errors.NewSDKErrorWithStatus(svcerr.ErrAuthentication, 401), - err: provision.ErrFailedToCreateToken, - }, - { - desc: "empty token with empty username and password", - config: provision.Config{ - Server: provision.ServiceConf{}, - Cert: provision.Cert{TTL: "1h"}, - }, - token: "", - thingID: testsutil.GenerateUUID(t), - ttl: "1h", - cert: "", - key: "", - sdkThingErr: nil, - sdkCertErr: nil, - sdkTokenErr: nil, - err: provision.ErrMissingCredentials, - }, - { - desc: "invalid thingID", - config: validConfig, - token: "invalid", - thingID: testsutil.GenerateUUID(t), - ttl: "1h", - cert: "", - key: "", - sdkThingErr: errors.NewSDKErrorWithStatus(svcerr.ErrAuthentication, 401), - sdkCertErr: nil, - sdkTokenErr: nil, - err: provision.ErrUnauthorized, - }, - { - desc: "invalid thingID", - config: validConfig, - token: validToken, - thingID: "invalid", - ttl: "1h", - cert: "", - key: "", - sdkThingErr: errors.NewSDKErrorWithStatus(repoerr.ErrNotFound, 404), - sdkCertErr: nil, - sdkTokenErr: nil, - err: provision.ErrUnauthorized, - }, - { - desc: "failed to issue cert", - config: validConfig, - token: validToken, - thingID: testsutil.GenerateUUID(t), - ttl: "1h", - cert: "", - key: "", - sdkThingErr: nil, - sdkTokenErr: nil, - sdkCertErr: errors.NewSDKError(repoerr.ErrCreateEntity), - err: repoerr.ErrCreateEntity, - }, - } - - for _, c := range cases { - t.Run(c.desc, func(t *testing.T) { - mgsdk := new(sdkmocks.SDK) - svc := provision.New(c.config, mgsdk, mglog.NewMock()) - - mgsdk.On("Thing", c.thingID, mock.Anything).Return(sdk.Thing{ID: c.thingID}, c.sdkThingErr) - mgsdk.On("IssueCert", c.thingID, c.config.Cert.TTL, mock.Anything).Return(sdk.Cert{ClientCert: c.cert, ClientKey: c.key}, c.sdkCertErr) - login := sdk.Login{ - Identity: c.config.Server.MgUser, - Secret: c.config.Server.MgPass, - DomainID: c.config.Server.MgDomainID, - } - mgsdk.On("CreateToken", login).Return(sdk.Token{AccessToken: validToken}, c.sdkTokenErr) - cert, key, err := svc.Cert(c.token, c.thingID, c.ttl) - assert.Equal(t, c.cert, cert) - assert.Equal(t, c.key, key) - assert.True(t, errors.Contains(err, c.err), fmt.Sprintf("expected error %v, got %v", c.err, err)) - }) - } -} diff --git a/readers/influxdb/messages.go b/readers/influxdb/messages.go index bd11c9b..68bbe3b 100644 --- a/readers/influxdb/messages.go +++ b/readers/influxdb/messages.go @@ -18,10 +18,8 @@ import ( influxdb2 "github.com/influxdata/influxdb-client-go/v2" ) -const ( - // Measurement for SenML messages. - defMeasurement = "messages" -) +// Measurement for SenML messages. +const defMeasurement = "messages" var _ readers.MessageRepository = (*influxRepository)(nil) diff --git a/readers/messages.go b/readers/messages.go deleted file mode 100644 index 19ce1c0..0000000 --- a/readers/messages.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) Abstract Machines -// SPDX-License-Identifier: Apache-2.0 - -package readers - -import "errors" - -const ( - // EqualKey represents the equal comparison operator key. - EqualKey = "eq" - // LowerThanKey represents the lower-than comparison operator key. - LowerThanKey = "lt" - // LowerThanEqualKey represents the lower-than-or-equal comparison operator key. - LowerThanEqualKey = "le" - // GreaterThanKey represents the greater-than-or-equal comparison operator key. - GreaterThanKey = "gt" - // GreaterThanEqualKey represents the greater-than-or-equal comparison operator key. - GreaterThanEqualKey = "ge" -) - -// ErrReadMessages indicates failure occurred while reading messages from database. -var ErrReadMessages = errors.New("failed to read messages from database") - -// MessageRepository specifies message reader API. -// -//go:generate mockery --name MessageRepository --output=./mocks --filename messages.go --quiet --note "Copyright (c) Abstract Machines" -type MessageRepository interface { - // ReadAll skips given number of messages for given channel and returns next - // limited number of messages. - ReadAll(chanID string, pm PageMetadata) (MessagesPage, error) -} - -// Message represents any message format. -type Message interface{} - -// MessagesPage contains page related metadata as well as list of messages that -// belong to this page. -type MessagesPage struct { - PageMetadata - Total uint64 - Messages []Message -} - -// PageMetadata represents the parameters used to create database queries. -type PageMetadata struct { - Offset uint64 `json:"offset"` - Limit uint64 `json:"limit"` - Subtopic string `json:"subtopic,omitempty"` - Publisher string `json:"publisher,omitempty"` - Protocol string `json:"protocol,omitempty"` - Name string `json:"name,omitempty"` - Value float64 `json:"v,omitempty"` - Comparator string `json:"comparator,omitempty"` - BoolValue bool `json:"vb,omitempty"` - StringValue string `json:"vs,omitempty"` - DataValue string `json:"vd,omitempty"` - From float64 `json:"from,omitempty"` - To float64 `json:"to,omitempty"` - Format string `json:"format,omitempty"` - Aggregation string `json:"aggregation,omitempty"` - Interval string `json:"interval,omitempty"` -} - -// ParseValueComparator convert comparison operator keys into mathematic anotation. -func ParseValueComparator(query map[string]interface{}) string { - comparator := "=" - val, ok := query["comparator"] - if ok { - switch val.(string) { - case EqualKey: - comparator = "=" - case LowerThanKey: - comparator = "<" - case LowerThanEqualKey: - comparator = "<=" - case GreaterThanKey: - comparator = ">" - case GreaterThanEqualKey: - comparator = ">=" - } - } - - return comparator -} diff --git a/scripts/ci.sh b/scripts/ci.sh new file mode 100755 index 0000000..74fb80d --- /dev/null +++ b/scripts/ci.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +# This script contains commands to be executed by the CI tool. +NPROC=$(nproc) +GO_VERSION=1.22.4 +PROTOC_VERSION=27.1 +PROTOC_GEN_VERSION=v1.34.2 +PROTOC_GRPC_VERSION=v1.4.0 +GOLANGCI_LINT_VERSION=v1.59.1 + +function version_gt() { test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; } + +update_go() { + CURRENT_GO_VERSION=$(go version | sed 's/[^0-9.]*\([0-9.]*\).*/\1/') + if version_gt $GO_VERSION $CURRENT_GO_VERSION; then + echo "Updating go version from $CURRENT_GO_VERSION to $GO_VERSION ..." + # remove other Go version from path + sudo rm -rf /usr/bin/go + sudo rm -rf /usr/local/go + sudo rm -rf /usr/local/bin/go + sudo rm -rf /usr/local/golang + sudo rm -rf $GOROOT $GOPAT $GOBIN + wget https://go.dev/dl/go$GO_VERSION.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf go$GO_VERSION.linux-amd64.tar.gz + export GOROOT=/usr/local/go + export PATH=$PATH:/usr/local/go/bin + fi + export GOBIN=$HOME/go/bin + export PATH=$PATH:$GOBIN + go version +} + +setup_protoc() { + # Execute `go get` for protoc dependencies outside of project dir. + echo "Setting up protoc..." + PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip + curl -0L https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP -o $PROTOC_ZIP + unzip -o $PROTOC_ZIP -d protoc3 + sudo mv protoc3/bin/* /usr/local/bin/ + sudo mv protoc3/include/* /usr/local/include/ + rm -rf $PROTOC_ZIP protoc3 + + go install google.golang.org/protobuf/cmd/protoc-gen-go@$PROTOC_GEN_VERSION + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@$PROTOC_GRPC_VERSION + + export PATH=$PATH:/usr/local/bin/protoc +} + +setup_mg() { + echo "Setting up Magistrala..." + for p in $(ls *.pb.go); do + mv $p $p.tmp + done + for p in $(ls pkg/*/*.pb.go); do + mv $p $p.tmp + done + make proto + for p in $(ls *.pb.go); do + if ! cmp -s $p $p.tmp; then + echo "Proto file and generated Go file $p are out of sync!" + exit 1 + fi + done + for p in $(ls pkg/*/*.pb.go); do + if ! cmp -s $p $p.tmp; then + echo "Proto file and generated Go file $p are out of sync!" + exit 1 + fi + done + echo "Compile check for rabbitmq..." + MG_MESSAGE_BROKER_TYPE=rabbitmq make http + echo "Compile check for redis..." + MG_ES_TYPE=redis make http + make -j$NPROC +} + +setup_lint() { + # binary will be $(go env GOBIN)/golangci-lint + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOBIN) $GOLANGCI_LINT_VERSION +} + +setup() { + echo "Setting up..." + update_go + setup_protoc + setup_mg + setup_lint +} + +run_test() { + echo "Running lint..." + golangci-lint run + echo "Running tests..." + echo "" > coverage.txt + for d in $(go list ./... | grep -v 'vendor\|cmd'); do + GOCACHE=off + go test -mod=vendor -v -race -tags test -coverprofile=profile.out -covermode=atomic $d + if [ -f profile.out ]; then + cat profile.out >> coverage.txt + rm profile.out + fi + done +} + +push() { + if test -n "$BRANCH_NAME" && test "$BRANCH_NAME" = "master"; then + echo "Pushing Docker images..." + make -j$NPROC latest + fi +} + +set -e +setup +run_test +push diff --git a/scripts/csv/channels.csv b/scripts/csv/channels.csv new file mode 100644 index 0000000..9b367f7 --- /dev/null +++ b/scripts/csv/channels.csv @@ -0,0 +1,3 @@ +channel_1 +channel_2 +channel_3 diff --git a/scripts/csv/things.csv b/scripts/csv/things.csv new file mode 100644 index 0000000..4636a47 --- /dev/null +++ b/scripts/csv/things.csv @@ -0,0 +1,10 @@ +thing_1 +thing_2 +thing_3 +thing_4 +thing_5 +thing_6 +thing_7 +thing_8 +thing_9 +thing_10 diff --git a/scripts/provision-dev.sh b/scripts/provision-dev.sh new file mode 100755 index 0000000..49b5080 --- /dev/null +++ b/scripts/provision-dev.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 +# + +### +# Provisions example user, thing and channel on a clean Magistrala installation. +# +# Expects a running Magistrala installation. +# +# +### + +if [ $# -lt 4 ] +then + echo "Usage: $0 user_email user_password device_name channel_name" + exit 1 +fi + +EMAIL=$1 +PASSWORD=$2 +DEVICE=$3 +CHANNEL=$4 + +#provision user: +printf "Provisoning user with email $EMAIL and password $PASSWORD \n" +curl -s -S --cacert docker/ssl/certs/magistrala-server.crt --insecure -X POST -H "Content-Type: application/json" https://localhost/users -d '{"credentials": {"identity": "'"$EMAIL"'","secret": "'"$PASSWORD"'"}, "status": "enabled", "role": "admin" }' + +#get jwt token +JWTTOKEN=$(curl -s -S --cacert docker/ssl/certs/magistrala-server.crt --insecure -X POST -H "Content-Type: application/json" https://localhost/users/tokens/issue -d '{"identity":"'"$EMAIL"'", "secret":"'"$PASSWORD"'"}' | grep -oP '"access_token":"\K[^"]+' ) +printf "JWT TOKEN for user is $JWTTOKEN \n" + +#provision thing +printf "Provisioning thing with name $DEVICE \n" +DEVICEID=$(curl -s -S --cacert docker/ssl/certs/magistrala-server.crt --insecure -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $JWTTOKEN" https://localhost/things -d '{"name":"'"$DEVICE"'", "status": "enabled"}' | grep -oP '"id":"\K[^"]+' ) +curl -s -S --cacert docker/ssl/certs/magistrala-server.crt --insecure -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $JWTTOKEN" https://localhost/things/$DEVICEID + +#get thing token +DEVICETOKEN=$(curl -s -S --cacert docker/ssl/certs/magistrala-server.crt --insecure -H "Authorization: Bearer $JWTTOKEN" https://localhost/things/$DEVICEID | grep -oP '"secret":"\K[^"]+' ) +printf "Device token is $DEVICETOKEN \n" + +#provision channel +printf "Provisioning channel with name $CHANNEL \n" +CHANNELID=$(curl -s -S --cacert docker/ssl/certs/magistrala-server.crt --insecure -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $JWTTOKEN" https://localhost/channels -d '{"name":"'"$CHANNEL"'", "status": "enabled"}' | grep -oP '"id":"\K[^"]+' ) +curl -s -S --cacert docker/ssl/certs/magistrala-server.crt --insecure -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $JWTTOKEN" https://localhost/channels/$CHANNELID + +#connect thing to channel +printf "Connecting thing of id $DEVICEID to channel of id $CHANNELID \n" +curl -s -S --cacert docker/ssl/certs/magistrala-server.crt --insecure -X PUT -H "Authorization: Bearer $JWTTOKEN" https://localhost/channels/$CHANNELID/things/$DEVICEID diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100755 index 0000000..8e0097a --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +### +# Runs all Magistrala microservices (must be previously built and installed). +# +# Expects that PostgreSQL and needed messaging DB are alredy running. +# Additionally, MQTT microservice demands that Redis is up and running. +# +### + +BUILD_DIR=../build + +# Kill all magistrala-* stuff +function cleanup { + pkill magistrala + pkill nats +} + +### +# NATS +### +nats-server & +counter=1 +until fuser 4222/tcp 1>/dev/null 2>&1; +do + sleep 0.5 + ((counter++)) + if [ ${counter} -gt 10 ] + then + echo "NATS failed to start in 5 sec, exiting" + exit 1 + fi + echo "Waiting for NATS server" +done + +### +# Users +### +MG_USERS_LOG_LEVEL=info MG_USERS_HTTP_PORT=9002 MG_USERS_GRPC_PORT=7001 MG_USERS_ADMIN_EMAIL=admin@magistrala.com MG_USERS_ADMIN_PASSWORD=12345678 MG_EMAIL_TEMPLATE=../docker/templates/users.tmpl $BUILD_DIR/magistrala-users & + +### +# Things +### +MG_THINGS_LOG_LEVEL=info MG_THINGS_HTTP_PORT=9000 MG_THINGS_AUTH_GRPC_PORT=7000 MG_THINGS_AUTH_HTTP_PORT=9002 $BUILD_DIR/magistrala-things & + +### +# HTTP +### +MG_HTTP_ADAPTER_LOG_LEVEL=info MG_HTTP_ADAPTER_PORT=8008 MG_THINGS_AUTH_GRPC_URL=localhost:7000 $BUILD_DIR/magistrala-http & + +### +# WS +### +MG_WS_ADAPTER_LOG_LEVEL=info MG_WS_ADAPTER_HTTP_PORT=8190 MG_THINGS_AUTH_GRPC_URL=localhost:7000 $BUILD_DIR/magistrala-ws & + +### +# MQTT +### +MG_MQTT_ADAPTER_LOG_LEVEL=info MG_THINGS_AUTH_GRPC_URL=localhost:7000 $BUILD_DIR/magistrala-mqtt & + +### +# CoAP +### +MG_COAP_ADAPTER_LOG_LEVEL=info MG_COAP_ADAPTER_PORT=5683 MG_THINGS_AUTH_GRPC_URL=localhost:7000 $BUILD_DIR/magistrala-coap & + +trap cleanup EXIT + +while : ; do sleep 1 ; done diff --git a/certs/mocks/doc.go b/tools/doc.go similarity index 52% rename from certs/mocks/doc.go rename to tools/doc.go index 16ed198..296a4b2 100644 --- a/certs/mocks/doc.go +++ b/tools/doc.go @@ -1,5 +1,5 @@ // Copyright (c) Abstract Machines // SPDX-License-Identifier: Apache-2.0 -// Package mocks contains mocks for testing purposes. -package mocks +// Package tools contains tools for Magistrala. +package tools diff --git a/tools/e2e/Makefile b/tools/e2e/Makefile new file mode 100644 index 0000000..fd27a8a --- /dev/null +++ b/tools/e2e/Makefile @@ -0,0 +1,15 @@ +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +PROGRAM = e2e +SOURCES = $(wildcard *.go) cmd/main.go + +all: $(PROGRAM) + +.PHONY: all clean + +$(PROGRAM): $(SOURCES) + go build -ldflags "-s -w" -o $@ cmd/main.go + +clean: + rm -rf $(PROGRAM) diff --git a/tools/e2e/README.md b/tools/e2e/README.md new file mode 100644 index 0000000..6e35845 --- /dev/null +++ b/tools/e2e/README.md @@ -0,0 +1,93 @@ +# Magistrala Users Groups Things and Channels E2E Testing Tool + +A simple utility to create a list of groups and users connected to these groups and channels and things connected to these channels. + +## Installation + +```bash +cd tools/e2e +make +``` + +### Usage + +```bash +./e2e --help +Tool for testing end-to-end flow of Magistrala by doing a couple of operations namely: +1. Creating, viewing, updating and changing status of users, groups, things and channels. +2. Connecting users and groups to each other and things and channels to each other. +3. Sending messages from things to channels on all 4 protocol adapters (HTTP, WS, CoAP and MQTT). +Complete documentation is available at https://docs.magistrala.abstractmachines.fr + + +Usage: + + e2e [flags] + + +Examples: + +Here is a simple example of using e2e tool. +Use the following commands from the root Magistrala directory: + +go run tools/e2e/cmd/main.go +go run tools/e2e/cmd/main.go --host 142.93.118.47 +go run tools/e2e/cmd/main.go --host localhost --num 10 --num_of_messages 100 --prefix e2e + + +Flags: + + -h, --help help for e2e + -H, --host string address for a running Magistrala instance (default "localhost") + -n, --num uint number of users, groups, channels and things to create and connect (default 10) + -N, --num_of_messages uint number of messages to send (default 10) + -p, --prefix string name prefix for users, groups, things and channels +``` + +To use `-H` option, you can specify the address for the Magistrala instance as an argument when running the program. For example, if the Magistrala instance is running on another computer with the IP address 192.168.0.1, you could use the following command: + +```bash +go run tools/e2e/cmd/main.go --host 142.93.118.47 +``` + +This will tell the program to connect to the Magistrala instance running on the specified IP address. + +If you want to create a list of channels with certificates: + +```bash +go run tools/e2e/cmd/main.go --host localhost --num 10 --num_of_messages 100 --prefix e2e +``` + +Example of output: + +```bash +created user with token eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2ODEyMDYwMjMsImlhdCI6MTY4MTIwNTEyMywiaWRlbnRpdHkiOiJlMmUtbGF0ZS1zaWxlbmNlQGVtYWlsLmNvbSIsImlzcyI6ImNsaWVudHMuYXV0aCIsInN1YiI6IjdlZDIyY2IyLTRlMzQtNDhiZi04Y2RlLTIxMjZiYzYyYzY4MyIsInR5cGUiOiJhY2Nlc3MifQ.AdExNYs5mVQNpo_ejJDq7KTC5dKkZWmgM9FJvTM2T_GM2LE9ASQv0ymC4wS3PDXKWf-OcaR8DJIxE6WiG3fztQ +created users of ids: +9e87bc1d-0889-4252-a3df-36e02edfc859 +c1e4901a-fb7f-45e9-b934-c55194b1d028 +c341a9cb-542b-4c3b-afd6-c98e04ed5e7e +8cfc886b-21fa-4205-80b4-3601827b94ff +334984d7-30eb-4b06-92b8-5ec182bebac5 +created groups of ids: +7744ec55-c767-4137-be96-0d79699772a4 +c8fe4d9d-3ad6-4687-83c0-171356f3e4f6 +513f7295-0923-4e21-b41a-3cfd1cb7b9b9 +54bd71ea-3c22-401e-89ea-d58162b983c0 +ae91b327-4c40-4e68-91fe-cd6223ee4e99 +created things of ids: +5909a907-7413-47d4-b793-e1eb36988a5f +f9b6bc18-1862-4a24-8973-adde11cb3303 +c2bd6eed-6f38-464c-989c-fe8ec8c084ba +8c76702c-0534-4246-8ed7-21816b4f91cf +25005ca8-e886-465f-9cd1-4f3c4a95c6c1 +created channels of ids: +ebb0e5f3-2241-4770-a7cc-f4bbd06134ca +d654948d-d6c1-4eae-b69a-29c853282c3d +2c2a5496-89cf-47e6-9d38-5fd5542337bd +7ab3319d-269c-4b07-9dc5-f9906693e894 +5d8fa139-10e7-4683-94f3-4e881b4db041 +created policies for users, groups, things and channels +viewed users, groups, things and channels +updated users, groups, things and channels +sent messages to channels +``` diff --git a/tools/e2e/cmd/main.go b/tools/e2e/cmd/main.go new file mode 100644 index 0000000..5574382 --- /dev/null +++ b/tools/e2e/cmd/main.go @@ -0,0 +1,58 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package main contains e2e tool for testing Magistrala. +package main + +import ( + "log" + + "github.com/absmach/magistrala/tools/e2e" + cc "github.com/ivanpirog/coloredcobra" + "github.com/spf13/cobra" +) + +const defNum = uint64(10) + +func main() { + econf := e2e.Config{} + + rootCmd := &cobra.Command{ + Use: "e2e", + Short: "e2e is end-to-end testing tool for Magistrala", + Long: "Tool for testing end-to-end flow of magistrala by doing a couple of operations namely:\n" + + "1. Creating, viewing, updating and changing status of users, groups, things and channels.\n" + + "2. Connecting users and groups to each other and things and channels to each other.\n" + + "3. Sending messages from things to channels on all 4 protocol adapters (HTTP, WS, CoAP and MQTT).\n" + + "Complete documentation is available at https://docs.magistrala.abstractmachines.fr", + Example: "Here is a simple example of using e2e tool.\n" + + "Use the following commands from the root magistrala directory:\n\n" + + "go run tools/e2e/cmd/main.go\n" + + "go run tools/e2e/cmd/main.go --host 142.93.118.47\n" + + "go run tools/e2e/cmd/main.go --host localhost --num 10 --num_of_messages 100 --prefix e2e", + Run: func(_ *cobra.Command, _ []string) { + e2e.Test(econf) + }, + } + + cc.Init(&cc.Config{ + RootCmd: rootCmd, + Headings: cc.HiCyan + cc.Bold + cc.Underline, + CmdShortDescr: cc.Magenta, + Example: cc.Italic + cc.Magenta, + ExecName: cc.Bold, + Flags: cc.HiGreen + cc.Bold, + FlagsDescr: cc.Green, + FlagsDataType: cc.White + cc.Italic, + }) + + // Root Flags + rootCmd.PersistentFlags().StringVarP(&econf.Host, "host", "H", "localhost", "address for a running magistrala instance") + rootCmd.PersistentFlags().StringVarP(&econf.Prefix, "prefix", "p", "", "name prefix for users, groups, things and channels") + rootCmd.PersistentFlags().Uint64VarP(&econf.Num, "num", "n", defNum, "number of users, groups, channels and things to create and connect") + rootCmd.PersistentFlags().Uint64VarP(&econf.NumOfMsg, "num_of_messages", "N", defNum, "number of messages to send") + + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) + } +} diff --git a/tools/e2e/doc.go b/tools/e2e/doc.go new file mode 100644 index 0000000..eb7fb08 --- /dev/null +++ b/tools/e2e/doc.go @@ -0,0 +1,5 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package e2e contains entry point for end-to-end tests. +package e2e diff --git a/tools/e2e/e2e.go b/tools/e2e/e2e.go new file mode 100644 index 0000000..e72c125 --- /dev/null +++ b/tools/e2e/e2e.go @@ -0,0 +1,628 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "fmt" + "math/rand" + "net/http" + "os" + "os/exec" + "reflect" + "strings" + "time" + + "github.com/0x6flab/namegenerator" + sdk "github.com/absmach/magistrala/pkg/sdk/go" + "github.com/gookit/color" + "github.com/gorilla/websocket" + "golang.org/x/sync/errgroup" +) + +const ( + defPass = "12345678" + defWSPort = "8186" + numAdapters = 4 + batchSize = 99 + usersPort = "9002" + thingsPort = "9000" + domainsPort = "8189" +) + +var ( + namesgenerator = namegenerator.NewGenerator() + msgFormat = `[{"bn":"demo", "bu":"V", "t": %d, "bver":5, "n":"voltage", "u":"V", "v":%d}]` +) + +// Config - test configuration. +type Config struct { + Host string + Num uint64 + NumOfMsg uint64 + SSL bool + CA string + CAKey string + Prefix string +} + +// Test - function that does actual end to end testing. +// The operations are: +// - Create a user +// - Create other users +// - Do Read, Update and Change of Status operations on users. + +// - Create groups using hierarchy +// - Do Read, Update and Change of Status operations on groups. + +// - Create things +// - Do Read, Update and Change of Status operations on things. + +// - Create channels +// - Do Read, Update and Change of Status operations on channels. + +// - Connect thing to channel +// - Publish message from HTTP, MQTT, WS and CoAP Adapters. +func Test(conf Config) { + sdkConf := sdk.Config{ + ThingsURL: fmt.Sprintf("http://%s:%s", conf.Host, thingsPort), + UsersURL: fmt.Sprintf("http://%s:%s", conf.Host, usersPort), + DomainsURL: fmt.Sprintf("http://%s:%s", conf.Host, domainsPort), + HTTPAdapterURL: fmt.Sprintf("http://%s/http", conf.Host), + MsgContentType: sdk.CTJSONSenML, + TLSVerification: false, + } + + s := sdk.NewSDK(sdkConf) + + magenta := color.FgLightMagenta.Render + + token, err := createUser(s, conf) + if err != nil { + errExit(fmt.Errorf("unable to create user: %w", err)) + } + color.Success.Printf("created user with token %s\n", magenta(token)) + + users, err := createUsers(s, conf, token) + if err != nil { + errExit(fmt.Errorf("unable to create users: %w", err)) + } + color.Success.Printf("created users of ids:\n%s\n", magenta(getIDS(users))) + + groups, err := createGroups(s, conf, token) + if err != nil { + errExit(fmt.Errorf("unable to create groups: %w", err)) + } + color.Success.Printf("created groups of ids:\n%s\n", magenta(getIDS(groups))) + + things, err := createThings(s, conf, token) + if err != nil { + errExit(fmt.Errorf("unable to create things: %w", err)) + } + color.Success.Printf("created things of ids:\n%s\n", magenta(getIDS(things))) + + channels, err := createChannels(s, conf, token) + if err != nil { + errExit(fmt.Errorf("unable to create channels: %w", err)) + } + color.Success.Printf("created channels of ids:\n%s\n", magenta(getIDS(channels))) + + // List users, groups, things and channels + if err := read(s, conf, token, users, groups, things, channels); err != nil { + errExit(fmt.Errorf("unable to read users, groups, things and channels: %w", err)) + } + color.Success.Println("viewed users, groups, things and channels") + + // Update users, groups, things and channels + if err := update(s, token, users, groups, things, channels); err != nil { + errExit(fmt.Errorf("unable to update users, groups, things and channels: %w", err)) + } + color.Success.Println("updated users, groups, things and channels") + + // Send messages to channels + if err := messaging(s, conf, token, things, channels); err != nil { + errExit(fmt.Errorf("unable to send messages to channels: %w", err)) + } + color.Success.Println("sent messages to channels") +} + +func errExit(err error) { + color.Error.Println(err.Error()) + os.Exit(1) +} + +func createUser(s sdk.SDK, conf Config) (string, error) { + user := sdk.User{ + Name: fmt.Sprintf("%s%s", conf.Prefix, namesgenerator.Generate()), + Credentials: sdk.Credentials{ + Identity: fmt.Sprintf("%s%s@email.com", conf.Prefix, namesgenerator.Generate()), + Secret: defPass, + }, + Status: sdk.EnabledStatus, + Role: "admin", + } + + if _, err := s.CreateUser(user, ""); err != nil { + return "", fmt.Errorf("unable to create user: %w", err) + } + + login := sdk.Login{ + Identity: user.Credentials.Identity, + Secret: user.Credentials.Secret, + } + token, err := s.CreateToken(login) + if err != nil { + return "", fmt.Errorf("unable to login user: %w", err) + } + + dname := fmt.Sprintf("%s%s", conf.Prefix, namesgenerator.Generate()) + domain := sdk.Domain{ + Name: dname, + Alias: strings.ToLower(dname), + Permission: "admin", + } + + domain, err = s.CreateDomain(domain, token.AccessToken) + if err != nil { + return "", fmt.Errorf("unable to create domain: %w", err) + } + + login = sdk.Login{ + Identity: user.Credentials.Identity, + Secret: user.Credentials.Secret, + DomainID: domain.ID, + } + token, err = s.CreateToken(login) + if err != nil { + return "", fmt.Errorf("unable to login user: %w", err) + } + + return token.AccessToken, nil +} + +func createUsers(s sdk.SDK, conf Config, token string) ([]sdk.User, error) { + var err error + users := []sdk.User{} + + for i := uint64(0); i < conf.Num; i++ { + user := sdk.User{ + Name: fmt.Sprintf("%s%s", conf.Prefix, namesgenerator.Generate()), + Credentials: sdk.Credentials{ + Identity: fmt.Sprintf("%s%s@email.com", conf.Prefix, namesgenerator.Generate()), + Secret: defPass, + }, + Status: sdk.EnabledStatus, + } + + user, err = s.CreateUser(user, token) + if err != nil { + return []sdk.User{}, fmt.Errorf("failed to create the users: %w", err) + } + users = append(users, user) + } + + return users, nil +} + +func createGroups(s sdk.SDK, conf Config, token string) ([]sdk.Group, error) { + var err error + groups := []sdk.Group{} + + for i := uint64(0); i < conf.Num; i++ { + group := sdk.Group{ + Name: fmt.Sprintf("%s%s", conf.Prefix, namesgenerator.Generate()), + Status: sdk.EnabledStatus, + } + + group, err = s.CreateGroup(group, token) + if err != nil { + return []sdk.Group{}, fmt.Errorf("failed to create the group: %w", err) + } + groups = append(groups, group) + } + + return groups, nil +} + +func createThingsInBatch(s sdk.SDK, conf Config, token string, num uint64) ([]sdk.Thing, error) { + var err error + things := make([]sdk.Thing, num) + + for i := uint64(0); i < num; i++ { + things[i] = sdk.Thing{ + Name: fmt.Sprintf("%s%s", conf.Prefix, namesgenerator.Generate()), + } + } + + things, err = s.CreateThings(things, token) + if err != nil { + return []sdk.Thing{}, fmt.Errorf("failed to create the things: %w", err) + } + + return things, nil +} + +func createThings(s sdk.SDK, conf Config, token string) ([]sdk.Thing, error) { + things := []sdk.Thing{} + + if conf.Num > batchSize { + batches := int(conf.Num) / batchSize + for i := 0; i < batches; i++ { + ths, err := createThingsInBatch(s, conf, token, batchSize) + if err != nil { + return []sdk.Thing{}, fmt.Errorf("Failed to create the things: %w", err) + } + things = append(things, ths...) + } + ths, err := createThingsInBatch(s, conf, token, conf.Num%uint64(batchSize)) + if err != nil { + return []sdk.Thing{}, fmt.Errorf("Failed to create the things: %w", err) + } + things = append(things, ths...) + } else { + ths, err := createThingsInBatch(s, conf, token, conf.Num) + if err != nil { + return []sdk.Thing{}, fmt.Errorf("Failed to create the things: %w", err) + } + things = append(things, ths...) + } + + return things, nil +} + +func createChannelsInBatch(s sdk.SDK, conf Config, token string, num uint64) ([]sdk.Channel, error) { + var err error + channels := make([]sdk.Channel, num) + + for i := uint64(0); i < num; i++ { + channels[i] = sdk.Channel{ + Name: fmt.Sprintf("%s%s", conf.Prefix, namesgenerator.Generate()), + } + channels[i], err = s.CreateChannel(channels[i], token) + if err != nil { + return []sdk.Channel{}, fmt.Errorf("failed to create the channels: %w", err) + } + } + + return channels, nil +} + +func createChannels(s sdk.SDK, conf Config, token string) ([]sdk.Channel, error) { + channels := []sdk.Channel{} + + if conf.Num > batchSize { + batches := int(conf.Num) / batchSize + for i := 0; i < batches; i++ { + chs, err := createChannelsInBatch(s, conf, token, batchSize) + if err != nil { + return []sdk.Channel{}, fmt.Errorf("Failed to create the channels: %w", err) + } + channels = append(channels, chs...) + } + chs, err := createChannelsInBatch(s, conf, token, conf.Num%uint64(batchSize)) + if err != nil { + return []sdk.Channel{}, fmt.Errorf("Failed to create the channels: %w", err) + } + channels = append(channels, chs...) + } else { + chs, err := createChannelsInBatch(s, conf, token, conf.Num) + if err != nil { + return []sdk.Channel{}, fmt.Errorf("Failed to create the channels: %w", err) + } + channels = append(channels, chs...) + } + + return channels, nil +} + +func read(s sdk.SDK, conf Config, token string, users []sdk.User, groups []sdk.Group, things []sdk.Thing, channels []sdk.Channel) error { + for _, user := range users { + if _, err := s.User(user.ID, token); err != nil { + return fmt.Errorf("failed to get user %w", err) + } + } + up, err := s.Users(sdk.PageMetadata{}, token) + if err != nil { + return fmt.Errorf("failed to get users %w", err) + } + if up.Total < conf.Num { + return fmt.Errorf("returned users %d less than created users %d", up.Total, conf.Num) + } + for _, group := range groups { + if _, err := s.Group(group.ID, token); err != nil { + return fmt.Errorf("failed to get group %w", err) + } + } + gp, err := s.Groups(sdk.PageMetadata{}, token) + if err != nil { + return fmt.Errorf("failed to get groups %w", err) + } + if gp.Total < conf.Num { + return fmt.Errorf("returned groups %d less than created groups %d", gp.Total, conf.Num) + } + for _, thing := range things { + if _, err := s.Thing(thing.ID, token); err != nil { + return fmt.Errorf("failed to get thing %w", err) + } + } + tp, err := s.Things(sdk.PageMetadata{}, token) + if err != nil { + return fmt.Errorf("failed to get things %w", err) + } + if tp.Total < conf.Num { + return fmt.Errorf("returned things %d less than created things %d", tp.Total, conf.Num) + } + for _, channel := range channels { + if _, err := s.Channel(channel.ID, token); err != nil { + return fmt.Errorf("failed to get channel %w", err) + } + } + cp, err := s.Channels(sdk.PageMetadata{}, token) + if err != nil { + return fmt.Errorf("failed to get channels %w", err) + } + if cp.Total < conf.Num { + return fmt.Errorf("returned channels %d less than created channels %d", cp.Total, conf.Num) + } + + return nil +} + +func update(s sdk.SDK, token string, users []sdk.User, groups []sdk.Group, things []sdk.Thing, channels []sdk.Channel) error { + for _, user := range users { + user.Name = namesgenerator.Generate() + user.Metadata = sdk.Metadata{"Update": namesgenerator.Generate()} + rUser, err := s.UpdateUser(user, token) + if err != nil { + return fmt.Errorf("failed to update user %w", err) + } + if rUser.Name != user.Name { + return fmt.Errorf("failed to update user name before %s after %s", user.Name, rUser.Name) + } + if rUser.Metadata["Update"] != user.Metadata["Update"] { + return fmt.Errorf("failed to update user metadata before %s after %s", user.Metadata["Update"], rUser.Metadata["Update"]) + } + user = rUser + user.Credentials.Identity = namesgenerator.Generate() + rUser, err = s.UpdateUserIdentity(user, token) + if err != nil { + return fmt.Errorf("failed to update user identity %w", err) + } + if rUser.Credentials.Identity != user.Credentials.Identity { + return fmt.Errorf("failed to update user identity before %s after %s", user.Credentials.Identity, rUser.Credentials.Identity) + } + user = rUser + user.Tags = []string{namesgenerator.Generate()} + rUser, err = s.UpdateUserTags(user, token) + if err != nil { + return fmt.Errorf("failed to update user tags %w", err) + } + if rUser.Tags[0] != user.Tags[0] { + return fmt.Errorf("failed to update user tags before %s after %s", user.Tags[0], rUser.Tags[0]) + } + user = rUser + rUser, err = s.DisableUser(user.ID, token) + if err != nil { + return fmt.Errorf("failed to disable user %w", err) + } + if rUser.Status != sdk.DisabledStatus { + return fmt.Errorf("failed to disable user before %s after %s", user.Status, rUser.Status) + } + user = rUser + rUser, err = s.EnableUser(user.ID, token) + if err != nil { + return fmt.Errorf("failed to enable user %w", err) + } + if rUser.Status != sdk.EnabledStatus { + return fmt.Errorf("failed to enable user before %s after %s", user.Status, rUser.Status) + } + } + for _, group := range groups { + group.Name = namesgenerator.Generate() + group.Metadata = sdk.Metadata{"Update": namesgenerator.Generate()} + rGroup, err := s.UpdateGroup(group, token) + if err != nil { + return fmt.Errorf("failed to update group %w", err) + } + if rGroup.Name != group.Name { + return fmt.Errorf("failed to update group name before %s after %s", group.Name, rGroup.Name) + } + if rGroup.Metadata["Update"] != group.Metadata["Update"] { + return fmt.Errorf("failed to update group metadata before %s after %s", group.Metadata["Update"], rGroup.Metadata["Update"]) + } + group = rGroup + rGroup, err = s.DisableGroup(group.ID, token) + if err != nil { + return fmt.Errorf("failed to disable group %w", err) + } + if rGroup.Status != sdk.DisabledStatus { + return fmt.Errorf("failed to disable group before %s after %s", group.Status, rGroup.Status) + } + group = rGroup + rGroup, err = s.EnableGroup(group.ID, token) + if err != nil { + return fmt.Errorf("failed to enable group %w", err) + } + if rGroup.Status != sdk.EnabledStatus { + return fmt.Errorf("failed to enable group before %s after %s", group.Status, rGroup.Status) + } + } + for _, thing := range things { + thing.Name = namesgenerator.Generate() + thing.Metadata = sdk.Metadata{"Update": namesgenerator.Generate()} + rThing, err := s.UpdateThing(thing, token) + if err != nil { + return fmt.Errorf("failed to update thing %w", err) + } + if rThing.Name != thing.Name { + return fmt.Errorf("failed to update thing name before %s after %s", thing.Name, rThing.Name) + } + if rThing.Metadata["Update"] != thing.Metadata["Update"] { + return fmt.Errorf("failed to update thing metadata before %s after %s", thing.Metadata["Update"], rThing.Metadata["Update"]) + } + thing = rThing + rThing, err = s.UpdateThingSecret(thing.ID, thing.Credentials.Secret, token) + if err != nil { + return fmt.Errorf("failed to update thing secret %w", err) + } + thing = rThing + thing.Tags = []string{namesgenerator.Generate()} + rThing, err = s.UpdateThingTags(thing, token) + if err != nil { + return fmt.Errorf("failed to update thing tags %w", err) + } + if rThing.Tags[0] != thing.Tags[0] { + return fmt.Errorf("failed to update thing tags before %s after %s", thing.Tags[0], rThing.Tags[0]) + } + thing = rThing + rThing, err = s.DisableThing(thing.ID, token) + if err != nil { + return fmt.Errorf("failed to disable thing %w", err) + } + if rThing.Status != sdk.DisabledStatus { + return fmt.Errorf("failed to disable thing before %s after %s", thing.Status, rThing.Status) + } + thing = rThing + rThing, err = s.EnableThing(thing.ID, token) + if err != nil { + return fmt.Errorf("failed to enable thing %w", err) + } + if rThing.Status != sdk.EnabledStatus { + return fmt.Errorf("failed to enable thing before %s after %s", thing.Status, rThing.Status) + } + } + for _, channel := range channels { + channel.Name = namesgenerator.Generate() + channel.Metadata = sdk.Metadata{"Update": namesgenerator.Generate()} + rChannel, err := s.UpdateChannel(channel, token) + if err != nil { + return fmt.Errorf("failed to update channel %w", err) + } + if rChannel.Name != channel.Name { + return fmt.Errorf("failed to update channel name before %s after %s", channel.Name, rChannel.Name) + } + if rChannel.Metadata["Update"] != channel.Metadata["Update"] { + return fmt.Errorf("failed to update channel metadata before %s after %s", channel.Metadata["Update"], rChannel.Metadata["Update"]) + } + channel = rChannel + rChannel, err = s.DisableChannel(channel.ID, token) + if err != nil { + return fmt.Errorf("failed to disable channel %w", err) + } + if rChannel.Status != sdk.DisabledStatus { + return fmt.Errorf("failed to disable channel before %s after %s", channel.Status, rChannel.Status) + } + channel = rChannel + rChannel, err = s.EnableChannel(channel.ID, token) + if err != nil { + return fmt.Errorf("failed to enable channel %w", err) + } + if rChannel.Status != sdk.EnabledStatus { + return fmt.Errorf("failed to enable channel before %s after %s", channel.Status, rChannel.Status) + } + } + + return nil +} + +func messaging(s sdk.SDK, conf Config, token string, things []sdk.Thing, channels []sdk.Channel) error { + for _, thing := range things { + for _, channel := range channels { + conn := sdk.Connection{ + ThingID: thing.ID, + ChannelID: channel.ID, + } + if err := s.Connect(conn, token); err != nil { + return fmt.Errorf("failed to connect thing %s to channel %s", thing.ID, channel.ID) + } + } + } + + g := new(errgroup.Group) + + bt := time.Now().Unix() + for i := uint64(0); i < conf.NumOfMsg; i++ { + for _, thing := range things { + for _, channel := range channels { + func(num int64, thing sdk.Thing, channel sdk.Channel) { + g.Go(func() error { + msg := fmt.Sprintf(msgFormat, num+1, rand.Int()) + return sendHTTPMessage(s, msg, thing, channel.ID) + }) + g.Go(func() error { + msg := fmt.Sprintf(msgFormat, num+2, rand.Int()) + return sendCoAPMessage(msg, thing, channel.ID) + }) + g.Go(func() error { + msg := fmt.Sprintf(msgFormat, num+3, rand.Int()) + return sendMQTTMessage(msg, thing, channel.ID) + }) + g.Go(func() error { + msg := fmt.Sprintf(msgFormat, num+4, rand.Int()) + return sendWSMessage(conf, msg, thing, channel.ID) + }) + }(bt, thing, channel) + bt += numAdapters + } + } + } + + return g.Wait() +} + +func sendHTTPMessage(s sdk.SDK, msg string, thing sdk.Thing, chanID string) error { + if err := s.SendMessage(chanID, msg, thing.Credentials.Secret); err != nil { + return fmt.Errorf("HTTP failed to send message from thing %s to channel %s: %w", thing.ID, chanID, err) + } + + return nil +} + +func sendCoAPMessage(msg string, thing sdk.Thing, chanID string) error { + cmd := exec.Command("coap-cli", "post", fmt.Sprintf("channels/%s/messages", chanID), "--auth", thing.Credentials.Secret, "-d", msg) + if _, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("CoAP failed to send message from thing %s to channel %s: %w", thing.ID, chanID, err) + } + + return nil +} + +func sendMQTTMessage(msg string, thing sdk.Thing, chanID string) error { + cmd := exec.Command("mosquitto_pub", "--id-prefix", "magistrala", "-u", thing.ID, "-P", thing.Credentials.Secret, "-t", fmt.Sprintf("channels/%s/messages", chanID), "-h", "localhost", "-m", msg) + if _, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("MQTT failed to send message from thing %s to channel %s: %w", thing.ID, chanID, err) + } + + return nil +} + +func sendWSMessage(conf Config, msg string, thing sdk.Thing, chanID string) error { + socketURL := fmt.Sprintf("ws://%s:%s/channels/%s/messages", conf.Host, defWSPort, chanID) + header := http.Header{"Authorization": []string{thing.Credentials.Secret}} + conn, _, err := websocket.DefaultDialer.Dial(socketURL, header) + if err != nil { + return fmt.Errorf("unable to connect to websocket: %w", err) + } + defer conn.Close() + if err := conn.WriteMessage(websocket.TextMessage, []byte(msg)); err != nil { + return fmt.Errorf("WS failed to send message from thing %s to channel %s: %w", thing.ID, chanID, err) + } + + return nil +} + +// getIDS returns a list of IDs of the given objects. +func getIDS(objects interface{}) string { + v := reflect.ValueOf(objects) + if v.Kind() != reflect.Slice { + panic("objects argument must be a slice") + } + ids := make([]string, v.Len()) + for i := 0; i < v.Len(); i++ { + id := v.Index(i).FieldByName("ID").String() + ids[i] = id + } + idList := strings.Join(ids, "\n") + + return idList +} diff --git a/tools/mqtt-bench/Makefile b/tools/mqtt-bench/Makefile new file mode 100644 index 0000000..f2b3bed --- /dev/null +++ b/tools/mqtt-bench/Makefile @@ -0,0 +1,15 @@ +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +PROGRAM = mqtt-bench +SOURCES = $(wildcard *.go) cmd/main.go + +all: $(PROGRAM) + +.PHONY: all clean + +$(PROGRAM): $(SOURCES) + go build -ldflags "-s -w" -o $@ cmd/main.go + +clean: + rm -rf $(PROGRAM) diff --git a/tools/mqtt-bench/README.md b/tools/mqtt-bench/README.md new file mode 100644 index 0000000..f94eb4d --- /dev/null +++ b/tools/mqtt-bench/README.md @@ -0,0 +1,109 @@ +# MQTT Benchmarking Tool + +A simple MQTT benchmarking tool for Magistrala platform. + +It connects Magistrala things as subscribers over a number of channels and +uses other Magistrala things to publish messages and create MQTT load. + +Magistrala things used must be pre-provisioned first, and Magistrala `provision` tool can be used for this purpose. + +## Installation + +``` +cd tools/mqtt-bench +make +``` + +## Usage + +The tool supports multiple concurrent clients, publishers and subscribers configurable message size, etc: + +``` +./mqtt-bench --help +Tool for extensive load and benchmarking of MQTT brokers used within Magistrala platform. +Complete documentation is available at https://docs.magistrala.abstractmachines.fr + +Usage: + mqtt-bench [flags] + +Flags: + -b, --broker string address for mqtt broker, for secure use tcps and 8883 (default "tcp://localhost:1883") + --ca string CA file (default "ca.crt") + -c, --config string config file for mqtt-bench (default "config.toml") + -n, --count int Number of messages sent per publisher (default 100) + -f, --format string Output format: text|json (default "text") + -h, --help help for mqtt-bench + -m, --magistrala string config file for Magistrala connections (default "connections.toml") + --mtls Use mtls for connection + -p, --pubs int Number of publishers (default 10) + -q, --qos int QoS for published messages, values 0 1 2 + --quiet Supress messages + -r, --retain Retain mqtt messages + -z, --size int Size of message payload bytes (default 100) + -t, --skipTLSVer Skip tls verification + -t, --timeout Timeout mqtt messages (default 10000) +``` + +Two output formats supported: human-readable plain text and JSON. + +Before use you need a `mgconn.toml` - a TOML file that describes Magistrala connection data (channels, thingIDs, thingKeys, certs). +You can use `provision` tool (in tools/provision) to create this TOML config file. + +```bash +go run tools/mqtt-bench/cmd/main.go -u test@magistrala.com -p test1234 --host http://127.0.0.1 --num 100 > tools/mqtt-bench/mgconn.toml +``` + +Example use and output + +Without mtls: + +``` +go run tools/mqtt-bench/cmd/main.go --broker tcp://localhost:1883 --count 100 --size 100 --qos 0 --format text --pubs 10 --magistrala tools/mqtt-bench/mgconn.toml +``` + +With mtls +go run tools/mqtt-bench/cmd/main.go --broker tcps://localhost:8883 --count 100 --size 100 --qos 0 --format text --pubs 10 --magistrala tools/mqtt-bench/mgconn.toml --mtls -ca docker/ssl/certs/ca.crt + +``` + +You can use `config.toml` to create tests with this tool: + +``` + +go run tools/mqtt-bench/cmd/main.go --config tools/mqtt-bench/config.toml + +``` + +Example of `config.toml`: + +``` + +[mqtt] +[mqtt.broker] +url = "tcp://localhost:1883" + +[mqtt.message] +size = 100 +format = "text" +qos = 2 +retain = true + +[mqtt.tls] +mtls = false +skiptlsver = true +ca = "ca.crt" + +[test] +pubs = 3 +count = 100 + +[log] +quiet = false + +[magistrala] +connections_file = "mgconn.toml" + +``` + +Based on this, a test scenario is provided in `templates/reference.toml` file. +``` diff --git a/tools/mqtt-bench/bench.go b/tools/mqtt-bench/bench.go new file mode 100644 index 0000000..2c46406 --- /dev/null +++ b/tools/mqtt-bench/bench.go @@ -0,0 +1,194 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package bench + +import ( + "crypto/rand" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "os" + "strconv" + "time" + + mglog "github.com/absmach/magistrala/logger" + "github.com/pelletier/go-toml" +) + +// Benchmark - main benchmarking function. +func Benchmark(cfg Config) error { + if err := checkConnection(cfg.MQTT.Broker.URL, 1); err != nil { + return err + } + logger, err := mglog.New(os.Stdout, "debug") + if err != nil { + return err + } + + subsResults := map[string](*[]float64){} + var caByte []byte + if cfg.MQTT.TLS.MTLS { + caFile, err := os.Open(cfg.MQTT.TLS.CA) + + defer func() { + if err = caFile.Close(); err != nil { + logger.Warn(fmt.Sprintf("Could not close file: %s", err)) + } + }() + if err != nil { + logger.Warn(err.Error()) + } + caByte, _ = io.ReadAll(caFile) + } + + data, err := os.ReadFile(cfg.Mg.ConnFile) + if err != nil { + return fmt.Errorf("error loading connections file: %s", err) + } + + mg := magistrala{} + if err := toml.Unmarshal(data, &mg); err != nil { + return fmt.Errorf("cannot load Magistrala connections config %s \nUse tools/provision to create file", cfg.Mg.ConnFile) + } + + resCh := make(chan *runResults) + finishedPub := make(chan bool) + + startStamp := time.Now() + + n := len(mg.Channels) + var cert tls.Certificate + + start := time.Now() + + // Publishers + for i := 0; i < cfg.Test.Pubs; i++ { + mgChan := mg.Channels[i%n] + mgThing := mg.Things[i%n] + + if cfg.MQTT.TLS.MTLS { + cert, err = tls.X509KeyPair([]byte(mgThing.MTLSCert), []byte(mgThing.MTLSKey)) + if err != nil { + return err + } + } + c, err := makeClient(i, cfg, mgChan, mgThing, startStamp, caByte, cert) + if err != nil { + return fmt.Errorf("unable to create message payload %s", err.Error()) + } + + errorChan := make(chan error) + go c.publish(resCh, errorChan) + + for { + err := <-errorChan + if err != nil { + return err + } + } + } + + // Collect the results + var results []*runResults + if cfg.Test.Pubs > 0 { + results = make([]*runResults, cfg.Test.Pubs) + } + + // Wait for publishers to finish + go func() { + for i := 0; i < cfg.Test.Pubs; i++ { + results[i] = <-resCh + } + finishedPub <- true + }() + + <-finishedPub + + totalTime := time.Since(start) + totals := calculateTotalResults(results, totalTime, subsResults) + if totals == nil { + return fmt.Errorf("totals not assigned") + } + + // Print sats + printResults(results, totals, cfg.MQTT.Message.Format, cfg.Log.Quiet) + return nil +} + +func getBytePayload(size int, m message) (handler, error) { + // Calculate payload size. + var b []byte + s, err := json.Marshal(&m) + if err != nil { + return nil, err + } + n := len(s) + if n < size { + sz := size - n + for { + b = make([]byte, sz) + if _, err = rand.Read(b); err != nil { + return nil, err + } + m.Payload = b + content, err := json.Marshal(&m) + if err != nil { + return nil, err + } + l := len(content) + // Use range because the size of generated JSON + // depends on current time and random byte array. + if l <= size+5 && l >= size-5 { + break + } + if l > size { + sz-- + } + if l < size { + sz++ + } + } + } + + ret := func(m *message) ([]byte, error) { + m.Payload = b + m.Sent = time.Now() + return json.Marshal(m) + } + return ret, nil +} + +func makeClient(i int, cfg Config, mgChan mgChannel, mgThing mgThing, start time.Time, caCert []byte, clientCert tls.Certificate) (*Client, error) { + c := &Client{ + ID: strconv.Itoa(i), + BrokerURL: cfg.MQTT.Broker.URL, + BrokerUser: mgThing.ThingID, + BrokerPass: mgThing.ThingKey, + MsgTopic: fmt.Sprintf("channels/%s/messages/%d/test", mgChan.ChannelID, start.UnixNano()), + MsgSize: cfg.MQTT.Message.Size, + MsgCount: cfg.Test.Count, + MsgQoS: byte(cfg.MQTT.Message.QoS), + Quiet: cfg.Log.Quiet, + MTLS: cfg.MQTT.TLS.MTLS, + SkipTLSVer: cfg.MQTT.TLS.SkipTLSVer, + CA: caCert, + timeout: cfg.MQTT.Timeout, + ClientCert: clientCert, + Retain: cfg.MQTT.Message.Retain, + } + msg := message{ + Topic: c.MsgTopic, + QoS: c.MsgQoS, + ID: c.ID, + Sent: time.Now(), + } + h, err := getBytePayload(cfg.MQTT.Message.Size, msg) + if err != nil { + return nil, err + } + + c.SendMsg = h + return c, nil +} diff --git a/tools/mqtt-bench/client.go b/tools/mqtt-bench/client.go new file mode 100644 index 0000000..1372990 --- /dev/null +++ b/tools/mqtt-bench/client.go @@ -0,0 +1,221 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package bench + +import ( + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "log" + "net" + "strings" + "sync" + "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" +) + +// Set default ping timeout to large value, so that ping +// won't fail in the case of broker pingresp delay. +const pingTimeout = 10000 + +// Client - represents mqtt client. +type Client struct { + ID string + BrokerURL string + BrokerUser string + BrokerPass string + MsgTopic string + MsgSize int + MsgCount int + MsgQoS byte + Quiet bool + timeout int + mqttClient *mqtt.Client + MTLS bool + SkipTLSVer bool + Retain bool + CA []byte + ClientCert tls.Certificate + ClientKey *rsa.PrivateKey + SendMsg handler +} + +type message struct { + ID string `json:"id"` + Topic string `json:"topic"` + QoS byte `json:"qos"` + Payload []byte `json:"payload"` + Sent time.Time `json:"sent"` + Delivered time.Time `json:"delivered"` + Error bool `json:"error"` +} + +type handler func(*message) ([]byte, error) + +func (c *Client) publish(r chan *runResults, errChan chan<- error) { + res := &runResults{} + times := make([]*float64, c.MsgCount) + + start := time.Now() + if c.connect() != nil { + flushMessages := make([]message, c.MsgCount) + for i, m := range flushMessages { + m.Error = true + times[i] = calcMsgRes(&m, res) + } + r <- calcRes(res, start, arr(times)) + } + if !c.Quiet { + log.Printf("Client %v is connected to the broker %v\n", c.ID, c.BrokerURL) + } + wg := sync.WaitGroup{} + mu := sync.Mutex{} + // Use a single message. + m := message{ + Topic: c.MsgTopic, + QoS: c.MsgQoS, + ID: c.ID, + Sent: time.Now(), + } + payload, err := c.SendMsg(&m) + if err != nil { + errChan <- fmt.Errorf("failed to marshal payload - %s", err.Error()) + } + + for i := 0; i < c.MsgCount; i++ { + wg.Add(1) + go func(mut *sync.Mutex, wg *sync.WaitGroup, i int, m message) { + defer wg.Done() + m.Sent = time.Now() + + token := (*c.mqttClient).Publish(m.Topic, m.QoS, c.Retain, payload) + if !token.WaitTimeout(time.Second*time.Duration(c.timeout)) || token.Error() != nil || !(*c.mqttClient).IsConnectionOpen() { + m.Error = true + mu.Lock() + times[i] = calcMsgRes(&m, res) + mu.Unlock() + return + } + + m.Delivered = time.Now() + m.Error = false + mu.Lock() + times[i] = calcMsgRes(&m, res) + mu.Unlock() + + if !c.Quiet && i > 0 && i%100 == 0 { + log.Printf("Client %v published %v messages and keeps publishing...\n", c.ID, i) + } + }(&mu, &wg, i, m) + } + wg.Wait() + + r <- calcRes(res, start, arr(times)) +} + +func (c *Client) connect() error { + opts := mqtt.NewClientOptions(). + AddBroker(c.BrokerURL). + SetClientID(c.ID). + SetCleanSession(false). + SetAutoReconnect(false). + SetOnConnectHandler(c.connected). + SetConnectionLostHandler(c.connLost). + SetPingTimeout(time.Second * pingTimeout). + SetAutoReconnect(true). + SetCleanSession(false) + + if c.BrokerUser != "" && c.BrokerPass != "" { + opts.SetUsername(c.BrokerUser) + opts.SetPassword(c.BrokerPass) + } + + if c.MTLS { + cfg := &tls.Config{ + InsecureSkipVerify: c.SkipTLSVer, + } + + if c.CA != nil { + cfg.RootCAs = x509.NewCertPool() + cfg.RootCAs.AppendCertsFromPEM(c.CA) + } + if c.ClientCert.Certificate != nil { + cfg.Certificates = []tls.Certificate{c.ClientCert} + } + + opts.SetTLSConfig(cfg) + opts.SetProtocolVersion(4) + } + + client := mqtt.NewClient(opts) + token := client.Connect() + token.Wait() + + c.mqttClient = &client + + if token.Error() != nil { + log.Printf("Client %v had error connecting to the broker: %s\n", c.ID, token.Error().Error()) + return token.Error() + } + + return nil +} + +func checkConnection(broker string, timeoutSecs int) error { + s := strings.Split(broker, ":") + if len(s) != 3 { + return errors.New("wrong host address format") + } + + network := s[0] + host := strings.Trim(s[1], "/") + port := s[2] + + log.Println("Testing connection...") + conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%s", host, port), time.Duration(timeoutSecs)*time.Second) + conClose := func() { + if conn != nil { + log.Println("Closing testing connection...") + conn.Close() + } + } + + defer conClose() + if err, ok := err.(*net.OpError); ok && err.Timeout() { + return fmt.Errorf("timeout error: %s", err.Error()) + } + + if err != nil { + return fmt.Errorf("error: %s", err.Error()) + } + + log.Printf("Connection to %s://%s:%s looks OK\n", network, host, port) + return nil +} + +func arr(a []*float64) []float64 { + ret := []float64{} + for _, v := range a { + if v != nil { + ret = append(ret, *v) + } + } + if len(ret) == 0 { + ret = append(ret, 0) + } + return ret +} + +func (c *Client) connected(client mqtt.Client) { + if !c.Quiet { + log.Printf("Client %v is connected to the broker %v\n", c.ID, c.BrokerURL) + } +} + +func (c *Client) connLost(client mqtt.Client, reason error) { + log.Printf("Client %v had lost connection to the broker: %s\n", c.ID, reason.Error()) +} diff --git a/tools/mqtt-bench/cmd/main.go b/tools/mqtt-bench/cmd/main.go new file mode 100644 index 0000000..f3edf7d --- /dev/null +++ b/tools/mqtt-bench/cmd/main.go @@ -0,0 +1,77 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package main contains the entry point of the mqtt-bench tool. +package main + +import ( + "log" + + bench "github.com/absmach/magistrala/tools/mqtt-bench" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func main() { + confFile := "" + bconf := bench.Config{} + + // Command + rootCmd := &cobra.Command{ + Use: "mqtt-bench", + Short: "mqtt-bench is MQTT benchmark tool for Magistrala", + Long: `Tool for exctensive load and benchmarking of MQTT brokers used within the Magistrala platform. +Complete documentation is available at https://docs.magistrala.abstractmachines.fr`, + Run: func(cmd *cobra.Command, args []string) { + if confFile != "" { + viper.SetConfigFile(confFile) + + if err := viper.ReadInConfig(); err != nil { + log.Printf("Failed to load config - %s", err) + } + + if err := viper.Unmarshal(&bconf); err != nil { + log.Printf("Unable to decode into struct, %v", err) + } + } + + if err := bench.Benchmark(bconf); err != nil { + log.Fatal(err) + } + }, + } + + // Flags + // MQTT Broker + rootCmd.PersistentFlags().StringVarP(&bconf.MQTT.Broker.URL, "broker", "b", "tcp://localhost:1883", + "address for mqtt broker, for secure use tcps and 8883") + + // MQTT Message + rootCmd.PersistentFlags().IntVarP(&bconf.MQTT.Message.Size, "size", "z", 100, "Size of message payload bytes") + rootCmd.PersistentFlags().StringVarP(&bconf.MQTT.Message.Payload, "payload", "l", "", "Template message") + rootCmd.PersistentFlags().StringVarP(&bconf.MQTT.Message.Format, "format", "f", "text", "Output format: text|json") + rootCmd.PersistentFlags().IntVarP(&bconf.MQTT.Message.QoS, "qos", "q", 0, "QoS for published messages, values 0 1 2") + rootCmd.PersistentFlags().BoolVarP(&bconf.MQTT.Message.Retain, "retain", "r", false, "Retain mqtt messages") + rootCmd.PersistentFlags().IntVarP(&bconf.MQTT.Timeout, "timeout", "o", 10000, "Timeout mqtt messages") + + // MQTT TLS + rootCmd.PersistentFlags().BoolVarP(&bconf.MQTT.TLS.MTLS, "mtls", "", false, "Use mtls for connection") + rootCmd.PersistentFlags().BoolVarP(&bconf.MQTT.TLS.SkipTLSVer, "skipTLSVer", "t", false, "Skip tls verification") + rootCmd.PersistentFlags().StringVarP(&bconf.MQTT.TLS.CA, "ca", "", "ca.crt", "CA file") + + // Test params + rootCmd.PersistentFlags().IntVarP(&bconf.Test.Count, "count", "n", 100, "Number of messages sent per publisher") + rootCmd.PersistentFlags().IntVarP(&bconf.Test.Subs, "subs", "s", 10, "Number of subscribers") + rootCmd.PersistentFlags().IntVarP(&bconf.Test.Pubs, "pubs", "p", 10, "Number of publishers") + + // Log params + rootCmd.PersistentFlags().BoolVarP(&bconf.Log.Quiet, "quiet", "", false, "Suppress messages") + + // Config file + rootCmd.PersistentFlags().StringVarP(&confFile, "config", "c", "config.toml", "config file for mqtt-bench") + rootCmd.PersistentFlags().StringVarP(&bconf.Mg.ConnFile, "magistrala", "m", "connections.toml", "config file for Magistrala connections") + + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) + } +} diff --git a/tools/mqtt-bench/config.go b/tools/mqtt-bench/config.go new file mode 100644 index 0000000..a67a12c --- /dev/null +++ b/tools/mqtt-bench/config.go @@ -0,0 +1,68 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package bench + +// Keep struct names exported, otherwise Viper unmarshalling won't work. +type mqttBrokerConfig struct { + URL string `toml:"url" mapstructure:"url"` +} + +type mqttMessageConfig struct { + Size int `toml:"size" mapstructure:"size"` + Payload string `toml:"payload" mapstructure:"payload"` + Format string `toml:"format" mapstructure:"format"` + QoS int `toml:"qos" mapstructure:"qos"` + Retain bool `toml:"retain" mapstructure:"retain"` +} + +type mqttTLSConfig struct { + MTLS bool `toml:"mtls" mapstructure:"mtls"` + SkipTLSVer bool `toml:"skiptlsver" mapstructure:"skiptlsver"` + CA string `toml:"ca" mapstructure:"ca"` +} + +type mqttConfig struct { + Broker mqttBrokerConfig `toml:"broker" mapstructure:"broker"` + Message mqttMessageConfig `toml:"message" mapstructure:"message"` + Timeout int `toml:"timeout" mapstructure:"timeout"` + TLS mqttTLSConfig `toml:"tls" mapstructure:"tls"` +} + +type testConfig struct { + Count int `toml:"count" mapstructure:"count"` + Pubs int `toml:"pubs" mapstructure:"pubs"` + Subs int `toml:"subs" mapstructure:"subs"` +} + +type logConfig struct { + Quiet bool `toml:"quiet" mapstructure:"quiet"` +} + +type magistralaFile struct { + ConnFile string `toml:"connections_file" mapstructure:"connections_file"` +} + +type mgThing struct { + ThingID string `toml:"thing_id" mapstructure:"thing_id"` + ThingKey string `toml:"thing_key" mapstructure:"thing_key"` + MTLSCert string `toml:"mtls_cert" mapstructure:"mtls_cert"` + MTLSKey string `toml:"mtls_key" mapstructure:"mtls_key"` +} + +type mgChannel struct { + ChannelID string `toml:"channel_id" mapstructure:"channel_id"` +} + +type magistrala struct { + Things []mgThing `toml:"things" mapstructure:"things"` + Channels []mgChannel `toml:"channels" mapstructure:"channels"` +} + +// Config struct holds benchmark configuration. +type Config struct { + MQTT mqttConfig `toml:"mqtt" mapstructure:"mqtt"` + Test testConfig `toml:"test" mapstructure:"test"` + Log logConfig `toml:"log" mapstructure:"log"` + Mg magistralaFile `toml:"magistrala" mapstructure:"magistrala"` +} diff --git a/tools/mqtt-bench/doc.go b/tools/mqtt-bench/doc.go new file mode 100644 index 0000000..6246514 --- /dev/null +++ b/tools/mqtt-bench/doc.go @@ -0,0 +1,5 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package bench contains benchmarking tool for MQTT broker. +package bench diff --git a/tools/mqtt-bench/results.go b/tools/mqtt-bench/results.go new file mode 100644 index 0000000..6d397e0 --- /dev/null +++ b/tools/mqtt-bench/results.go @@ -0,0 +1,194 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package bench + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "time" + + "gonum.org/v1/gonum/mat" + "gonum.org/v1/gonum/stat" +) + +type subsResults map[string](*[]float64) + +type runResults struct { + ID string `json:"id"` + Successes int64 `json:"successes"` + Failures int64 `json:"failures"` + RunTime float64 `json:"run_time"` + MsgTimeMin float64 `json:"msg_time_min"` + MsgTimeMax float64 `json:"msg_time_max"` + MsgTimeMean float64 `json:"msg_time_mean"` + MsgTimeStd float64 `json:"msg_time_std"` + MsgDelTimeMin float64 `json:"msg_del_time_min"` + MsgDelTimeMax float64 `json:"msg_del_time_max"` + MsgDelTimeMean float64 `json:"msg_del_time_mean"` + MsgDelTimeStd float64 `json:"msg_del_time_std"` + MsgsPerSec float64 `json:"msgs_per_sec"` +} + +type totalResults struct { + Ratio float64 `json:"ratio"` + Successes int64 `json:"successes"` + Failures int64 `json:"failures"` + TotalRunTime float64 `json:"total_run_time"` + AvgRunTime float64 `json:"avg_run_time"` + MsgTimeMin float64 `json:"msg_time_min"` + MsgTimeMax float64 `json:"msg_time_max"` + MsgDelTimeMin float64 `json:"msg_del_time_min"` + MsgDelTimeMax float64 `json:"msg_del_time_max"` + MsgTimeMeanAvg float64 `json:"msg_time_mean_avg"` + MsgTimeMeanStd float64 `json:"msg_time_mean_std"` + MsgDelTimeMeanAvg float64 `json:"msg_del_time_mean_avg"` + MsgDelTimeMeanStd float64 `json:"msg_del_time_mean_std"` + TotalMsgsPerSec float64 `json:"total_msgs_per_sec"` + AvgMsgsPerSec float64 `json:"avg_msgs_per_sec"` +} + +// JSONResults are used to export results as a JSON document. +type JSONResults struct { + Runs []*runResults `json:"runs"` + Totals *totalResults `json:"totals"` +} + +func calcMsgRes(m *message, res *runResults) *float64 { + if m.Error { + res.Failures++ + return nil + } + res.Successes++ + diff := float64(m.Delivered.Sub(m.Sent).Nanoseconds() / 1000) // in microseconds + return &diff +} + +func calcRes(r *runResults, start time.Time, times []float64) *runResults { + duration := time.Since(start) + timeMatrix := mat.NewDense(1, len(times), times) + r.MsgTimeMin = mat.Min(timeMatrix) + r.MsgTimeMax = mat.Max(timeMatrix) + r.MsgTimeMean = stat.Mean(times, nil) + r.MsgTimeStd = stat.StdDev(times, nil) + r.RunTime = duration.Seconds() + r.MsgsPerSec = float64(r.Successes) / duration.Seconds() + return r +} + +func calculateTotalResults(results []*runResults, totalTime time.Duration, sr subsResults) *totalResults { + if results == nil || len(results) < 1 { + return nil + } + totals := new(totalResults) + msgTimeMeans := make([]float64, len(results)) + msgTimeMeansDelivered := make([]float64, len(results)) + msgsPerSecs := make([]float64, len(results)) + runTimes := make([]float64, len(results)) + bws := make([]float64, len(results)) + + totals.TotalRunTime = totalTime.Seconds() + + totals.MsgTimeMin = results[0].MsgTimeMin + for i, res := range results { + totals.Successes += res.Successes + totals.Failures += res.Failures + totals.TotalMsgsPerSec += res.MsgsPerSec + + // Don't count those client that sent no messages. + if res.MsgsPerSec == 0 { + continue + } + + if res.MsgTimeMin < totals.MsgTimeMin { + totals.MsgTimeMin = res.MsgTimeMin + } + + if res.MsgTimeMax > totals.MsgTimeMax { + totals.MsgTimeMax = res.MsgTimeMax + } + + if res.MsgDelTimeMin < totals.MsgDelTimeMin { + totals.MsgDelTimeMin = res.MsgDelTimeMin + } + + if res.MsgDelTimeMax > totals.MsgDelTimeMax { + totals.MsgDelTimeMax = res.MsgDelTimeMax + } + + msgTimeMeansDelivered[i] = res.MsgDelTimeMean + msgTimeMeans[i] = res.MsgTimeMean + msgsPerSecs[i] = res.MsgsPerSec + runTimes[i] = res.RunTime + bws[i] = res.MsgsPerSec + } + + for _, v := range sr { + times := mat.NewDense(1, len(*v), *v) + totals.MsgDelTimeMin = mat.Min(times) / 1000 + totals.MsgDelTimeMax = mat.Max(times) / 1000 + totals.MsgDelTimeMeanAvg = stat.Mean(*v, nil) / 1000 + totals.MsgDelTimeMeanStd = stat.StdDev(*v, nil) / 1000 + } + + totals.Ratio = float64(totals.Successes) / float64(totals.Successes+totals.Failures) + totals.AvgMsgsPerSec = stat.Mean(msgsPerSecs, nil) + totals.AvgRunTime = stat.Mean(runTimes, nil) + totals.MsgDelTimeMeanAvg = stat.Mean(msgTimeMeansDelivered, nil) + totals.MsgDelTimeMeanStd = stat.StdDev(msgTimeMeansDelivered, nil) + totals.MsgTimeMeanAvg = stat.Mean(msgTimeMeans, nil) + totals.MsgTimeMeanStd = stat.StdDev(msgTimeMeans, nil) + + return totals +} + +func printResults(results []*runResults, totals *totalResults, format string, quiet bool) { + switch format { + case "json": + jr := JSONResults{ + Runs: results, + Totals: totals, + } + data, err := json.Marshal(jr) + if err != nil { + log.Printf("Failed to prepare results for printing - %s\n", err.Error()) + } + var out bytes.Buffer + if err = json.Indent(&out, data, "", "\t"); err != nil { + return + } + + fmt.Println(out.String()) + default: + if !quiet { + for _, res := range results { + fmt.Printf("======= CLIENT %s =======\n", res.ID) + fmt.Printf("Ratio: %.6f (%d/%d)\n", float64(res.Successes)/float64(res.Successes+res.Failures), res.Successes, res.Successes+res.Failures) + fmt.Printf("Succeeded: %d\n", res.Successes) + fmt.Printf("Failed: %d\n", res.Failures) + fmt.Printf("Runtime (s): %.3f\n", res.RunTime) + fmt.Printf("Msg time min (µs): %.3f\n", res.MsgTimeMin) + fmt.Printf("Msg time max (µs): %.3f\n", res.MsgTimeMax) + fmt.Printf("Msg time mean (µs): %.3f\n", res.MsgTimeMean) + fmt.Printf("Msg time std (µs): %.3f\n\n", res.MsgTimeStd) + + fmt.Printf("Bandwidth (msg/sec): %.3f\n\n", res.MsgsPerSec) + } + } + fmt.Printf("========= TOTAL (%d) =========\n", len(results)) + fmt.Printf("Total Ratio: %.3f (%d/%d)\n", totals.Ratio, totals.Successes, totals.Successes+totals.Failures) + fmt.Printf("Succeeded: %d\n", totals.Successes) + fmt.Printf("Failed: %d\n", totals.Failures) + fmt.Printf("Total Runtime (sec): %.3f\n", totals.TotalRunTime) + fmt.Printf("Average Runtime (sec): %.3f\n", totals.AvgRunTime) + fmt.Printf("Msg time min (µs): %.3f\n", totals.MsgTimeMin) + fmt.Printf("Msg time max (µs): %.3f\n", totals.MsgTimeMax) + fmt.Printf("Msg time mean (µs): %.3f\n", totals.MsgTimeMeanAvg) + fmt.Printf("Msg time mean std (µs): %.3f\n", totals.MsgTimeMeanStd) + + fmt.Printf("Average Bandwidth (msg/sec): %.3f\n", totals.AvgMsgsPerSec) + fmt.Printf("Total Bandwidth (msg/sec): %.3f\n", totals.TotalMsgsPerSec) + } +} diff --git a/tools/mqtt-bench/scripts/mqtt-bench.sh b/tools/mqtt-bench/scripts/mqtt-bench.sh new file mode 100755 index 0000000..5142b7b --- /dev/null +++ b/tools/mqtt-bench/scripts/mqtt-bench.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +i=0 +echo "BEGIN TEST " > result.$1.out +for mtls in true +do + for ret in false true + do + for qos in 0 1 2 + do + for pub in 1 10 100 + do + for sub in 1 10 + do + for message in 100 1000 + do + if [[ $pub -eq 100 && $message -eq 1000 ]]; + then + continue + fi + + for size in 100 500 + do + let "i += 1" + echo "=================================TEST $i=========================================" >> $1-$i.out + echo "MTLS: $mtls RETAIN: $ret, QOS $qos" >> $1-$i.out + echo "Pub:" $pub ", Sub:" $sub ", MsgSize:" $size ", MsgPerPub:" $message >> $1-$i.out + echo "=================================================================================" >> $1-$i.out + if [ "$mtls" = true ]; + then + echo "| " >> $1-$i.out + echo "| ./mqtt-bench --channels $3 -s $size -n $message --subs $sub --pubs $pub -q $qos --retain=$ret -m=true -b tcps://$2:8883 --quiet=true --ca ../../../docker/ssl/certs/ca.crt -t=true" >> $1-$i.out + echo "| " >> $1-$i.out + ../cmd/mqtt-bench --channels $3 -s $size -n $message --subs $sub --pubs $pub -q $qos --retain=$ret -m=true -b tcps://$2:8883 --quiet=true --ca ../../../docker/ssl/certs/ca.crt -t=true >> $1-$i.out + else + echo "| " >> $1-$i.out + echo "| ./mqtt-bench --channels $3 -s $size -n $message --subs $sub --pubs $pub -q $qos --retain=$ret -b tcp://$2:1883 --quiet=true" >> $1-$i.out + echo "| " >> $1-$i.out + ../cmd/mqtt-bench --channels $3 -s $size -n $message --subs $sub --pubs $pub -q $qos --retain=$ret -b tcp://$2:1883 --quiet=true >> $1-$i.out + fi + sleep 2 + done + done + done + done + done + + done +done +files=`ls test*.out | sort --version-sort ` +for file in $files +do + cat $file >> result.$1.out +done +echo "END TEST " >> result.$1.out diff --git a/tools/mqtt-bench/templates/reference.toml b/tools/mqtt-bench/templates/reference.toml new file mode 100644 index 0000000..5a60e8a --- /dev/null +++ b/tools/mqtt-bench/templates/reference.toml @@ -0,0 +1,29 @@ +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +[mqtt] + timeout = 1000 + [mqtt.broker] + url = "tcp://localhost:1883" + + [mqtt.message] + size = 1000 + format = "text" + qos = 2 + retain = true + payload = "{\"bn\":\"some-base-name\",\"bt\":1.276020076001e+09, \"bu\":\"A\",\"bver\":5, \"n\":\"voltage\",\"u\":\"V\",\"v\":120.1}" + + [mqtt.tls] + mtls = false + skiptlsver = true + ca = "ca.crt" + +[test] +pubs = 2000 +count = 70 + +[log] +quiet = true + +[magistrala] +connections_file = "../provision/mgconn.toml" diff --git a/tools/provision/Makefile b/tools/provision/Makefile new file mode 100644 index 0000000..7b8abc5 --- /dev/null +++ b/tools/provision/Makefile @@ -0,0 +1,15 @@ +# Copyright (c) Abstract Machines +# SPDX-License-Identifier: Apache-2.0 + +PROGRAM = provision +SOURCES = $(wildcard *.go) cmd/main.go + +all: $(PROGRAM) + +.PHONY: all clean + +$(PROGRAM): $(SOURCES) + go build -ldflags "-s -w" -o $@ cmd/main.go + +clean: + rm -rf $(PROGRAM) diff --git a/tools/provision/README.md b/tools/provision/README.md new file mode 100644 index 0000000..77d7068 --- /dev/null +++ b/tools/provision/README.md @@ -0,0 +1,146 @@ +# Magistrala Things and Channels Provisioning Tool + +A simple utility to create a list of channels and things connected to these channels with possibility to create certificates for mTLS use case. + +This tool is useful for testing, and it creates a TOML format output (on stdout, can be redirected into the file as needed) +that can be used by Magistrala MQTT benchmarking tool (`mqtt-bench`). + +## Installation +``` +cd tools/provision +make +``` + +### Usage +``` +./provision --help +Tool for provisioning series of Magistrala channels and things and connecting them together. +Complete documentation is available at https://docs.magistrala.abstractmachines.fr + +Usage: + provision [flags] + +Flags: + --ca string CA for creating and signing things certificate (default "ca.crt") + --cakey string ca.key for creating and signing things certificate (default "ca.key") + -h, --help help for provision + --host string address for magistrala instance (default "https://localhost") + --num int number of channels and things to create and connect (default 10) + -p, --password string magistrala users password + --ssl create certificates for mTLS access + -u, --username string magistrala user + --prefix string name prefix for things and channels +``` + +Example: +``` +go run tools/provision/cmd/main.go -u test@magistrala.com -p test1234 --host https://142.93.118.47 +``` + +If you want to create a list of channels with certificates: + +``` +go run tools/provision/cmd/main.go --host http://localhost --num 10 -u test@magistrala.com -p test1234 --ssl true --ca docker/ssl/certs/ca.crt --cakey docker/ssl/certs/ca.key + +``` + +>`ca.crt` and `ca.key` are used for creating things certificate and for HTTPS, +> if you are provisioning on remote server you will have to get these files to your local +> directory so that you can create certificates for things + + +Example of output: + +``` +# List of things that can be connected to MQTT broker +[[things]] +thing_id = "0eac601b-6d54-4767-b8b7-594aaf9990d3" +thing_key = "07713103-513f-43c7-b7fe-500c1af23d7d" +mtls_cert = """-----BEGIN CERTIFICATE----- +MIIEmTCCA4GgAwIBAgIRAO50qOfXsU+cHm/QY2NYu+0wDQYJKoZIhvcNAQELBQAw +VzESMBAGA1UEAwwJbG9jYWxob3N0MREwDwYDVQQKDAhNYWluZmx1eDEMMAoGA1UE +CwwDSW9UMSAwHgYJKoZIhvcNAQkBFhFpbmZvQG1haW5mbHV4LmNvbTAeFw0xOTEx +MTUxNzU2MzhaFw0yMDAyMjMxNzU2MzhaMFUxETAPBgNVBAoTCE1haW5mbHV4MREw +DwYDVQQLEwhtYWluZmx1eDEtMCsGA1UEAxMkMDc3MTMxMDMtNTEzZi00M2M3LWI3 +ZmUtNTAwYzFhZjIzZDdkMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA +zsIYoovZJGJxfu7e4X3P3wnHDi9/wvRMhGW1EZEB5vNvfxvmmt4PhiE1c73mCypT +AUdui0j+hrCx8P90v12LEcJqty3yBnw+ge2/xCLNLKZh2/MjBQ7A7PMQpmOo31LR +hxFSthW41C296iwVYyvRa19y7g5mcUrzWvI2EVZbbGEDym1U/PI4aKhdQ3a7fF6B +GfvXYbGOa4/8VUIj8KHTRg2Z6/iLhxYgUnHd3xMCjihQkwLvB7/avVr9Ih9oLEe+ +h7H9Pl5hMEpHP4BvHokUFhtbzqofuHNBKuEUf5r/cQ1oVAl6F77Fs5vZbQ59bLxw +etclDxW7nvOgIxEIUcJAkdd+nOxhpfbDM8QFsPXGSfb9vWUTaoQDIeWx9pPY5tsY +tbtW2HeKRGHO9jGFSzonY6sbTiaIzQ0F2PNPS1BoBIo2A95YNwt2ScfuRTs5ZK62 +2+RNWbs+pDXJ5ZGcWDfjSxEYXy+jGUyvDExGCtryUu5Ufp7XuZ4O767iDzaj7dFG +rXSXfXrqwm8u2CMwucNzdVqikNG2gDToHDyIjLRd62m2pHk9gXbk3FGI+5x52pBs ++xdRaddMY8+DJ2R88PFoq3kqexxs2HJathCu6RfoP452zH9iU0gvPLR7fXuPoZ6Y +5NqE1CebZ6IiwwivD7kU1LxmhmQUY9DaHdHNYd66bd0CAwEAAaNiMGAwDgYDVR0P +AQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAOBgNVHQ4E +BwQFAQIDBAYwHwYDVR0jBBgwFoAUbOMUfdahIzURpsN/dcUu8ek3PvIwDQYJKoZI +hvcNAQELBQADggEBAI+DdKYKKPVi4CPUbl+R81dq+Otd8L9i/RxM7G89XU0aGkSO +GSJzURKYbmLGgWdVWcdYMUfbpiE8vH1dLuDQdRywpDDjSMx7h0PwpYvk25HHKMSs +OIKpxvI1DyuNcwxrPuH863zw1Mo1hpGGin7yZc8VBf6nbR3RMNbQ2elMH1m7no4v +YM4HrTeR9n1bakIVw9OLnFpB03sT3keBdWsLDbAZ0yZfvxqdn6Hr7NRnab3vyrOz +GrYPJ51B/FGZC9n0ZR+SWzipen15vaG46SvoCv9HfDZ9cbSVR4eyPy/OIx+5CBVY +uGpJ+kN8jH5tuoxrmHZOsPMA+a6CZD2cKTaRu+Y= +-----END CERTIFICATE----- +""" +mtls_key = """-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEAzsIYoovZJGJxfu7e4X3P3wnHDi9/wvRMhGW1EZEB5vNvfxvm +mt4PhiE1c73mCypTAUdui0j+hrCx8P90v12LEcJqty3yBnw+ge2/xCLNLKZh2/Mj +BQ7A7PMQpmOo31LRhxFSthW41C296iwVYyvRa19y7g5mcUrzWvI2EVZbbGEDym1U +/PI4aKhdQ3a7fF6BGfvXYbGOa4/8VUIj8KHTRg2Z6/iLhxYgUnHd3xMCjihQkwLv +B7/avVr9Ih9oLEe+h7H9Pl5hMEpHP4BvHokUFhtbzqofuHNBKuEUf5r/cQ1oVAl6 +F77Fs5vZbQ59bLxwetclDxW7nvOgIxEIUcJAkdd+nOxhpfbDM8QFsPXGSfb9vWUT +aoQDIeWx9pPY5tsYtbtW2HeKRGHO9jGFSzonY6sbTiaIzQ0F2PNPS1BoBIo2A95Y +Nwt2ScfuRTs5ZK622+RNWbs+pDXJ5ZGcWDfjSxEYXy+jGUyvDExGCtryUu5Ufp7X +uZ4O767iDzaj7dFGrXSXfXrqwm8u2CMwucNzdVqikNG2gDToHDyIjLRd62m2pHk9 +gXbk3FGI+5x52pBs+xdRaddMY8+DJ2R88PFoq3kqexxs2HJathCu6RfoP452zH9i +U0gvPLR7fXuPoZ6Y5NqE1CebZ6IiwwivD7kU1LxmhmQUY9DaHdHNYd66bd0CAwEA +AQKCAgAj2sr03TWhtqSh84CZL/0tW3+2eQw53a2rRAv7aN8gktSiAU+jSaD9jKK9 +WJAdHZDZZu7Hnrfs2ZVyCorPaMRmJwXkkEYpU8BvPbCErdhQxuWvg+FtzhosvRYF +FMFDQRRuzNVAGFI+EVSe2Fg5I28kpJ/EoqCnQu0it2Ai74vZJpXGs+EKIGMh2xiZ +S2zF64mN3PuDyIu/IXALxPWAlD+UJWWs4yQnH/Io+fAU8DIAPwOCCv8yo9WmArJl +CXdCPorO81HMUAegnTDv1TDv5aujDcmE9EGd9fa2HeQ1IMbtbvrJn/8ZQQ79z6gL +3nhns+H5m3ekvwsTTIJXsmtz6jDSCek5C78gKJ6fIH/urKkgG0Pcw4HdOtt5PYQS +KnAKN9KuPEqwxJCDpwKcENDxBul9Huc9i4m1J8hq4qtEBk8k1rqfjWAxigBmhdQV +jY0q//ou/VYgD07RIqezCovVZwJDqvEKg2A5e2YmUXIbYmG1BTCN5NIDcnwqO65C +gD4V9vgn2+ek7z8rBr5VHJ/3LNqc+XFzQW+GjzVFLUfzkgipMGt4DVQdseXWKaiz +v6LV7Nn4hPKETZ5pYzNll4SH+PkVG0Pwc9g8yZF0CcvQt/4wry78LdihgXUBtI7G ++5cH/DXOCd1itaauggHQwEm6GF4VR3uPthoU++QvPKqSAvWnQQKCAQEA7n6xDE2J +iWEBCj8gDYcKKgMUlwWmnWc7MprOU2oCR4DXLcDNcmJLKwb2UC1Z4dxQy5pJs6Yk +5f6rOFwQ0sMM36PcmRJcBNeMTsj2ilZ79TbVYl4pgtjZLJl4JptwXFZFeVdTx1Sa +QoZasqlyO44Uw5D3+ztddHpnOVPCLd36xV6R3e1scKuXCrE4Pl/+YmkYG8NrRKoe +vHUhmmtcukxsEPhGJhQqpbMhm75hBFfHJw2gMu1bBGDGYzfX9bBkF1ZRq+7X6/g0 +Zvr5Gh1tZhkHDR9JwRMNbTSQgVvJD0eToBo5kZbWF4+giAhNkV+wGiCMJgdGWJQo +4Cz5rY+Nv2Rz7QKCAQEA3e8SzLm4Gvft9AZUy96kuk5uKckAXW/FnDKfa+zFoT7w +KyEz9yOZRFXoPdrReZLzgk8GDZVbYAyXmONx9Sjq1GmZ/fDkXpUtdr6PmDR19Hea +CVqUfkBYmMTmA0zFpS6rsI+dIwCP2h7slJQ4eUESYVRiXWyOKEhQVGM0t9liUfrr +lfRnVj6q9I3vqCcqgBuODoAS/iFaFpSfh05XSKdl9XW2t/sd33acPqh9zKBczlsR +H6dyrO02znbbOgrBCBbxtFdq4YLuHKsBB2umz/NKfpnoOUHLeTU2VaqyOtDK9BIA +XtCPu6KJNZ86eFAbtHwBpHn7u7iQZtcaWK9LuESDsQKCAQEAiMV/I18UEQTgY8/v +wdI/sfgyRqmm833QJSVCTfPterQYstRu/boBAZvshe58LVr7usewnKYbYwq5hojF +3RieuWJvkBlHTD+Q5124hX0zeV0I4nC9vZw+b6VTklByD4IqNXwvP5D1JlGGkg86 +w4ynu7/XduyEm9fWerneEg/LUIT7gho2pibBaBBaAOtsJ2O9v65CRg6Jseo6ayRG ++U/6aYD4Ob429u/Txk1XtfXg8DSQOqSEHe6h1ySfZPbTb87A56kBiwG8i5JCaQeX +RYX01UGsOl2Cxa3vcUAB/hE+SALCIQwvmzNzDJA2a7hEdbdUqDpjzUiqaGViinZZ +A/nHwQKCAQAkTxLCT7ghIWLaw5Zn7DsDCAXZ7DqVDs5DqbyPSaNjqApe5AW+byKK +HYvrYrtWqoYQUaFp43+ZjTXYG43vUAxrSAObmieimcFgZfjUK/EIV/Dpito0dY6J +H92JuKu1RJduQXCx40ulod2OyVkb7Vt2dPnK0xHG4V3TEI/1bCk7xFN6qwuk/oe1 +jusglZfMcbWiBa4VyZsViqc22chJ6KkzqViFbR4MCzmwvpwmOC42zItWpGyMghqv +WJ6xNkUyb56HpK2ly2ftZMS8VA5sgx8y6zck9vC1GdGT3mNeX/50Q+WvnWuGhSbx +kOVd/a0qsAcMw7A9nApz6Mk0rSk0MnFhAoIBAQCI6dU5c1sTp/LNp+z6yQmcJD3Z +HNYdVhf8pxHpRWZ8r5otFwi1lr5vk15Zh59B5nMLQHP3UWJ7R66HUjXCtFe86ojV +xngL3lXJNtLcCWXQHM/nkWZ1TVCeZ6mS8aJndcy4sY0lPUqRtYaXSV/EyzpQJUmf +xcEeQuOhBZ4s8uSyuLgEPYbeYyi7Vpujm7UpplTN55dIZrQ7tMefRNgHjybFfC8P +QsxPR4lWoFpr9xFvtBORlP+In8LjD3Z2EDm2guIRAWebEJGsY7ftAv7CEFrLOJd5 +uCRt+TFMyEfqilipmNsV7esgbroiyEGXGMI8JdBY9OsnK6ZSlXaMnQ9vq2kK +-----END RSA PRIVATE KEY----- +""" + +# List of channels that things can publish to +# each channel is connected to each thing from things list +# Things connected to channel 1f18afa1-29c4-4634-99d1-68dfa1b74e6a: 0eac601b-6d54-4767-b8b7-594aaf9990d3 +[[channels]] +channel_id = "1f18afa1-29c4-4634-99d1-68dfa1b74e6a" + +``` diff --git a/tools/provision/cmd/main.go b/tools/provision/cmd/main.go new file mode 100644 index 0000000..1b7461e --- /dev/null +++ b/tools/provision/cmd/main.go @@ -0,0 +1,42 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package main contains entry point for provisioning tool. +package main + +import ( + "log" + + "github.com/absmach/magistrala/tools/provision" + "github.com/spf13/cobra" +) + +func main() { + pconf := provision.Config{} + + rootCmd := &cobra.Command{ + Use: "provision", + Short: "provision is provisioning tool for Magistrala", + Long: `Tool for provisioning series of Magistrala channels and things and connecting them together. +Complete documentation is available at https://docs.magistrala.abstractmachines.fr`, + Run: func(_ *cobra.Command, _ []string) { + if err := provision.Provision(pconf); err != nil { + log.Fatal(err) + } + }, + } + + // Root Flags + rootCmd.PersistentFlags().StringVarP(&pconf.Host, "host", "", "https://localhost", "address for magistrala instance") + rootCmd.PersistentFlags().StringVarP(&pconf.Prefix, "prefix", "", "", "name prefix for things and channels") + rootCmd.PersistentFlags().StringVarP(&pconf.Username, "username", "u", "", "magistrala user") + rootCmd.PersistentFlags().StringVarP(&pconf.Password, "password", "p", "", "magistrala users password") + rootCmd.PersistentFlags().IntVarP(&pconf.Num, "num", "", 10, "number of channels and things to create and connect") + rootCmd.PersistentFlags().BoolVarP(&pconf.SSL, "ssl", "", false, "create certificates for mTLS access") + rootCmd.PersistentFlags().StringVarP(&pconf.CAKey, "cakey", "", "ca.key", "ca.key for creating and signing things certificate") + rootCmd.PersistentFlags().StringVarP(&pconf.CA, "ca", "", "ca.crt", "CA for creating and signing things certificate") + + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) + } +} diff --git a/tools/provision/doc.go b/tools/provision/doc.go new file mode 100644 index 0000000..342b0ab --- /dev/null +++ b/tools/provision/doc.go @@ -0,0 +1,7 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +// Package provision is a simple utility to create +// a list of channels and things connected to these channels +// with possibility to create certificates for mTLS use case. +package provision diff --git a/tools/provision/provision.go b/tools/provision/provision.go new file mode 100644 index 0000000..c4dbe45 --- /dev/null +++ b/tools/provision/provision.go @@ -0,0 +1,274 @@ +// Copyright (c) Abstract Machines +// SPDX-License-Identifier: Apache-2.0 + +package provision + +import ( + "bufio" + "bytes" + "crypto/ecdsa" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "log" + "math/big" + "os" + "time" + + "github.com/0x6flab/namegenerator" + sdk "github.com/absmach/magistrala/pkg/sdk/go" +) + +const ( + defPass = "12345678" + defReaderURL = "http://localhost:9005" +) + +var namesgenerator = namegenerator.NewGenerator() + +// MgConn - structure describing Magistrala connection set. +type MgConn struct { + ChannelID string + ThingID string + ThingKey string + MTLSCert string + MTLSKey string +} + +// Config - provisioning configuration. +type Config struct { + Host string + Username string + Password string + Num int + SSL bool + CA string + CAKey string + Prefix string +} + +// Provision - function that does actual provisiong. +func Provision(conf Config) error { + const ( + rsaBits = 4096 + ttl = "2400h" + ) + + msgContentType := string(sdk.CTJSONSenML) + sdkConf := sdk.Config{ + ThingsURL: conf.Host, + UsersURL: conf.Host, + ReaderURL: defReaderURL, + HTTPAdapterURL: fmt.Sprintf("%s/http", conf.Host), + BootstrapURL: conf.Host, + CertsURL: conf.Host, + MsgContentType: sdk.ContentType(msgContentType), + TLSVerification: false, + } + + s := sdk.NewSDK(sdkConf) + + user := sdk.User{ + Credentials: sdk.Credentials{ + Identity: conf.Username, + Secret: conf.Password, + }, + } + + if user.Credentials.Identity == "" { + user.Credentials.Identity = fmt.Sprintf("%s@email.com", namesgenerator.Generate()) + user.Credentials.Secret = defPass + } + + // Create new user + if _, err := s.CreateUser(user, ""); err != nil { + return fmt.Errorf("unable to create new user: %s", err.Error()) + } + + var err error + + // Login user + token, err := s.CreateToken(sdk.Login{Identity: user.Credentials.Identity, Secret: user.Credentials.Secret}) + if err != nil { + return fmt.Errorf("unable to login user: %s", err.Error()) + } + + var tlsCert tls.Certificate + var caCert *x509.Certificate + + if conf.SSL { + tlsCert, err = tls.LoadX509KeyPair(conf.CA, conf.CAKey) + if err != nil { + return fmt.Errorf("failed to load CA cert") + } + + b, err := os.ReadFile(conf.CA) + if err != nil { + return fmt.Errorf("failed to load CA cert") + } + + block, _ := pem.Decode(b) + if block == nil { + return fmt.Errorf("no PEM data found, failed to decode CA") + } + + caCert, err = x509.ParseCertificate(block.Bytes) + if err != nil { + return fmt.Errorf("failed to decode certificate - %s", err.Error()) + } + } + + // Create things and channels + things := make([]sdk.Thing, conf.Num) + channels := make([]sdk.Channel, conf.Num) + cIDs := []string{} + tIDs := []string{} + + fmt.Println("# List of things that can be connected to MQTT broker") + + for i := 0; i < conf.Num; i++ { + things[i] = sdk.Thing{Name: fmt.Sprintf("%s-thing-%d", conf.Prefix, i)} + channels[i] = sdk.Channel{Name: fmt.Sprintf("%s-channel-%d", conf.Prefix, i)} + } + + things, err = s.CreateThings(things, token.AccessToken) + if err != nil { + return fmt.Errorf("failed to create the things: %s", err.Error()) + } + + var chs []sdk.Channel + for _, c := range channels { + c, err = s.CreateChannel(c, token.AccessToken) + if err != nil { + return fmt.Errorf("failed to create the chennels: %s", err.Error()) + } + chs = append(chs, c) + } + channels = chs + + for _, t := range things { + tIDs = append(tIDs, t.ID) + } + + for _, c := range channels { + cIDs = append(cIDs, c.ID) + } + + for i := 0; i < conf.Num; i++ { + cert := "" + key := "" + + if conf.SSL { + var priv interface{} + priv, _ = rsa.GenerateKey(rand.Reader, rsaBits) + + notBefore := time.Now() + validFor, err := time.ParseDuration(ttl) + if err != nil { + return fmt.Errorf("failed to set date %v", validFor) + } + notAfter := notBefore.Add(validFor) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return fmt.Errorf("failed to generate serial number: %s", err) + } + + tmpl := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Magistrala"}, + CommonName: things[i].Credentials.Secret, + OrganizationalUnit: []string{"magistrala"}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + SubjectKeyId: []byte{1, 2, 3, 4, 6}, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, caCert, publicKey(priv), tlsCert.PrivateKey) + if err != nil { + return fmt.Errorf("failed to create certificate: %s", err) + } + + var bw, keyOut bytes.Buffer + buffWriter := bufio.NewWriter(&bw) + buffKeyOut := bufio.NewWriter(&keyOut) + + if err := pem.Encode(buffWriter, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + return fmt.Errorf("failed to write cert pem data: %s", err) + } + buffWriter.Flush() + cert = bw.String() + + if err := pem.Encode(buffKeyOut, pemBlockForKey(priv)); err != nil { + return fmt.Errorf("failed to write key pem data: %s", err) + } + buffKeyOut.Flush() + key = keyOut.String() + } + + // Print output + fmt.Printf("[[things]]\nthing_id = \"%s\"\nthing_key = \"%s\"\n", things[i].ID, things[i].Credentials.Secret) + if conf.SSL { + fmt.Printf("mtls_cert = \"\"\"%s\"\"\"\n", cert) + fmt.Printf("mtls_key = \"\"\"%s\"\"\"\n", key) + } + fmt.Println("") + } + + fmt.Printf("# List of channels that things can publish to\n" + + "# each channel is connected to each thing from things list\n") + for i := 0; i < conf.Num; i++ { + fmt.Printf("[[channels]]\nchannel_id = \"%s\"\n\n", cIDs[i]) + } + + for _, cID := range cIDs { + for _, tID := range tIDs { + conIDs := sdk.Connection{ + ThingID: tID, + ChannelID: cID, + } + if err := s.Connect(conIDs, token.AccessToken); err != nil { + log.Fatalf("Failed to connect things %s to channels %s: %s", tID, cID, err) + } + } + } + + return nil +} + +func publicKey(priv interface{}) interface{} { + switch k := priv.(type) { + case *rsa.PrivateKey: + return &k.PublicKey + case *ecdsa.PrivateKey: + return &k.PublicKey + default: + return nil + } +} + +func pemBlockForKey(priv interface{}) *pem.Block { + switch k := priv.(type) { + case *rsa.PrivateKey: + return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)} + case *ecdsa.PrivateKey: + b, err := x509.MarshalECPrivateKey(k) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to marshal ECDSA private key: %v", err) + os.Exit(2) + } + return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b} + default: + return nil + } +}