From 5073052c3ce5310f06ab3e8ef8b1c7c67628a926 Mon Sep 17 00:00:00 2001 From: sreepuramsudheer Date: Thu, 30 May 2024 19:31:15 +0530 Subject: [PATCH 1/8] Initial draft of telemetry. Signed-off-by: sreepuramsudheer --- src/oc_erchef/apps/chef_telemetry/.gitignore | 15 ++ src/oc_erchef/apps/chef_telemetry/Makefile | 30 ++++ src/oc_erchef/apps/chef_telemetry/README.md | 3 + .../apps/chef_telemetry/priv/.gitkeep | 0 .../apps/chef_telemetry/rebar.config | 30 ++++ .../chef_telemetry/src/chef_telemetry.app.src | 32 +++++ .../chef_telemetry/src/chef_telemetry.erl | 52 +++++++ .../chef_telemetry/src/chef_telemetry_app.erl | 34 +++++ .../chef_telemetry/src/chef_telemetry_sup.erl | 37 +++++ .../src/chef_telemetry_worker.erl | 133 ++++++++++++++++++ .../src/chef_teleterty_http.erl | 96 +++++++++++++ 11 files changed, 462 insertions(+) create mode 100644 src/oc_erchef/apps/chef_telemetry/.gitignore create mode 100644 src/oc_erchef/apps/chef_telemetry/Makefile create mode 100644 src/oc_erchef/apps/chef_telemetry/README.md create mode 100644 src/oc_erchef/apps/chef_telemetry/priv/.gitkeep create mode 100644 src/oc_erchef/apps/chef_telemetry/rebar.config create mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src create mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl create mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_app.erl create mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_sup.erl create mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl create mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_teleterty_http.erl diff --git a/src/oc_erchef/apps/chef_telemetry/.gitignore b/src/oc_erchef/apps/chef_telemetry/.gitignore new file mode 100644 index 0000000000..6a325fcd24 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/.gitignore @@ -0,0 +1,15 @@ +.eunit +deps/ +ebin/ +ebin_dialyzer/ +TAGS +.DS_Store +doc/*.html +*.beam +/doc/edoc-info +/doc/erlang.png +/doc/stylesheet.css +/deps.plt +test/*.out +.rebar +log/ diff --git a/src/oc_erchef/apps/chef_telemetry/Makefile b/src/oc_erchef/apps/chef_telemetry/Makefile new file mode 100644 index 0000000000..410366ea1c --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/Makefile @@ -0,0 +1,30 @@ +REBAR3_URL=https://s3.amazonaws.com/rebar3/rebar3 + +# If there is a rebar in the current directory, use it +ifeq ($(wildcard rebar3),rebar3) +REBAR3 = $(CURDIR)/rebar3 +endif + +# Fallback to rebar on PATH +REBAR3 ?= $(shell which rebar3) + +# And finally, prep to download rebar if all else fails +ifeq ($(REBAR3),) +REBAR3 = rebar3 +endif + +all: $(REBAR3) + @$(REBAR3) do clean, compile, eunit, dialyzer + +rel: all + @$(REBAR3) release + +distclean: + @rm -rf _build + +$(REBAR3): + curl -Lo rebar3 $(REBAR3_URL) || wget $(REBAR3_URL) + chmod a+x rebar3 + +install: $(REBAR3) distclean + $(REBAR3) update diff --git a/src/oc_erchef/apps/chef_telemetry/README.md b/src/oc_erchef/apps/chef_telemetry/README.md new file mode 100644 index 0000000000..a08073773c --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/README.md @@ -0,0 +1,3 @@ +# Telemetry + +Telemetry is an HTTP exporter of Chef Server node stats for external services. diff --git a/src/oc_erchef/apps/chef_telemetry/priv/.gitkeep b/src/oc_erchef/apps/chef_telemetry/priv/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/oc_erchef/apps/chef_telemetry/rebar.config b/src/oc_erchef/apps/chef_telemetry/rebar.config new file mode 100644 index 0000000000..a84b5bc0c3 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/rebar.config @@ -0,0 +1,30 @@ +%% -*- mode: erlang -*- +%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 ft=erlang et + +{deps, + [ + %% lager has to come first since we use its parse transform + {lager, ".*", + {git, "https://github.com/erlang-lager/lager", {branch, "master"}}}, + {opscoderl_httpc, ".*", + {git, "https://github.com/chef/opscoderl_httpc", {branch, "main"}}}, + {pooler, ".*", + {git, "https://github.com/chef/pooler", {branch, "master"}}} + ] +}. + +{profiles, [{ + test, [ + {deps, [meck]}, + {erl_opts, [export_all]} + ] +}]}. + +{erl_opts, [ + warnings_as_errors, + {parse_transform, lager_transform}, + debug_info +]}. + +{cover_enabled, true}. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src new file mode 100644 index 0000000000..54d495ed2e --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src @@ -0,0 +1,32 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- +%% ex: ts=4 sw=4 et +%% +%% +%% Copyright 2016 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +{application, chef_telemetry, [ + {description, "Chef Server Telemetry"}, + {vsn, {cmd,"cat ../../VERSION | awk '{print $0}'"}}, + {registered, []}, + {applications, [ + lager, + chef_secrets, + opscoderl_httpc + ]}, + {mod, {chef_telemetry_app, []}} +]}. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl new file mode 100644 index 0000000000..180a3726be --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl @@ -0,0 +1,52 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- +%% ex: ts=4 sw=4 et +%% +%% +%% Copyright 2016 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(chef_telemetry). +-export([ + ping/0, + is_enabled/0, + token/0 + ]). + +-spec ping() -> pong | pang. +ping() -> + case chef_telemetry_http:get("/") of + ok -> pong; + _ -> pang + end. + +-spec is_enabled() -> boolean(). +is_enabled() -> + case application:get_env(chef_telemetry, root_url) of + {ok, _Value} -> + true; + undefined -> + false + end. + +-spec token() -> string() | atom(). +token() -> + case chef_secrets:get(<<"data_collector">>, <<"token">>) of + {ok, Token} -> + erlang:binary_to_list(Token); + {error, not_found} -> + undefined + end. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_app.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_app.erl new file mode 100644 index 0000000000..cacbb18a06 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_app.erl @@ -0,0 +1,34 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- +%% ex: ts=4 sw=4 et +%% +%% Copyright 2016 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(chef_telemetry_app). + +-behaviour(application). + +%% API +-export([start/2, + stop/1 + ]). + +start(_StartType, _StartArgs) -> + chef_telemetry_sup:start_link(). + +stop(_State) -> + ok. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_sup.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_sup.erl new file mode 100644 index 0000000000..94b1c02463 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_sup.erl @@ -0,0 +1,37 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- +%% ex: ts=4 sw=4 et +%% +%% +%% Copyright 2016 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(chef_telemetry_sup). + +-behaviour(supervisor). + +-export([init/1, + start_link/0 + ]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + Worker = {chef_telemetry_worker, + {chef_telemetry_worker, start_link, []}, + permanent, 5000, supervisor, [chef_telemetry_worker]}, + {ok, {{one_for_one, 10, 10}, [Worker]}}. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl new file mode 100644 index 0000000000..bdea7abd90 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl @@ -0,0 +1,133 @@ +-module(chef_telemetry_worker). + +-behaviour(gen_server). + +-export([ + start_link/0, + solr_search/1 +]). + +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + code_change/2, + terminate/2 +]). + +-record(state, { + http_client, + timer_ref, + report_time +}). + +-record(oc_chef_organization, { + server_api_version, + id, + authz_id, + name, + full_name, + assigned_at, + last_updated_by, + created_at, + updated_at + }). + +-define(DEFAULT_DAYS, 30). + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +init(_Config) -> + % RootUrl = proplists:get_value(root_url, Config), + % Options = proplists:get_value(ibrowse_options, Config, []), + % HttpClient = oc_httpc_worker:start_link(RootUrl, Options, []), + State = #state{ + report_time = {12, 00}}, + gen_server:cast(self(), init_timer), + {ok, State}. + +handle_call(_Message, _From, State) -> + {noreply, State}. + +handle_cast(send_data, State) -> + ReqId = base64:encode(term_to_binary(make_ref())), + EpochNow = erlang:system_time(seconds), + TimeDuration = ?DEFAULT_DAYS * 86400, + Epoch30DaysOld = EpochNow - TimeDuration, + QueryString = lists:flatten(io_lib:format("ohai_time:{~p TO ~p}", [Epoch30DaysOld, EpochNow])), + Query1 = chef_index:query_from_params("node", QueryString, undefined, undefined), + DbContext = chef_db:make_context("1.0", ReqId, false), + _Count = + case chef_db:count_nodes(DbContext) of + Count1 when is_integer(Count1) -> Count1; + Error -> throw({db_error, Error}) + end, + Orgs = + case chef_db:list(#oc_chef_organization{}, DbContext) of + Orgs1 when is_list(Orgs1) -> Orgs1; + Error1 -> throw({db_error, Error1}) + end, + _Stats = [ {Org, get_org_nodes(Org, Query1, ReqId, DbContext)} || Org <- Orgs ], + % sending data logic hear + gen_server:cast(self(), init_timer), + {noreply, State}; + +handle_cast(init_timer, State) -> + {_Date, {Hour, Min, _Sec}} = erlang:universaltime(), + {RHour, RMin} = State#state.report_time, + CurrentDaySeconds = Hour * 3600 + Min * 60, + ReportingSeconds = RHour * 3600 + RMin * 60, + Diff = ReportingSeconds - CurrentDaySeconds, + if + Diff == 0 -> + gen_server:cast(self(), send_data); + Diff > 0 -> + timer:apply_after(Diff * 1000, gen_server, cast, [self(), send_data]); + Diff < 0 -> + timer:apply_after((Diff + 86400) * 1000, gen_server, cast, [self(), send_data]) + end, + {noreply, State}; + +handle_cast(_Message, State) -> + {noreply, State}. + +handle_info(_Message, State) -> + {noreply, State}. + +code_change(_OldVsn, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +get_org_nodes(OrgName, Query1, ReqId, DbContext) -> + {Guid1, _AuthzId1} = + case chef_db:fetch_org_metadata(DbContext, OrgName) of + not_found -> throw({org_not_found, OrgName}); + {Guid, AuthzId} -> {Guid, AuthzId} + end, + Query = chef_index:add_org_guid_to_query(Query1, Guid1), + case search(Query, ReqId) of + {ok, _Start0, _SolrNumFound, Ids} -> + erlang:length(Ids); + {error, {solr_400, _}=Why} -> + io:format("error while getting statestics ~p~n", Why); + {error, {solr_500, _}=Why} -> + io:format("error while getting statestics ~p~n", Why) + end. + +search(Query, ReqId) -> + stats_hero:ctime(ReqId, {chef_solr, search}, + fun() -> + solr_search(Query) + end). + +solr_search(Query) -> + try + chef_index:search(Query) + catch + Error:Reason -> + {Error, Reason} + end. \ No newline at end of file diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_teleterty_http.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_teleterty_http.erl new file mode 100644 index 0000000000..f6eed554a4 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_teleterty_http.erl @@ -0,0 +1,96 @@ +-module(chef_telemetry_http). + +-export([ + request/3, + request/4, + get/1, + get/2, + get/3, + post/2, + post/3, + delete/2, + delete/3, + create_pool/0, + delete_pool/0 + ]). + +-define(DEFAULT_HEADERS, [{"Content-Type", "application/json"}]). + +default_headers() -> + maybe_add_data_collector_token_header(?DEFAULT_HEADERS). + +maybe_add_data_collector_token_header(Headers) -> + case data_collector:token() of + undefined -> + Headers; + Token when is_list(Token) -> + [{"x-data-collector-token", Token} | Headers] + end. + +request(Path, Method, Body) -> + request(Path, Method, Body, default_headers()). + +request(Path, Method, Body, Headers) -> + {ok, Timeout} = application:get_env(data_collector, timeout), + oc_httpc:request(?MODULE, Path, Headers, Method, Body, Timeout). + +%% +%% Simple helpers for requets when you only +%% care about success or failure. +%% +-spec get(list()) -> ok | {error, term()}. +get(Path) -> + get(Path, [], default_headers()). + +-spec get(list(), iolist() | binary()) -> ok | {error, term()}. +get(Path, Body) -> + get(Path, Body, default_headers()). + +-spec get(list(), iolist() | binary(), list()) -> ok | {error, term()}. +get(Path, Body, Headers) -> + request_with_caught_errors(Path, get, Body, Headers). + +-spec post(list(), iolist() | binary()) -> ok | {error, term()}. +post(Path, Body) -> + post(Path, Body, default_headers()). + +-spec post(list(), iolist() | binary(), list()) -> ok | {error, term()}. +post(Path, Body, Headers) -> + request_with_caught_errors(Path, post, Body, Headers). + +-spec delete(list(), iolist() | binary()) -> ok | {error, term()}. +delete(Path, Body) -> + delete(Path, Body, default_headers()). + +-spec delete(list(), iolist() | binary(), list()) -> ok | {error, term()}. +delete(Path, Body, Headers) -> + request_with_caught_errors(Path, delete, Body, Headers). + +request_with_caught_errors(Path, Method, Body, Headers) when is_list(Body)-> + request_with_caught_errors(Path, Method, iolist_to_binary(Body), Headers); +request_with_caught_errors(Path, Method, Body, Headers) -> + try + case request(Path, Method, Body, Headers) of + {ok, [$2|_], _Head, _RespBody} -> ok; + Error -> {error, Error} + end + catch + How:Why -> + error_logger:error_report({?MODULE, Method, How, Why}), + {error, Why} + end. + +%% +%% Pool management functions +%% +-spec create_pool() -> ok. +create_pool() -> + lager:info("Creating Data Collector HTTP pool"), + oc_httpc:add_pool(?MODULE, application:get_all_env()), + ok. + +-spec delete_pool() -> ok. +delete_pool() -> + lager:info("Removing Data Collector HTTP pool"), + oc_httpc:delete_pool(?MODULE), + ok. From a61fd170d105e5b4d6a42fa5a7d9e46d116a1d9c Mon Sep 17 00:00:00 2001 From: sreepuramsudheer Date: Thu, 13 Jun 2024 17:12:30 +0530 Subject: [PATCH 2/8] Added code to get company name and server version. Signed-off-by: sreepuramsudheer --- .../src/chef_telemetry_worker.erl | 199 +++++++++++++++--- .../src/chef_teleterty_http.erl | 96 --------- 2 files changed, 170 insertions(+), 125 deletions(-) delete mode 100644 src/oc_erchef/apps/chef_telemetry/src/chef_teleterty_http.erl diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl index bdea7abd90..a35a2794b8 100644 --- a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl @@ -2,9 +2,10 @@ -behaviour(gen_server). +-include("../../../include/chef_types.hrl"). + -export([ - start_link/0, - solr_search/1 + start_link/0 ]). -export([ @@ -17,9 +18,10 @@ ]). -record(state, { - http_client, timer_ref, - report_time + report_time, + reporting_url, + http_client }). -record(oc_chef_organization, { @@ -36,15 +38,31 @@ -define(DEFAULT_DAYS, 30). +%% Setting this value for local ip because +%% 1) I don't have the server URL. +%% 2) easy for testing. +%% should be changed to actual server URL ASAP. +-define(DEFAULT_REPORTING_URL, "http://127.0.0.1:9001"). +-define(DEFAULT_REPORTING_TIME, {12, 00}). +-define(DEFAULT_IBROWSE_OPTIONS, []). + +-define(WINDOW_SECONDS, 300). + start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). init(_Config) -> - % RootUrl = proplists:get_value(root_url, Config), - % Options = proplists:get_value(ibrowse_options, Config, []), - % HttpClient = oc_httpc_worker:start_link(RootUrl, Options, []), + ReportingUrl = envy:get(chef_telemetry, reporting_url, ?DEFAULT_REPORTING_URL, string), + Fun = fun({Hour, Min}) -> + Hour >= 0 andalso Hour < 24 andalso Min >= 0 andalso Min < 60 + end, + ReportingTime = envy:get(chef_telemetry, reporting_time, ?DEFAULT_REPORTING_TIME, Fun), + Options = envy:get(chef_telemetry, ibrowse_options, ?DEFAULT_IBROWSE_OPTIONS, list), + HttpClient = oc_httpc_worker:start_link(ReportingUrl, Options, []), State = #state{ - report_time = {12, 00}}, + report_time = ReportingTime, + reporting_url = ReportingUrl, + http_client = HttpClient}, gen_server:cast(self(), init_timer), {ok, State}. @@ -52,25 +70,14 @@ handle_call(_Message, _From, State) -> {noreply, State}. handle_cast(send_data, State) -> + % code to get chef_server version. + [{_Server, ServerVersion, _, _}] = release_handler:which_releases(permanent), ReqId = base64:encode(term_to_binary(make_ref())), - EpochNow = erlang:system_time(seconds), - TimeDuration = ?DEFAULT_DAYS * 86400, - Epoch30DaysOld = EpochNow - TimeDuration, - QueryString = lists:flatten(io_lib:format("ohai_time:{~p TO ~p}", [Epoch30DaysOld, EpochNow])), - Query1 = chef_index:query_from_params("node", QueryString, undefined, undefined), - DbContext = chef_db:make_context("1.0", ReqId, false), - _Count = - case chef_db:count_nodes(DbContext) of - Count1 when is_integer(Count1) -> Count1; - Error -> throw({db_error, Error}) - end, - Orgs = - case chef_db:list(#oc_chef_organization{}, DbContext) of - Orgs1 when is_list(Orgs1) -> Orgs1; - Error1 -> throw({db_error, Error1}) - end, - _Stats = [ {Org, get_org_nodes(Org, Query1, ReqId, DbContext)} || Org <- Orgs ], - % sending data logic hear + {TotalNodes, _EndTime, _StartTime, ActiveNodes} = get_nodes(ReqId), + ScanTime = erlang:system_time(seconds), + CompanyName = get_company_name(), + Req = generate_request(CompanyName, list_to_binary(ServerVersion), TotalNodes, ActiveNodes, ScanTime), + send_data(Req, State), gen_server:cast(self(), init_timer), {noreply, State}; @@ -79,14 +86,15 @@ handle_cast(init_timer, State) -> {RHour, RMin} = State#state.report_time, CurrentDaySeconds = Hour * 3600 + Min * 60, ReportingSeconds = RHour * 3600 + RMin * 60, + DelaySeconds = rand:normal() * ?WINDOW_SECONDS, Diff = ReportingSeconds - CurrentDaySeconds, if Diff == 0 -> - gen_server:cast(self(), send_data); + timer:apply_after(DelaySeconds * 1000, get_server, cast, [self(), send_data]); Diff > 0 -> - timer:apply_after(Diff * 1000, gen_server, cast, [self(), send_data]); + timer:apply_after((Diff + DelaySeconds) * 1000, gen_server, cast, [self(), send_data]); Diff < 0 -> - timer:apply_after((Diff + 86400) * 1000, gen_server, cast, [self(), send_data]) + timer:apply_after((Diff + DelaySeconds + 86400) * 1000, gen_server, cast, [self(), send_data]) end, {noreply, State}; @@ -130,4 +138,137 @@ solr_search(Query) -> catch Error:Reason -> {Error, Reason} + end. + +get_company_name() -> + case sqerl:adhoc_select([email], users, all, []) of + {ok, Ids1} -> + Ids = [Id || [{_, Id}] <- Ids1], + Fun = + fun(Email) -> + case re:run(Email, "^[^@]*@\([^.]*\)\..*$") of + {match, [_, {Pos, Len} | _]} -> + {true, binary:part(Email, Pos, Len)}; + _ -> + false + end + end, + CompanyNames = lists:filtermap(Fun, Ids), + if length(CompanyNames) -> + throw("no valid Email Ids."); + true -> + get_most_occuring(CompanyNames) + end; + Error -> + throw(Error) + end. + +get_most_occuring(List) -> + FirstElement = lists:nth(1, List), + Fun = fun(Element, Map) -> + Count = maps:get(Element, Map, 0), + maps:put(Element, Count + 1, Map) + end, + Map1 = lists:foldl(Fun, #{}, List), + + Fun1 = + fun(Key1, Count1, {Key2, Count2}) -> + if + Count1 > Count2 -> + {Key1, Count1}; + true -> + {Key2, Count2} + end + end, + Res1 = maps:fold(Fun1, {FirstElement, 0}, Map1), + element(1, Res1). + +get_nodes(ReqId) -> + EpochNow = erlang:system_time(seconds), + TimeDuration = ?DEFAULT_DAYS * 86400, + Epoch30DaysOld = EpochNow - TimeDuration, + QueryString = lists:flatten(io_lib:format("ohai_time:{~p TO ~p}", [Epoch30DaysOld, EpochNow])), + Query1 = chef_index:query_from_params("node", QueryString, undefined, undefined), + DbContext = chef_db:make_context("1.0", ReqId, false), + Count = + case chef_db:count_nodes(DbContext) of + Count1 when is_integer(Count1) -> Count1; + Error -> throw({db_error, Error}) + end, + Orgs = + case chef_db:list(#oc_chef_organization{}, DbContext) of + Orgs1 when is_list(Orgs1) -> Orgs1; + Error1 -> throw({db_error, Error1}) + end, + Stats = [ {Org, get_org_nodes(Org, Query1, ReqId, DbContext)} || Org <- Orgs ], + Fun = fun({_Org, Nodes}, Sum) -> + Sum + Nodes + end, + ActiveNodes = lists:foldl(Fun, 0, Stats), + {Count, calendar:system_time_to_rfc3339(EpochNow,[{offset, "Z"}]), calendar:system_time_to_rfc3339(Epoch30DaysOld,[{offset, "Z"}]), ActiveNodes}. + +generate_request(CompanyName, ServerVersion, TotalNodes, ActiveNodes, ScanTime) -> + jiffy:encode({[ + {<<"licenseId">>, <<"Infra-Server-license-Id">>}, + %%{<<"customerId">>, <<"">>}, + %%{<<"expiration">>, <<"2023-11-30T00:00:00Z">>}, + {<<"customerName">>, to_binary(CompanyName)}, + {<<"periods">>, [ + {[ + {<<"version">>, to_binary(ServerVersion)}, + {<<"date">>, to_binary(epoch_to_string(ScanTime))}, + {<<"period">>, {[ + {<<"start">>, to_binary(epoch_to_string(ScanTime))}, + {<<"end">>, to_binary(epoch_to_string(ScanTime))} + ]}}, + {<<"summary">>, {[ + {<<"nodes">>, {[ + {<<"total">>, TotalNodes}, + {<<"active">>, ActiveNodes} + ]}}, + {<<"scans">>, {[ + {<<"total">>, 0}, + {<<"targets">>, 0} + ]}}, + {<<"services">>, {[ + {<<"targets">>, 0} + ]}} + ]}}, + {<<"evidence">>, {[ + {<<"nodes">>, null}, + {<<"scans">>, null}, + {<<"content">>, null} + ]}} + ]} + ]}, + {<<"metadata">>, {[ + {<<"Infra Server">>, {[ + {<<"deploymentType">>, <<"">>}, + {<<"instanceId">>, <<"">>}, + {<<"fqdn">>, <<"">>}, + {<<"config_location">>, <<"">>}, + {<<"binary_location">>, <<"">>} + ]}} + ]}}, + {<<"source">>, <<"Infra Server">>}, + {<<"scannerVersion">>, <<"0.1.0">>}, + {<<"scannedOn">>, to_binary(epoch_to_string(ScanTime))} + ]}). + +to_binary(String) when is_list(String) -> + list_to_binary(String); + +to_binary(Bin) when is_binary(Bin) -> + Bin; + +to_binary(Element) -> + throw({not_a_binary_or_string, Element}). + +epoch_to_string(Epoch) -> + calendar:system_time_to_rfc3339(Epoch, [{offset, "Z"}]). + +send_data(Req, State) -> + case oc_httpc_worker:request(State#state.http_client,"", [], post, Req, 5000) of + {ok, _Status, _ResponseHeaders, _ResponseBody} -> ok; + {error, Reason} -> throw({failed_sending_request, Reason}) end. \ No newline at end of file diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_teleterty_http.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_teleterty_http.erl deleted file mode 100644 index f6eed554a4..0000000000 --- a/src/oc_erchef/apps/chef_telemetry/src/chef_teleterty_http.erl +++ /dev/null @@ -1,96 +0,0 @@ --module(chef_telemetry_http). - --export([ - request/3, - request/4, - get/1, - get/2, - get/3, - post/2, - post/3, - delete/2, - delete/3, - create_pool/0, - delete_pool/0 - ]). - --define(DEFAULT_HEADERS, [{"Content-Type", "application/json"}]). - -default_headers() -> - maybe_add_data_collector_token_header(?DEFAULT_HEADERS). - -maybe_add_data_collector_token_header(Headers) -> - case data_collector:token() of - undefined -> - Headers; - Token when is_list(Token) -> - [{"x-data-collector-token", Token} | Headers] - end. - -request(Path, Method, Body) -> - request(Path, Method, Body, default_headers()). - -request(Path, Method, Body, Headers) -> - {ok, Timeout} = application:get_env(data_collector, timeout), - oc_httpc:request(?MODULE, Path, Headers, Method, Body, Timeout). - -%% -%% Simple helpers for requets when you only -%% care about success or failure. -%% --spec get(list()) -> ok | {error, term()}. -get(Path) -> - get(Path, [], default_headers()). - --spec get(list(), iolist() | binary()) -> ok | {error, term()}. -get(Path, Body) -> - get(Path, Body, default_headers()). - --spec get(list(), iolist() | binary(), list()) -> ok | {error, term()}. -get(Path, Body, Headers) -> - request_with_caught_errors(Path, get, Body, Headers). - --spec post(list(), iolist() | binary()) -> ok | {error, term()}. -post(Path, Body) -> - post(Path, Body, default_headers()). - --spec post(list(), iolist() | binary(), list()) -> ok | {error, term()}. -post(Path, Body, Headers) -> - request_with_caught_errors(Path, post, Body, Headers). - --spec delete(list(), iolist() | binary()) -> ok | {error, term()}. -delete(Path, Body) -> - delete(Path, Body, default_headers()). - --spec delete(list(), iolist() | binary(), list()) -> ok | {error, term()}. -delete(Path, Body, Headers) -> - request_with_caught_errors(Path, delete, Body, Headers). - -request_with_caught_errors(Path, Method, Body, Headers) when is_list(Body)-> - request_with_caught_errors(Path, Method, iolist_to_binary(Body), Headers); -request_with_caught_errors(Path, Method, Body, Headers) -> - try - case request(Path, Method, Body, Headers) of - {ok, [$2|_], _Head, _RespBody} -> ok; - Error -> {error, Error} - end - catch - How:Why -> - error_logger:error_report({?MODULE, Method, How, Why}), - {error, Why} - end. - -%% -%% Pool management functions -%% --spec create_pool() -> ok. -create_pool() -> - lager:info("Creating Data Collector HTTP pool"), - oc_httpc:add_pool(?MODULE, application:get_all_env()), - ok. - --spec delete_pool() -> ok. -delete_pool() -> - lager:info("Removing Data Collector HTTP pool"), - oc_httpc:delete_pool(?MODULE), - ok. From 780590d0878f06482aa3774fd9cb609191049279 Mon Sep 17 00:00:00 2001 From: sreepuramsudheer Date: Tue, 2 Jul 2024 17:58:09 +0530 Subject: [PATCH 3/8] Added code to send http request. --- .../src/chef_telemetry_worker.erl | 144 ++++++++++++++---- 1 file changed, 111 insertions(+), 33 deletions(-) diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl index a35a2794b8..8a6c77a5a0 100644 --- a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl @@ -17,11 +17,23 @@ terminate/2 ]). +-record(current_scan, { + scan_start_time, + scan_end_time, + total_nodes, + active_nodes, + company_name +}). + -record(state, { timer_ref, report_time, reporting_url, - http_client + http_client, + db_context, + req_id, + scan_time, + current_scan }). -record(oc_chef_organization, { @@ -58,7 +70,7 @@ init(_Config) -> end, ReportingTime = envy:get(chef_telemetry, reporting_time, ?DEFAULT_REPORTING_TIME, Fun), Options = envy:get(chef_telemetry, ibrowse_options, ?DEFAULT_IBROWSE_OPTIONS, list), - HttpClient = oc_httpc_worker:start_link(ReportingUrl, Options, []), + {ok, HttpClient} = oc_httpc_worker:start_link(ReportingUrl, Options, []), State = #state{ report_time = ReportingTime, reporting_url = ReportingUrl, @@ -70,23 +82,29 @@ handle_call(_Message, _From, State) -> {noreply, State}. handle_cast(send_data, State) -> + State1 = init_req(State), % code to get chef_server version. - [{_Server, ServerVersion, _, _}] = release_handler:which_releases(permanent), - ReqId = base64:encode(term_to_binary(make_ref())), - {TotalNodes, _EndTime, _StartTime, ActiveNodes} = get_nodes(ReqId), - ScanTime = erlang:system_time(seconds), - CompanyName = get_company_name(), - Req = generate_request(CompanyName, list_to_binary(ServerVersion), TotalNodes, ActiveNodes, ScanTime), - send_data(Req, State), + ShouldSend = check_send(State1), + case ShouldSend of + true -> + [{_Server, ServerVersion, _, _}] = release_handler:which_releases(permanent), + State2 = get_nodes(State1), + State3 = get_company_name(State2), + Req = generate_request(list_to_binary(ServerVersion), State3), + send_data(Req, State3), + State3; + _ -> + State1 + end, gen_server:cast(self(), init_timer), - {noreply, State}; + {noreply, State1}; handle_cast(init_timer, State) -> - {_Date, {Hour, Min, _Sec}} = erlang:universaltime(), + {_Date, {Hour, Min, _Sec}} = erlang:now_to_universal_time(State#state.scan_time), {RHour, RMin} = State#state.report_time, CurrentDaySeconds = Hour * 3600 + Min * 60, ReportingSeconds = RHour * 3600 + RMin * 60, - DelaySeconds = rand:normal() * ?WINDOW_SECONDS, + DelaySeconds = floor(rand:uniform() * ?WINDOW_SECONDS), Diff = ReportingSeconds - CurrentDaySeconds, if Diff == 0 -> @@ -110,6 +128,21 @@ code_change(_OldVsn, State) -> terminate(_Reason, _State) -> ok. +init_req(State) -> + ReqId = base64:encode(term_to_binary(make_ref())), + DbContext = chef_db:make_context("1.0", ReqId, false), + CurrentTime = erlang:system_time(seconds), + StartTime = CurrentTime - (?DEFAULT_DAYS * 86400), + State#state{ + req_id = ReqId, + db_context = DbContext, + scan_time = CurrentTime, + current_scan = + #current_scan{ + scan_start_time = StartTime, + scan_end_time = CurrentTime} + }. + get_org_nodes(OrgName, Query1, ReqId, DbContext) -> {Guid1, _AuthzId1} = case chef_db:fetch_org_metadata(DbContext, OrgName) of @@ -140,7 +173,8 @@ solr_search(Query) -> {Error, Reason} end. -get_company_name() -> +get_company_name(State) -> + CompanyName = case sqerl:adhoc_select([email], users, all, []) of {ok, Ids1} -> Ids = [Id || [{_, Id}] <- Ids1], @@ -154,14 +188,17 @@ get_company_name() -> end end, CompanyNames = lists:filtermap(Fun, Ids), - if length(CompanyNames) -> + if length(CompanyNames) == 0 -> throw("no valid Email Ids."); true -> get_most_occuring(CompanyNames) end; Error -> throw(Error) - end. + end, + CurrentScan = State#state.current_scan, + State#state{ + current_scan = CurrentScan#current_scan{company_name = CompanyName}}. get_most_occuring(List) -> FirstElement = lists:nth(1, List), @@ -183,13 +220,12 @@ get_most_occuring(List) -> Res1 = maps:fold(Fun1, {FirstElement, 0}, Map1), element(1, Res1). -get_nodes(ReqId) -> - EpochNow = erlang:system_time(seconds), - TimeDuration = ?DEFAULT_DAYS * 86400, - Epoch30DaysOld = EpochNow - TimeDuration, - QueryString = lists:flatten(io_lib:format("ohai_time:{~p TO ~p}", [Epoch30DaysOld, EpochNow])), +get_nodes(#state{req_id = ReqId, db_context = DbContext} = State) -> + CurrentScan = State#state.current_scan, + ScanStartTime = CurrentScan#current_scan.scan_start_time, + ScanEndTime = CurrentScan#current_scan.scan_end_time, + QueryString = lists:flatten(io_lib:format("ohai_time:{~p TO ~p}", [ScanStartTime, ScanEndTime])), Query1 = chef_index:query_from_params("node", QueryString, undefined, undefined), - DbContext = chef_db:make_context("1.0", ReqId, false), Count = case chef_db:count_nodes(DbContext) of Count1 when is_integer(Count1) -> Count1; @@ -205,26 +241,30 @@ get_nodes(ReqId) -> Sum + Nodes end, ActiveNodes = lists:foldl(Fun, 0, Stats), - {Count, calendar:system_time_to_rfc3339(EpochNow,[{offset, "Z"}]), calendar:system_time_to_rfc3339(Epoch30DaysOld,[{offset, "Z"}]), ActiveNodes}. + State#state{ + current_scan = CurrentScan#current_scan{ + total_nodes = Count, + active_nodes = ActiveNodes}}. -generate_request(CompanyName, ServerVersion, TotalNodes, ActiveNodes, ScanTime) -> +generate_request(ServerVersion, State) -> + CurrentScan = State#state.current_scan, jiffy:encode({[ {<<"licenseId">>, <<"Infra-Server-license-Id">>}, %%{<<"customerId">>, <<"">>}, %%{<<"expiration">>, <<"2023-11-30T00:00:00Z">>}, - {<<"customerName">>, to_binary(CompanyName)}, + {<<"customerName">>, to_binary(State#state.current_scan#current_scan.company_name)}, {<<"periods">>, [ {[ {<<"version">>, to_binary(ServerVersion)}, - {<<"date">>, to_binary(epoch_to_string(ScanTime))}, + {<<"date">>, to_binary(epoch_to_string(CurrentScan#current_scan.scan_end_time))}, {<<"period">>, {[ - {<<"start">>, to_binary(epoch_to_string(ScanTime))}, - {<<"end">>, to_binary(epoch_to_string(ScanTime))} + {<<"start">>, to_binary(epoch_to_string(CurrentScan#current_scan.scan_start_time))}, + {<<"end">>, to_binary(epoch_to_string(CurrentScan#current_scan.scan_end_time))} ]}}, {<<"summary">>, {[ {<<"nodes">>, {[ - {<<"total">>, TotalNodes}, - {<<"active">>, ActiveNodes} + {<<"total">>, CurrentScan#current_scan.total_nodes}, + {<<"active">>, CurrentScan#current_scan.active_nodes} ]}}, {<<"scans">>, {[ {<<"total">>, 0}, @@ -252,7 +292,7 @@ generate_request(CompanyName, ServerVersion, TotalNodes, ActiveNodes, ScanTime) ]}}, {<<"source">>, <<"Infra Server">>}, {<<"scannerVersion">>, <<"0.1.0">>}, - {<<"scannedOn">>, to_binary(epoch_to_string(ScanTime))} + {<<"scannedOn">>, to_binary(epoch_to_string(State#state.scan_time))} ]}). to_binary(String) when is_list(String) -> @@ -268,7 +308,45 @@ epoch_to_string(Epoch) -> calendar:system_time_to_rfc3339(Epoch, [{offset, "Z"}]). send_data(Req, State) -> - case oc_httpc_worker:request(State#state.http_client,"", [], post, Req, 5000) of + case catch ibrowse:send_req(State#state.reporting_url, [], post, Req, [], 5000) of {ok, _Status, _ResponseHeaders, _ResponseBody} -> ok; - {error, Reason} -> throw({failed_sending_request, Reason}) - end. \ No newline at end of file + Error -> throw({failed_sending_request, Error}) + end. + +check_send(State) -> + Node = erlang:atom_to_binary(node()), + Now = calendar:system_time_to_universal_time(State#state.scan_time, second), + case sqerl:adhoc_select([timestamp1], telemetry, {property, equals, "last_send"}, []) of + {ok, Rows} when is_list(Rows) -> + LastSend = to_system_time(Rows), + case should_send(LastSend, State) of + true -> + sqerl:adhoc_delete(telemetry, {<<"property">>, equals, <<"last_send">>}), + sqerl:adhoc_insert(telemetry, [[{<<"property">>, <<"last_send">>}, {<<"valuestring">>, Node}, {<<"timestamp1">>, Now}]]), + true; + false -> + false + end; + {ok, Rows} when is_list(Rows) andalso length(Rows) == 0 -> + sqerl:adhoc_insert(telemetry, [[{<<"property">>, <<"last_send">>}, {<<"valuestring">>, Node}, {<<"timestamp1">>, Now}]]), + true; + Error -> + throw({not_able_to_gead_from_db, Error}) + end. + +to_system_time(Rows) -> + TimeStamps1 = [ proplists:get_value(<<"timestamp1">>, Row) || Row <- Rows, not (proplists:get_value(<<"timestamp1">>, Row) == undefined) ], + SystemTimes = [calendar:datetime_to_gregorian_seconds({Date, {H, M, floor(S1)}}) - 62167219200 || {Date, {H, M, S1}} <- TimeStamps1], + MaxFun = + fun(Time, Max) -> + case Time > Max of + true -> + Time; + _ -> + Max + end + end, + lists:foldl( MaxFun, 0, SystemTimes). + +should_send(LastSend, State) -> + LastSend < State#state.scan_time. \ No newline at end of file From 29c32621d581c6b4939beda1021369e98fcf0624 Mon Sep 17 00:00:00 2001 From: sreepuramsudheer Date: Tue, 23 Jul 2024 16:43:27 +0530 Subject: [PATCH 4/8] Added config and ctl location details. Signed-off-by: sreepuramsudheer --- .../infra-server/attributes/default.rb | 2 + .../templates/default/oc_erchef.config.erb | 8 +++- .../src/chef_telemetry_worker.erl | 46 +++++++++++++++---- src/oc_erchef/src/oc_erchef.app.src | 3 +- 4 files changed, 47 insertions(+), 12 deletions(-) diff --git a/omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb b/omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb index 3f76386963..77f85dcdfe 100755 --- a/omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb +++ b/omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb @@ -893,6 +893,8 @@ # Select whether data_collector affects overall status in _status endpoint default['private_chef']['data_collector']['health_check'] = true +default['private_chef']['ctl_command'] = "#{ChefUtils::Dist::Server::SERVER_CTL}" +default['private_chef']['running_filepath'] = "/etc/#{ChefUtils::Dist::Org::LEGACY_CONF_DIR}/#{ChefUtils::Dist::Server::SERVER}-running.json" ## # Compliance Profiles ## diff --git a/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb b/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb index ee0bfc2656..c020307c0f 100755 --- a/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb +++ b/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb @@ -320,7 +320,13 @@ }, {metrics_module, folsom_metrics} -]} + ]}, + + {chef_telemetry, [ + {running_filepath, "<%= node['private_chef']['running_filepath'] %>"}, + {ctl_command, "<%= node['private_chef']['ctl_command'] %>"} + ] + } <% if !node['private_chef']['opscode-erchef']['ssl_session_caching']['enabled'] -%> , diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl index 8a6c77a5a0..1a8c49b7fd 100644 --- a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl @@ -33,7 +33,10 @@ db_context, req_id, scan_time, - current_scan + current_scan, + running_file, + ctl_command, + fqdns }). -record(oc_chef_organization, { @@ -70,11 +73,17 @@ init(_Config) -> end, ReportingTime = envy:get(chef_telemetry, reporting_time, ?DEFAULT_REPORTING_TIME, Fun), Options = envy:get(chef_telemetry, ibrowse_options, ?DEFAULT_IBROWSE_OPTIONS, list), + ConfigFile= envy:get(chef_telemetry, config_file, "", string), + Ctl = envy:get(chef_telemetry, ctl_command, "", string), + Cmd = "which " ++ Ctl, + CtlLocation = os:cmd(Cmd), {ok, HttpClient} = oc_httpc_worker:start_link(ReportingUrl, Options, []), State = #state{ report_time = ReportingTime, reporting_url = ReportingUrl, - http_client = HttpClient}, + http_client = HttpClient, + running_file = ConfigFile, + ctl_command = CtlLocation}, gen_server:cast(self(), init_timer), {ok, State}. @@ -83,14 +92,14 @@ handle_call(_Message, _From, State) -> handle_cast(send_data, State) -> State1 = init_req(State), - % code to get chef_server version. - ShouldSend = check_send(State1), - case ShouldSend of + insert_fqdn(State1), + case check_send(State1) of true -> [{_Server, ServerVersion, _, _}] = release_handler:which_releases(permanent), State2 = get_nodes(State1), State3 = get_company_name(State2), - Req = generate_request(list_to_binary(ServerVersion), State3), + State4 = get_api_fqdn(State3), + Req = generate_request(list_to_binary(ServerVersion), State4), send_data(Req, State3), State3; _ -> @@ -143,6 +152,15 @@ init_req(State) -> scan_end_time = CurrentTime} }. +get_api_fqdn(State) -> + case sqerl:execute(<<"select property from telemetry where property like 'FQDN:%'">>) of + {ok, Rows} when is_list(Rows) -> + FQDNs = [binary:part(FQDN, 5, size(FQDN) -5) || [{<<"property">>, FQDN}] <- Rows], + State#state{fqdns = FQDNs}; + _ -> + State + end. + get_org_nodes(OrgName, Query1, ReqId, DbContext) -> {Guid1, _AuthzId1} = case chef_db:fetch_org_metadata(DbContext, OrgName) of @@ -285,9 +303,9 @@ generate_request(ServerVersion, State) -> {<<"Infra Server">>, {[ {<<"deploymentType">>, <<"">>}, {<<"instanceId">>, <<"">>}, - {<<"fqdn">>, <<"">>}, - {<<"config_location">>, <<"">>}, - {<<"binary_location">>, <<"">>} + {<<"fqdn">>, State#state.fqdns}, + {<<"config_location">>, to_binary(State#state.running_file)}, + {<<"binary_location">>, to_binary(State#state.ctl_command)} ]}} ]}}, {<<"source">>, <<"Infra Server">>}, @@ -349,4 +367,12 @@ to_system_time(Rows) -> lists:foldl( MaxFun, 0, SystemTimes). should_send(LastSend, State) -> - LastSend < State#state.scan_time. \ No newline at end of file + LastSend < State#state.scan_time. + +insert_fqdn(State) -> + {ok, HostName} = inet:gethostname(), + Now = calendar:system_time_to_universal_time(State#state.scan_time, second), + %%Hostname = os:cmd('hostname -f'), + HostName1 = "FQDN:" ++ HostName, + sqerl:adhoc_delete(telemetry, {property, equals, HostName1}), + sqerl:adhoc_insert(telemetry, [[{<<"property">>, to_binary(HostName1)}, {<<"timestamp1">>, Now}]]). diff --git a/src/oc_erchef/src/oc_erchef.app.src b/src/oc_erchef/src/oc_erchef.app.src index 3a6615781b..4395947ecc 100644 --- a/src/oc_erchef/src/oc_erchef.app.src +++ b/src/oc_erchef/src/oc_erchef.app.src @@ -31,7 +31,8 @@ opscoderl_httpc, oc_chef_authz, oc_chef_wm, - data_collector + data_collector, + chef_telemetry ]}, {applications, [lager, chef_secrets, From d56d168623acc8932e0fd120d567f1184915c06d Mon Sep 17 00:00:00 2001 From: sreepuramsudheer Date: Tue, 30 Jul 2024 19:59:06 +0530 Subject: [PATCH 5/8] Added schema changes. Signed-off-by: sreepuramsudheer --- .../apps/chef_telemetry/rebar.config | 6 +- .../chef_telemetry/src/chef_telemetry.erl | 26 +--- .../src/chef_telemetry_worker.erl | 126 +++++++++--------- src/oc_erchef/schema/deploy/telemetry.sql | 11 ++ src/oc_erchef/schema/revert/telemetry.sql | 7 + src/oc_erchef/schema/sqitch.plan | 2 + src/oc_erchef/schema/verify/telemetry.sql | 7 + 7 files changed, 96 insertions(+), 89 deletions(-) create mode 100644 src/oc_erchef/schema/deploy/telemetry.sql create mode 100644 src/oc_erchef/schema/revert/telemetry.sql create mode 100644 src/oc_erchef/schema/verify/telemetry.sql diff --git a/src/oc_erchef/apps/chef_telemetry/rebar.config b/src/oc_erchef/apps/chef_telemetry/rebar.config index a84b5bc0c3..7cbf7d6114 100644 --- a/src/oc_erchef/apps/chef_telemetry/rebar.config +++ b/src/oc_erchef/apps/chef_telemetry/rebar.config @@ -6,11 +6,7 @@ [ %% lager has to come first since we use its parse transform {lager, ".*", - {git, "https://github.com/erlang-lager/lager", {branch, "master"}}}, - {opscoderl_httpc, ".*", - {git, "https://github.com/chef/opscoderl_httpc", {branch, "main"}}}, - {pooler, ".*", - {git, "https://github.com/chef/pooler", {branch, "master"}}} + {git, "https://github.com/erlang-lager/lager", {branch, "master"}}} ] }. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl index 180a3726be..dd9a6bbb2e 100644 --- a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl @@ -21,32 +21,14 @@ -module(chef_telemetry). -export([ - ping/0, - is_enabled/0, - token/0 + is_enabled/0 ]). --spec ping() -> pong | pang. -ping() -> - case chef_telemetry_http:get("/") of - ok -> pong; - _ -> pang - end. - -spec is_enabled() -> boolean(). is_enabled() -> - case application:get_env(chef_telemetry, root_url) of - {ok, _Value} -> + case envy:get(chef_telemetry, is_enabled, true, boolean) of + true -> true; - undefined -> + _ -> false end. - --spec token() -> string() | atom(). -token() -> - case chef_secrets:get(<<"data_collector">>, <<"token">>) of - {ok, Token} -> - erlang:binary_to_list(Token); - {error, not_found} -> - undefined - end. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl index 1a8c49b7fd..36f6da3e44 100644 --- a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl @@ -29,7 +29,6 @@ timer_ref, report_time, reporting_url, - http_client, db_context, req_id, scan_time, @@ -70,18 +69,15 @@ init(_Config) -> ReportingUrl = envy:get(chef_telemetry, reporting_url, ?DEFAULT_REPORTING_URL, string), Fun = fun({Hour, Min}) -> Hour >= 0 andalso Hour < 24 andalso Min >= 0 andalso Min < 60 - end, + end, ReportingTime = envy:get(chef_telemetry, reporting_time, ?DEFAULT_REPORTING_TIME, Fun), - Options = envy:get(chef_telemetry, ibrowse_options, ?DEFAULT_IBROWSE_OPTIONS, list), - ConfigFile= envy:get(chef_telemetry, config_file, "", string), + ConfigFile= envy:get(chef_telemetry, running_filepath, "", string), Ctl = envy:get(chef_telemetry, ctl_command, "", string), Cmd = "which " ++ Ctl, - CtlLocation = os:cmd(Cmd), - {ok, HttpClient} = oc_httpc_worker:start_link(ReportingUrl, Options, []), + CtlLocation = string:trim(os:cmd(Cmd)), State = #state{ report_time = ReportingTime, reporting_url = ReportingUrl, - http_client = HttpClient, running_file = ConfigFile, ctl_command = CtlLocation}, gen_server:cast(self(), init_timer), @@ -91,36 +87,41 @@ handle_call(_Message, _From, State) -> {noreply, State}. handle_cast(send_data, State) -> - State1 = init_req(State), - insert_fqdn(State1), - case check_send(State1) of - true -> - [{_Server, ServerVersion, _, _}] = release_handler:which_releases(permanent), - State2 = get_nodes(State1), - State3 = get_company_name(State2), - State4 = get_api_fqdn(State3), - Req = generate_request(list_to_binary(ServerVersion), State4), - send_data(Req, State3), - State3; - _ -> - State1 - end, + State6 = + case chef_telemetry:is_enabled() of + true -> + State1 = init_req(State), + insert_fqdn(State1), + case check_send(State1) of + true -> + [{_Server, ServerVersion, _, _}] = release_handler:which_releases(permanent), + State2 = get_nodes(State1), + State3 = get_company_name(State2), + State4 = get_api_fqdn(State3), + Req = generate_request(list_to_binary(ServerVersion), State4), + send_data(Req, State3), + State3; + _ -> + State1 + end; + _ -> + State + end, gen_server:cast(self(), init_timer), - {noreply, State1}; + {noreply, State6}; handle_cast(init_timer, State) -> - {_Date, {Hour, Min, _Sec}} = erlang:now_to_universal_time(State#state.scan_time), + {_Date, {Hour, Min, _Sec}} = calendar:now_to_universal_time(erlang:timestamp()), {RHour, RMin} = State#state.report_time, CurrentDaySeconds = Hour * 3600 + Min * 60, ReportingSeconds = RHour * 3600 + RMin * 60, DelaySeconds = floor(rand:uniform() * ?WINDOW_SECONDS), - Diff = ReportingSeconds - CurrentDaySeconds, - if - Diff == 0 -> + case ReportingSeconds - CurrentDaySeconds of + Diff when Diff == 0 -> timer:apply_after(DelaySeconds * 1000, get_server, cast, [self(), send_data]); - Diff > 0 -> + Diff when Diff > 0 -> timer:apply_after((Diff + DelaySeconds) * 1000, gen_server, cast, [self(), send_data]); - Diff < 0 -> + Diff -> timer:apply_after((Diff + DelaySeconds + 86400) * 1000, gen_server, cast, [self(), send_data]) end, {noreply, State}; @@ -146,13 +147,14 @@ init_req(State) -> req_id = ReqId, db_context = DbContext, scan_time = CurrentTime, - current_scan = + current_scan = #current_scan{ scan_start_time = StartTime, scan_end_time = CurrentTime} }. get_api_fqdn(State) -> + sqerl:execute(<<"delete from telemetry where property like 'FQDN:%' and event_timestamp < (current_timestamp - interval '86700')">>), case sqerl:execute(<<"select property from telemetry where property like 'FQDN:%'">>) of {ok, Rows} when is_list(Rows) -> FQDNs = [binary:part(FQDN, 5, size(FQDN) -5) || [{<<"property">>, FQDN}] <- Rows], @@ -160,9 +162,9 @@ get_api_fqdn(State) -> _ -> State end. - + get_org_nodes(OrgName, Query1, ReqId, DbContext) -> - {Guid1, _AuthzId1} = + {Guid1, _AuthzId1} = case chef_db:fetch_org_metadata(DbContext, OrgName) of not_found -> throw({org_not_found, OrgName}); {Guid, AuthzId} -> {Guid, AuthzId} @@ -172,9 +174,9 @@ get_org_nodes(OrgName, Query1, ReqId, DbContext) -> {ok, _Start0, _SolrNumFound, Ids} -> erlang:length(Ids); {error, {solr_400, _}=Why} -> - io:format("error while getting statestics ~p~n", Why); + throw({error_getting_nodes, Why}); {error, {solr_500, _}=Why} -> - io:format("error while getting statestics ~p~n", Why) + throw({error_getting_nodes, Why}) end. search(Query, ReqId) -> @@ -192,12 +194,12 @@ solr_search(Query) -> end. get_company_name(State) -> - CompanyName = - case sqerl:adhoc_select([email], users, all, []) of + CompanyName = + case sqerl:adhoc_select([<<"email">>], <<"users">>, all) of {ok, Ids1} -> Ids = [Id || [{_, Id}] <- Ids1], - Fun = - fun(Email) -> + Fun = + fun(Email) -> case re:run(Email, "^[^@]*@\([^.]*\)\..*$") of {match, [_, {Pos, Len} | _]} -> {true, binary:part(Email, Pos, Len)}; @@ -206,12 +208,13 @@ get_company_name(State) -> end end, CompanyNames = lists:filtermap(Fun, Ids), - if length(CompanyNames) == 0 -> - throw("no valid Email Ids."); - true -> - get_most_occuring(CompanyNames) + case length(CompanyNames) == 0 of + true -> + throw("no valid Email Ids."); + _ -> + get_most_occuring(CompanyNames) end; - Error -> + Error -> throw(Error) end, CurrentScan = State#state.current_scan, @@ -226,12 +229,12 @@ get_most_occuring(List) -> end, Map1 = lists:foldl(Fun, #{}, List), - Fun1 = + Fun1 = fun(Key1, Count1, {Key2, Count2}) -> - if - Count1 > Count2 -> - {Key1, Count1}; + case Count1 > Count2 of true -> + {Key1, Count1}; + _ -> {Key2, Count2} end end, @@ -244,12 +247,12 @@ get_nodes(#state{req_id = ReqId, db_context = DbContext} = State) -> ScanEndTime = CurrentScan#current_scan.scan_end_time, QueryString = lists:flatten(io_lib:format("ohai_time:{~p TO ~p}", [ScanStartTime, ScanEndTime])), Query1 = chef_index:query_from_params("node", QueryString, undefined, undefined), - Count = + Count = case chef_db:count_nodes(DbContext) of Count1 when is_integer(Count1) -> Count1; Error -> throw({db_error, Error}) end, - Orgs = + Orgs = case chef_db:list(#oc_chef_organization{}, DbContext) of Orgs1 when is_list(Orgs1) -> Orgs1; Error1 -> throw({db_error, Error1}) @@ -268,8 +271,6 @@ generate_request(ServerVersion, State) -> CurrentScan = State#state.current_scan, jiffy:encode({[ {<<"licenseId">>, <<"Infra-Server-license-Id">>}, - %%{<<"customerId">>, <<"">>}, - %%{<<"expiration">>, <<"2023-11-30T00:00:00Z">>}, {<<"customerName">>, to_binary(State#state.current_scan#current_scan.company_name)}, {<<"periods">>, [ {[ @@ -326,7 +327,7 @@ epoch_to_string(Epoch) -> calendar:system_time_to_rfc3339(Epoch, [{offset, "Z"}]). send_data(Req, State) -> - case catch ibrowse:send_req(State#state.reporting_url, [], post, Req, [], 5000) of + case catch ibrowse:send_req(State#state.reporting_url, [{"Content-Type", "application/json"}], post, Req, [], 5000) of {ok, _Status, _ResponseHeaders, _ResponseBody} -> ok; Error -> throw({failed_sending_request, Error}) end. @@ -334,28 +335,28 @@ send_data(Req, State) -> check_send(State) -> Node = erlang:atom_to_binary(node()), Now = calendar:system_time_to_universal_time(State#state.scan_time, second), - case sqerl:adhoc_select([timestamp1], telemetry, {property, equals, "last_send"}, []) of + case sqerl:adhoc_select([<<"event_timestamp">>], <<"telemetry">>, {<<"property">>, equals, <<"last_send">>}, []) of {ok, Rows} when is_list(Rows) -> LastSend = to_system_time(Rows), case should_send(LastSend, State) of true -> - sqerl:adhoc_delete(telemetry, {<<"property">>, equals, <<"last_send">>}), - sqerl:adhoc_insert(telemetry, [[{<<"property">>, <<"last_send">>}, {<<"valuestring">>, Node}, {<<"timestamp1">>, Now}]]), + sqerl:adhoc_delete(<<"telemetry">>, {<<"property">>, equals, <<"last_send">>}), + sqerl:adhoc_insert(<<"telemetry">>, [[{<<"property">>, <<"last_send">>}, {<<"value_string">>, Node}, {<<"event_timestamp">>, Now}]]), true; false -> false end; {ok, Rows} when is_list(Rows) andalso length(Rows) == 0 -> - sqerl:adhoc_insert(telemetry, [[{<<"property">>, <<"last_send">>}, {<<"valuestring">>, Node}, {<<"timestamp1">>, Now}]]), + sqerl:adhoc_insert(<<"telemetry">>, [[{<<"property">>, <<"last_send">>}, {<<"value_string">>, Node}, {<<"event_timestamp">>, Now}]]), true; - Error -> - throw({not_able_to_gead_from_db, Error}) + Error -> + throw({not_able_to_read_from_db, Error}) end. to_system_time(Rows) -> - TimeStamps1 = [ proplists:get_value(<<"timestamp1">>, Row) || Row <- Rows, not (proplists:get_value(<<"timestamp1">>, Row) == undefined) ], + TimeStamps1 = [ proplists:get_value(<<"event_timestamp">>, Row) || Row <- Rows, not (proplists:get_value(<<"event_timestamp">>, Row) == undefined) ], SystemTimes = [calendar:datetime_to_gregorian_seconds({Date, {H, M, floor(S1)}}) - 62167219200 || {Date, {H, M, S1}} <- TimeStamps1], - MaxFun = + MaxFun = fun(Time, Max) -> case Time > Max of true -> @@ -371,8 +372,9 @@ should_send(LastSend, State) -> insert_fqdn(State) -> {ok, HostName} = inet:gethostname(), + HostName1 = string:trim(HostName), Now = calendar:system_time_to_universal_time(State#state.scan_time, second), %%Hostname = os:cmd('hostname -f'), - HostName1 = "FQDN:" ++ HostName, - sqerl:adhoc_delete(telemetry, {property, equals, HostName1}), - sqerl:adhoc_insert(telemetry, [[{<<"property">>, to_binary(HostName1)}, {<<"timestamp1">>, Now}]]). + HostName2 = to_binary("FQDN:" ++ HostName1), + sqerl:adhoc_delete(<<"telemetry">>, {<<"property">>, equals, HostName2}), + sqerl:adhoc_insert(<<"telemetry">>, [[{<<"property">>, HostName2}, {<<"event_timestamp">>, Now}, {<<"value_string">>, <<"">>}]]). diff --git a/src/oc_erchef/schema/deploy/telemetry.sql b/src/oc_erchef/schema/deploy/telemetry.sql new file mode 100644 index 0000000000..1bb3a48819 --- /dev/null +++ b/src/oc_erchef/schema/deploy/telemetry.sql @@ -0,0 +1,11 @@ +-- Deploy telemetry + +BEGIN; + +CREATE TABLE IF NOT EXISTS telemetry( + property CHAR(32), + value_string CHAR(32) NOT NULL, + event_timestamp TIMESTAMP +); + +COMMIT; \ No newline at end of file diff --git a/src/oc_erchef/schema/revert/telemetry.sql b/src/oc_erchef/schema/revert/telemetry.sql new file mode 100644 index 0000000000..ecacb73bd5 --- /dev/null +++ b/src/oc_erchef/schema/revert/telemetry.sql @@ -0,0 +1,7 @@ +-- Revert telemetry + +BEGIN; + +DROP TABLE IF EXISTS telemetry; + +COMMIT; \ No newline at end of file diff --git a/src/oc_erchef/schema/sqitch.plan b/src/oc_erchef/schema/sqitch.plan index 46f39faf15..9bb8670e05 100644 --- a/src/oc_erchef/schema/sqitch.plan +++ b/src/oc_erchef/schema/sqitch.plan @@ -84,3 +84,5 @@ users_email_functional_index [users] 2017-05-30T12:10:32Z Stephan Renatus # Adding functions to create and update users by inserting a sentinel value in the public_key_columns keys_update_trigger [keys_update_trigger@users_email_functional_index] 2017-08-23T23:12:29Z Prajakta Purohit # The insert and update triggers ignore rows with sentinel value for public_key. @sentinel_public_key_for_users 2017-08-24T14:32:04Z Prajakta Purohit # public_key only updated in the keys table + +telemetry 2024-08-01T08:37:30Z Sreepuram Sudheer # Adding telemetry to chef-server. diff --git a/src/oc_erchef/schema/verify/telemetry.sql b/src/oc_erchef/schema/verify/telemetry.sql new file mode 100644 index 0000000000..9f580de290 --- /dev/null +++ b/src/oc_erchef/schema/verify/telemetry.sql @@ -0,0 +1,7 @@ +-- Verify enterprise_chef:telemetry on pg + +BEGIN; + +-- XXX Add verifications here. + +ROLLBACK; From da902a41926b5dfc12e8cf993c3da6d1da44d470 Mon Sep 17 00:00:00 2001 From: sreepuramsudheer Date: Wed, 7 Aug 2024 19:38:54 +0530 Subject: [PATCH 6/8] Verify pipeline fix. Signed-off-by: sreepuramsudheer --- .../apps/chef_telemetry/src/chef_telemetry_worker.erl | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl index 36f6da3e44..3ffb8f7863 100644 --- a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl @@ -246,7 +246,7 @@ get_nodes(#state{req_id = ReqId, db_context = DbContext} = State) -> ScanStartTime = CurrentScan#current_scan.scan_start_time, ScanEndTime = CurrentScan#current_scan.scan_end_time, QueryString = lists:flatten(io_lib:format("ohai_time:{~p TO ~p}", [ScanStartTime, ScanEndTime])), - Query1 = chef_index:query_from_params("node", QueryString, undefined, undefined), + Query1 = chef_index:query_from_params("node", QueryString, "0", "100000"), Count = case chef_db:count_nodes(DbContext) of Count1 when is_integer(Count1) -> Count1; @@ -317,9 +317,6 @@ generate_request(ServerVersion, State) -> to_binary(String) when is_list(String) -> list_to_binary(String); -to_binary(Bin) when is_binary(Bin) -> - Bin; - to_binary(Element) -> throw({not_a_binary_or_string, Element}). @@ -336,7 +333,7 @@ check_send(State) -> Node = erlang:atom_to_binary(node()), Now = calendar:system_time_to_universal_time(State#state.scan_time, second), case sqerl:adhoc_select([<<"event_timestamp">>], <<"telemetry">>, {<<"property">>, equals, <<"last_send">>}, []) of - {ok, Rows} when is_list(Rows) -> + {ok, Rows} when is_list(Rows) andalso length(Rows) > 0 -> LastSend = to_system_time(Rows), case should_send(LastSend, State) of true -> @@ -346,7 +343,7 @@ check_send(State) -> false -> false end; - {ok, Rows} when is_list(Rows) andalso length(Rows) == 0 -> + {ok, Rows} when is_list(Rows) -> sqerl:adhoc_insert(<<"telemetry">>, [[{<<"property">>, <<"last_send">>}, {<<"value_string">>, Node}, {<<"event_timestamp">>, Now}]]), true; Error -> From c0f19b70dbfa029d20edbf7a2288795b621b0342 Mon Sep 17 00:00:00 2001 From: sreepuramsudheer Date: Thu, 8 Aug 2024 02:10:52 +0530 Subject: [PATCH 7/8] verify fix. Signed-off-by: sreepuramsudheer --- .../chef_telemetry/src/chef_telemetry.app.src | 2 + .../src/chef_telemetry_worker.erl | 14 +- .../test/chef_telemetry_test_utils.erl | 12 ++ .../test/chef_telemetry_worker_test.erl | 154 ++++++++++++++++++ .../apps/chef_telemetry/test/payload.erl | 11 ++ .../data_collector/src/data_collector.app.src | 2 + 6 files changed, 189 insertions(+), 6 deletions(-) create mode 100644 src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_test_utils.erl create mode 100644 src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_worker_test.erl create mode 100644 src/oc_erchef/apps/chef_telemetry/test/payload.erl diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src index 54d495ed2e..faf85f4774 100644 --- a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src @@ -24,6 +24,8 @@ {vsn, {cmd,"cat ../../VERSION | awk '{print $0}'"}}, {registered, []}, {applications, [ + kernel, + stdlib, lager, chef_secrets, opscoderl_httpc diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl index 3ffb8f7863..69a765d073 100644 --- a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl @@ -98,7 +98,7 @@ handle_cast(send_data, State) -> State2 = get_nodes(State1), State3 = get_company_name(State2), State4 = get_api_fqdn(State3), - Req = generate_request(list_to_binary(ServerVersion), State4), + Req = generate_request(ServerVersion, State4), send_data(Req, State3), State3; _ -> @@ -269,9 +269,9 @@ get_nodes(#state{req_id = ReqId, db_context = DbContext} = State) -> generate_request(ServerVersion, State) -> CurrentScan = State#state.current_scan, - jiffy:encode({[ + Res = jiffy:encode({[ {<<"licenseId">>, <<"Infra-Server-license-Id">>}, - {<<"customerName">>, to_binary(State#state.current_scan#current_scan.company_name)}, + {<<"customerName">>, State#state.current_scan#current_scan.company_name}, {<<"periods">>, [ {[ {<<"version">>, to_binary(ServerVersion)}, @@ -312,7 +312,8 @@ generate_request(ServerVersion, State) -> {<<"source">>, <<"Infra Server">>}, {<<"scannerVersion">>, <<"0.1.0">>}, {<<"scannedOn">>, to_binary(epoch_to_string(State#state.scan_time))} - ]}). + ]}), + Res. to_binary(String) when is_list(String) -> list_to_binary(String); @@ -332,7 +333,7 @@ send_data(Req, State) -> check_send(State) -> Node = erlang:atom_to_binary(node()), Now = calendar:system_time_to_universal_time(State#state.scan_time, second), - case sqerl:adhoc_select([<<"event_timestamp">>], <<"telemetry">>, {<<"property">>, equals, <<"last_send">>}, []) of + case sqerl:adhoc_select([<<"event_timestamp">>], <<"telemetry">>, {<<"property">>, equals, <<"last_send">>}) of {ok, Rows} when is_list(Rows) andalso length(Rows) > 0 -> LastSend = to_system_time(Rows), case should_send(LastSend, State) of @@ -365,7 +366,8 @@ to_system_time(Rows) -> lists:foldl( MaxFun, 0, SystemTimes). should_send(LastSend, State) -> - LastSend < State#state.scan_time. + LastSend < calendar:datetime_to_gregorian_seconds( + calendar:system_time_to_universal_time(State#state.scan_time, seconds)) - 62167219200. insert_fqdn(State) -> {ok, HostName} = inet:gethostname(), diff --git a/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_test_utils.erl b/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_test_utils.erl new file mode 100644 index 0000000000..8459c8f3a8 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_test_utils.erl @@ -0,0 +1,12 @@ +-module(chef_telemetry_test_utils). + +-export([ + start_server/1 + ]). + +start_server(_Inputs) -> + application:start(inets), + {_Httpd_State, _Httpd_Pid} = + inets:start(httpd, [{port, 9001}, + {server_name, "localhost"}, {document_root, "/tmp"}, + {modules,[mod_esi]},{server_root, "/tmp"}, {erl_script_alias, {"/esi", [payload]}}]). diff --git a/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_worker_test.erl b/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_worker_test.erl new file mode 100644 index 0000000000..911b8ec4f5 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_worker_test.erl @@ -0,0 +1,154 @@ +-module(chef_telemetry_worker_test). + +-include_lib("eunit/include/eunit.hrl"). + +-export([ + get_execute/1, + adhoc_select/3, + count_nodes/1, + chef_db_list/2, + org_metadata/2, + index_search/1 +]). +%%-record(oc_chef_organization, { +%% server_api_version, +%% id, +%% authz_id, +%% name, +%% full_name, +%% assigned_at, +%% last_updated_by, +%% created_at, +%% updated_at +%% }). + +-define(DEFAULT_CONFIG, []). + +-record(state, { + fqdn_select, + last_send_timestamp, + user_emails = [], + nodes_count = 0, + organizations = ["org1", "org2"], + index_search = [["node1_org1", "node2_org1"], ["node1_org2", "node2_org2"]] +}). + +-record(expected, { + company_name = <<"">>, + nodes_count = 0, + active_nodes = 0, + fqdn = []}). + +feild_value_test() -> + State = #state{fqdn_select = {ok, []}, + last_send_timestamp = {{2024, 8, 7}, {0, 0, 1}}, + user_emails = [[{<<"email">>, <<"test@testorg.com">>}]], + nodes_count = 10 + }, + Expected = #expected{company_name = <<"testorg">>, + nodes_count = 10, + active_nodes = 4}, + execute(State, Expected, []). + +enable_flag_test() -> + State = #state{fqdn_select = {ok, []}, + last_send_timestamp = {{2024, 8, 7}, {0, 0, 1}}, + user_emails = [[{<<"email">>, <<"test@testorg.com">>}]], + nodes_count = 10 + }, + Expected = #expected{company_name = <<"testorg">>, + nodes_count = 10, + active_nodes = 4}, + ?_assertException(error, no_request1, execute(State, Expected, [{chef_telemetry, is_enabled, false}])). + +execute(State, Expected, Env) -> + set_env([{chef_telemetry, reporting_url, "http://127.0.0.1:9001/esi/payload:io"}] ++ Env), + application:start(ibrowse), + put(state, State), + setup(), + chef_telemetry_test_utils:start_server([]), + register(telemetry_mock_consumer, self()), + trigger_send_data(), + Req = get_message(), + Req1 = jiffy:decode(Req), + validate(Req1, Expected), + io:format(user, "json ~p", [jiffy:decode(Req)]). + +setup() -> + meck:new(sqerl, [passthrough]), + meck:new(chef_db, [passthrough]), + meck:new(chef_index, [passthrough]), + meck:new(release_handler, [passthrough]), + meck:new(stats_hero, [passthrough]), + meck:expect(sqerl, adhoc_insert, fun(_Table, _Rows) -> ok end), + meck:expect(sqerl, adhoc_delete, fun(_Table, _Clause) -> ok end), + meck:expect(sqerl, execute, fun get_execute/1 ), + meck:expect(sqerl, adhoc_select, fun adhoc_select/3 ), + meck:expect(chef_db, count_nodes, fun count_nodes/1 ), + meck:expect(chef_db, list, fun chef_db_list/2 ), + meck:expect(chef_db, fetch_org_metadata, fun org_metadata/2 ), + meck:expect(chef_index, search, fun index_search/1), + meck:expect(release_handler, which_releases, fun(_) -> [{"chef_server", "15.9.38", [], []}] end), + meck:expect(stats_hero, ctime, fun(_, _, Fun) -> Fun() end). + +get_execute(<<"select property from telemetry where property like 'FQDN:%'">>) -> + State = get(state), + State#state.fqdn_select; + +get_execute(_) -> + ok. + +adhoc_select([<<"email">>], <<"users">>, all) -> + State = get(state), + {ok, State#state.user_emails}; + +adhoc_select([<<"event_timestamp">>], <<"telemetry">>, {<<"property">>, equals, <<"last_send">>}) -> + State = get(state), + {ok, [[{<<"event_timestamp">>, State#state.last_send_timestamp}]]}. + +count_nodes(_Context) -> + State = get(state), + State#state.nodes_count. + +chef_db_list(Record, _context) -> + RecordName = element(1, Record), + State = get(state), + case RecordName of + oc_chef_organization -> State#state.organizations; + _ -> [] + end. + +org_metadata(_context, OrgName) -> + OrgName1 = list_to_binary(OrgName), + {OrgName1, OrgName1}. + +index_search(_) -> + State = get(state), + [Nodes | Rest] = State#state.index_search, + State1 = State#state{index_search = Rest}, + put(state, State1), + {ok, 0, length(Nodes), Nodes}. + +trigger_send_data() -> + {ok, State} = chef_telemetry_worker:init([]), + chef_telemetry_worker:handle_cast(send_data, State). + +get_message() -> + receive + {http_request, _From, Req} -> + Req + after 5000 -> + throw(no_request) + end. + +validate(Req, Expected) -> + Licence = ej:get({<<"licenseId">>}, Req), + TotalNodes = ej:get({<<"periods">>, 1, <<"summary">>, <<"nodes">>, <<"total">>}, Req), + ActiveNodes = ej:get({<<"periods">>, 1, <<"summary">>, <<"nodes">>, <<"active">>}, Req), + ?assertEqual(<<"Infra-Server-license-Id">>, Licence), + ?assertEqual(Expected#expected.nodes_count, TotalNodes), + ?assertEqual(Expected#expected.active_nodes, ActiveNodes). + +set_env(ConfigList) -> + ConfigList1 = ?DEFAULT_CONFIG ++ ConfigList, + [ application:set_env(App, Parameter, Value) || {App, Parameter, Value} <- ConfigList1 ]. diff --git a/src/oc_erchef/apps/chef_telemetry/test/payload.erl b/src/oc_erchef/apps/chef_telemetry/test/payload.erl new file mode 100644 index 0000000000..322f118b68 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/test/payload.erl @@ -0,0 +1,11 @@ +-module(payload). + +-export([ + io/3]). + +io(SessionID, Env, Input) -> + io:format(user, "pid ~p", [whereis(telemetry_mock_consumer)]), + telemetry_mock_consumer ! {http_request, self(), Input}, + io:format(user, "input parameters ~p, ~p, ~p", [SessionID, Env, Input]), + mod_esi:deliver(SessionID, "status:201 Created\r\nContent-Type:text/html\r\n\r\n"), + ok. \ No newline at end of file diff --git a/src/oc_erchef/apps/data_collector/src/data_collector.app.src b/src/oc_erchef/apps/data_collector/src/data_collector.app.src index 97e61ab169..218fea9d90 100644 --- a/src/oc_erchef/apps/data_collector/src/data_collector.app.src +++ b/src/oc_erchef/apps/data_collector/src/data_collector.app.src @@ -26,6 +26,8 @@ {vsn, {cmd,"cat ../../VERSION | awk '{print $0}'"}}, {registered, []}, {applications, [ + kernel, + stdlib, lager, chef_secrets, opscoderl_httpc From fcc837e0288872b41fa9b1f28bb12ce8909aacd2 Mon Sep 17 00:00:00 2001 From: sreepuramsudheer Date: Mon, 19 Aug 2024 19:30:46 +0530 Subject: [PATCH 8/8] Added code for FQDN masking. Signed-off-by: sreepuramsudheer --- .../src/chef_telemetry_worker.erl | 48 ++++++++++++++++--- .../test/chef_telemetry_worker_test.erl | 25 ++++++++-- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl index 69a765d073..9d820d661d 100644 --- a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl @@ -56,8 +56,8 @@ %% 1) I don't have the server URL. %% 2) easy for testing. %% should be changed to actual server URL ASAP. --define(DEFAULT_REPORTING_URL, "http://127.0.0.1:9001"). --define(DEFAULT_REPORTING_TIME, {12, 00}). +-define(DEFAULT_REPORTING_URL, "https://services.chef.io/usage/v1/payload"). +-define(DEFAULT_REPORTING_TIME, {4, 00}). -define(DEFAULT_IBROWSE_OPTIONS, []). -define(WINDOW_SECONDS, 300). @@ -66,7 +66,7 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). init(_Config) -> - ReportingUrl = envy:get(chef_telemetry, reporting_url, ?DEFAULT_REPORTING_URL, string), + ReportingUrl = envy:get(chef_telemetry, reporting_time, ?DEFAULT_REPORTING_TIME, Fun), Fun = fun({Hour, Min}) -> Hour >= 0 andalso Hour < 24 andalso Min >= 0 andalso Min < 60 end, @@ -88,7 +88,7 @@ handle_call(_Message, _From, State) -> handle_cast(send_data, State) -> State6 = - case chef_telemetry:is_enabled() of + try chef_telemetry:is_enabled() of true -> State1 = init_req(State), insert_fqdn(State1), @@ -106,6 +106,9 @@ handle_cast(send_data, State) -> end; _ -> State + catch + _:_ -> + State end, gen_server:cast(self(), init_timer), {noreply, State6}; @@ -158,7 +161,8 @@ get_api_fqdn(State) -> case sqerl:execute(<<"select property from telemetry where property like 'FQDN:%'">>) of {ok, Rows} when is_list(Rows) -> FQDNs = [binary:part(FQDN, 5, size(FQDN) -5) || [{<<"property">>, FQDN}] <- Rows], - State#state{fqdns = FQDNs}; + FQDNs1 = mask(FQDNs), + State#state{fqdns = FQDNs1}; _ -> State end. @@ -319,7 +323,7 @@ to_binary(String) when is_list(String) -> list_to_binary(String); to_binary(Element) -> - throw({not_a_binary_or_string, Element}). + throw({not_a_string, Element}). epoch_to_string(Epoch) -> calendar:system_time_to_rfc3339(Epoch, [{offset, "Z"}]). @@ -377,3 +381,35 @@ insert_fqdn(State) -> HostName2 = to_binary("FQDN:" ++ HostName1), sqerl:adhoc_delete(<<"telemetry">>, {<<"property">>, equals, HostName2}), sqerl:adhoc_insert(<<"telemetry">>, [[{<<"property">>, HostName2}, {<<"event_timestamp">>, Now}, {<<"value_string">>, <<"">>}]]). + +mask(FQDNs) -> + Fun = fun(FQDN) -> + case re:run(FQDN, + <<"(?:(.*?):\/\/?)?\/?(?:[^\/\.]+\.)*?([^\/\.]+)\.?([^\/:]*)(?::([^?\/]*)?)?(.*)?">>, + [{capture, all_but_first, binary}]) of + {match, Parts} -> + [Protocall, SubDomain, Domain, Rest1, Rest2] = Parts, + Hash = crypto:hash(md5, SubDomain), + Hash1 = base64:encode(Hash), + Len = binary:longest_common_suffix([Hash1, <<"===">>]), + Hash2 = binary:part(Hash1, {0, size(Hash1) - Len}), + Res1 = + case Protocall /= <<"">> of + true -> + <>; + false -> + <> + end, + Res2 = + case Rest1 /= <<"">> of + true -> + <>; + _ -> + <> + end, + Res2; + _ -> + <<"">> + end + end, + lists:map(Fun, FQDNs). diff --git a/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_worker_test.erl b/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_worker_test.erl index 911b8ec4f5..07ba07d127 100644 --- a/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_worker_test.erl +++ b/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_worker_test.erl @@ -40,14 +40,19 @@ fqdn = []}). feild_value_test() -> - State = #state{fqdn_select = {ok, []}, + State = #state{fqdn_select = {ok, [{<<"property">>, <<"FQDN:node1.domain1.com">>}, + {<<"property">>, <<"FQDN:node2.subdomain2.domain2.com">>}, + {<<"property">>, <<"FQDN:node3.subdomain3.domain3.co.uk">>}]}, last_send_timestamp = {{2024, 8, 7}, {0, 0, 1}}, user_emails = [[{<<"email">>, <<"test@testorg.com">>}]], nodes_count = 10 }, Expected = #expected{company_name = <<"testorg">>, nodes_count = 10, - active_nodes = 4}, + active_nodes = 4, + fqdn = [<<".*\.domain1.com$">>, + <<".*\.subdomain2\.domain2\.com$">>, + <<".*\.subdomain3\.domain3\.co\.uk$">>]}, execute(State, Expected, []). enable_flag_test() -> @@ -145,10 +150,22 @@ validate(Req, Expected) -> Licence = ej:get({<<"licenseId">>}, Req), TotalNodes = ej:get({<<"periods">>, 1, <<"summary">>, <<"nodes">>, <<"total">>}, Req), ActiveNodes = ej:get({<<"periods">>, 1, <<"summary">>, <<"nodes">>, <<"active">>}, Req), - ?assertEqual(<<"Infra-Server-license-Id">>, Licence), + FQDNs = ej:get({<<"metadata">>, <<"Infra Server">>, <<"fqdn">>}, Req), + ?assertEqual(<<"Infra-Server-license-Id">>, Licence), ?assertEqual(Expected#expected.nodes_count, TotalNodes), - ?assertEqual(Expected#expected.active_nodes, ActiveNodes). + ?assertEqual(Expected#expected.active_nodes, ActiveNodes), + ?assertEqual(true, check_fqdn(FQDNs, Expected#expected.fqdn)). set_env(ConfigList) -> ConfigList1 = ?DEFAULT_CONFIG ++ ConfigList, [ application:set_env(App, Parameter, Value) || {App, Parameter, Value} <- ConfigList1 ]. + +check_fqdn(ReqFQDNs, Expected) -> + MatchFun = + fun(FQDN) -> + lists:any( + fun(Pattern) -> + match == re:run(FQDN, Pattern, [{capture, none}]) + end, Expected) + end, + lists:all(MatchFun, ReqFQDNs). \ No newline at end of file