diff --git a/omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb b/omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb index 3f76386963..77f85dcdfe 100755 --- a/omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb +++ b/omnibus/files/server-ctl-cookbooks/infra-server/attributes/default.rb @@ -893,6 +893,8 @@ # Select whether data_collector affects overall status in _status endpoint default['private_chef']['data_collector']['health_check'] = true +default['private_chef']['ctl_command'] = "#{ChefUtils::Dist::Server::SERVER_CTL}" +default['private_chef']['running_filepath'] = "/etc/#{ChefUtils::Dist::Org::LEGACY_CONF_DIR}/#{ChefUtils::Dist::Server::SERVER}-running.json" ## # Compliance Profiles ## diff --git a/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb b/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb index ee0bfc2656..c020307c0f 100755 --- a/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb +++ b/omnibus/files/server-ctl-cookbooks/infra-server/templates/default/oc_erchef.config.erb @@ -320,7 +320,13 @@ }, {metrics_module, folsom_metrics} -]} + ]}, + + {chef_telemetry, [ + {running_filepath, "<%= node['private_chef']['running_filepath'] %>"}, + {ctl_command, "<%= node['private_chef']['ctl_command'] %>"} + ] + } <% if !node['private_chef']['opscode-erchef']['ssl_session_caching']['enabled'] -%> , diff --git a/src/oc_erchef/apps/chef_telemetry/.gitignore b/src/oc_erchef/apps/chef_telemetry/.gitignore new file mode 100644 index 0000000000..6a325fcd24 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/.gitignore @@ -0,0 +1,15 @@ +.eunit +deps/ +ebin/ +ebin_dialyzer/ +TAGS +.DS_Store +doc/*.html +*.beam +/doc/edoc-info +/doc/erlang.png +/doc/stylesheet.css +/deps.plt +test/*.out +.rebar +log/ diff --git a/src/oc_erchef/apps/chef_telemetry/Makefile b/src/oc_erchef/apps/chef_telemetry/Makefile new file mode 100644 index 0000000000..410366ea1c --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/Makefile @@ -0,0 +1,30 @@ +REBAR3_URL=https://s3.amazonaws.com/rebar3/rebar3 + +# If there is a rebar in the current directory, use it +ifeq ($(wildcard rebar3),rebar3) +REBAR3 = $(CURDIR)/rebar3 +endif + +# Fallback to rebar on PATH +REBAR3 ?= $(shell which rebar3) + +# And finally, prep to download rebar if all else fails +ifeq ($(REBAR3),) +REBAR3 = rebar3 +endif + +all: $(REBAR3) + @$(REBAR3) do clean, compile, eunit, dialyzer + +rel: all + @$(REBAR3) release + +distclean: + @rm -rf _build + +$(REBAR3): + curl -Lo rebar3 $(REBAR3_URL) || wget $(REBAR3_URL) + chmod a+x rebar3 + +install: $(REBAR3) distclean + $(REBAR3) update diff --git a/src/oc_erchef/apps/chef_telemetry/README.md b/src/oc_erchef/apps/chef_telemetry/README.md new file mode 100644 index 0000000000..a08073773c --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/README.md @@ -0,0 +1,3 @@ +# Telemetry + +Telemetry is an HTTP exporter of Chef Server node stats for external services. diff --git a/src/oc_erchef/apps/chef_telemetry/priv/.gitkeep b/src/oc_erchef/apps/chef_telemetry/priv/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/oc_erchef/apps/chef_telemetry/rebar.config b/src/oc_erchef/apps/chef_telemetry/rebar.config new file mode 100644 index 0000000000..7cbf7d6114 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/rebar.config @@ -0,0 +1,26 @@ +%% -*- mode: erlang -*- +%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 ft=erlang et + +{deps, + [ + %% lager has to come first since we use its parse transform + {lager, ".*", + {git, "https://github.com/erlang-lager/lager", {branch, "master"}}} + ] +}. + +{profiles, [{ + test, [ + {deps, [meck]}, + {erl_opts, [export_all]} + ] +}]}. + +{erl_opts, [ + warnings_as_errors, + {parse_transform, lager_transform}, + debug_info +]}. + +{cover_enabled, true}. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src new file mode 100644 index 0000000000..faf85f4774 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.app.src @@ -0,0 +1,34 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- +%% ex: ts=4 sw=4 et +%% +%% +%% Copyright 2016 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +{application, chef_telemetry, [ + {description, "Chef Server Telemetry"}, + {vsn, {cmd,"cat ../../VERSION | awk '{print $0}'"}}, + {registered, []}, + {applications, [ + kernel, + stdlib, + lager, + chef_secrets, + opscoderl_httpc + ]}, + {mod, {chef_telemetry_app, []}} +]}. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl new file mode 100644 index 0000000000..dd9a6bbb2e --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry.erl @@ -0,0 +1,34 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- +%% ex: ts=4 sw=4 et +%% +%% +%% Copyright 2016 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(chef_telemetry). +-export([ + is_enabled/0 + ]). + +-spec is_enabled() -> boolean(). +is_enabled() -> + case envy:get(chef_telemetry, is_enabled, true, boolean) of + true -> + true; + _ -> + false + end. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_app.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_app.erl new file mode 100644 index 0000000000..cacbb18a06 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_app.erl @@ -0,0 +1,34 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- +%% ex: ts=4 sw=4 et +%% +%% Copyright 2016 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(chef_telemetry_app). + +-behaviour(application). + +%% API +-export([start/2, + stop/1 + ]). + +start(_StartType, _StartArgs) -> + chef_telemetry_sup:start_link(). + +stop(_State) -> + ok. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_sup.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_sup.erl new file mode 100644 index 0000000000..94b1c02463 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_sup.erl @@ -0,0 +1,37 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil; fill-column: 92 -*- +%% ex: ts=4 sw=4 et +%% +%% +%% Copyright 2016 Chef Software, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% + +-module(chef_telemetry_sup). + +-behaviour(supervisor). + +-export([init/1, + start_link/0 + ]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + Worker = {chef_telemetry_worker, + {chef_telemetry_worker, start_link, []}, + permanent, 5000, supervisor, [chef_telemetry_worker]}, + {ok, {{one_for_one, 10, 10}, [Worker]}}. diff --git a/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl new file mode 100644 index 0000000000..9d820d661d --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/src/chef_telemetry_worker.erl @@ -0,0 +1,415 @@ +-module(chef_telemetry_worker). + +-behaviour(gen_server). + +-include("../../../include/chef_types.hrl"). + +-export([ + start_link/0 +]). + +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + code_change/2, + terminate/2 +]). + +-record(current_scan, { + scan_start_time, + scan_end_time, + total_nodes, + active_nodes, + company_name +}). + +-record(state, { + timer_ref, + report_time, + reporting_url, + db_context, + req_id, + scan_time, + current_scan, + running_file, + ctl_command, + fqdns +}). + +-record(oc_chef_organization, { + server_api_version, + id, + authz_id, + name, + full_name, + assigned_at, + last_updated_by, + created_at, + updated_at + }). + +-define(DEFAULT_DAYS, 30). + +%% Setting this value for local ip because +%% 1) I don't have the server URL. +%% 2) easy for testing. +%% should be changed to actual server URL ASAP. +-define(DEFAULT_REPORTING_URL, "https://services.chef.io/usage/v1/payload"). +-define(DEFAULT_REPORTING_TIME, {4, 00}). +-define(DEFAULT_IBROWSE_OPTIONS, []). + +-define(WINDOW_SECONDS, 300). + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +init(_Config) -> + ReportingUrl = envy:get(chef_telemetry, reporting_time, ?DEFAULT_REPORTING_TIME, Fun), + Fun = fun({Hour, Min}) -> + Hour >= 0 andalso Hour < 24 andalso Min >= 0 andalso Min < 60 + end, + ReportingTime = envy:get(chef_telemetry, reporting_time, ?DEFAULT_REPORTING_TIME, Fun), + ConfigFile= envy:get(chef_telemetry, running_filepath, "", string), + Ctl = envy:get(chef_telemetry, ctl_command, "", string), + Cmd = "which " ++ Ctl, + CtlLocation = string:trim(os:cmd(Cmd)), + State = #state{ + report_time = ReportingTime, + reporting_url = ReportingUrl, + running_file = ConfigFile, + ctl_command = CtlLocation}, + gen_server:cast(self(), init_timer), + {ok, State}. + +handle_call(_Message, _From, State) -> + {noreply, State}. + +handle_cast(send_data, State) -> + State6 = + try chef_telemetry:is_enabled() of + true -> + State1 = init_req(State), + insert_fqdn(State1), + case check_send(State1) of + true -> + [{_Server, ServerVersion, _, _}] = release_handler:which_releases(permanent), + State2 = get_nodes(State1), + State3 = get_company_name(State2), + State4 = get_api_fqdn(State3), + Req = generate_request(ServerVersion, State4), + send_data(Req, State3), + State3; + _ -> + State1 + end; + _ -> + State + catch + _:_ -> + State + end, + gen_server:cast(self(), init_timer), + {noreply, State6}; + +handle_cast(init_timer, State) -> + {_Date, {Hour, Min, _Sec}} = calendar:now_to_universal_time(erlang:timestamp()), + {RHour, RMin} = State#state.report_time, + CurrentDaySeconds = Hour * 3600 + Min * 60, + ReportingSeconds = RHour * 3600 + RMin * 60, + DelaySeconds = floor(rand:uniform() * ?WINDOW_SECONDS), + case ReportingSeconds - CurrentDaySeconds of + Diff when Diff == 0 -> + timer:apply_after(DelaySeconds * 1000, get_server, cast, [self(), send_data]); + Diff when Diff > 0 -> + timer:apply_after((Diff + DelaySeconds) * 1000, gen_server, cast, [self(), send_data]); + Diff -> + timer:apply_after((Diff + DelaySeconds + 86400) * 1000, gen_server, cast, [self(), send_data]) + end, + {noreply, State}; + +handle_cast(_Message, State) -> + {noreply, State}. + +handle_info(_Message, State) -> + {noreply, State}. + +code_change(_OldVsn, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +init_req(State) -> + ReqId = base64:encode(term_to_binary(make_ref())), + DbContext = chef_db:make_context("1.0", ReqId, false), + CurrentTime = erlang:system_time(seconds), + StartTime = CurrentTime - (?DEFAULT_DAYS * 86400), + State#state{ + req_id = ReqId, + db_context = DbContext, + scan_time = CurrentTime, + current_scan = + #current_scan{ + scan_start_time = StartTime, + scan_end_time = CurrentTime} + }. + +get_api_fqdn(State) -> + sqerl:execute(<<"delete from telemetry where property like 'FQDN:%' and event_timestamp < (current_timestamp - interval '86700')">>), + case sqerl:execute(<<"select property from telemetry where property like 'FQDN:%'">>) of + {ok, Rows} when is_list(Rows) -> + FQDNs = [binary:part(FQDN, 5, size(FQDN) -5) || [{<<"property">>, FQDN}] <- Rows], + FQDNs1 = mask(FQDNs), + State#state{fqdns = FQDNs1}; + _ -> + State + end. + +get_org_nodes(OrgName, Query1, ReqId, DbContext) -> + {Guid1, _AuthzId1} = + case chef_db:fetch_org_metadata(DbContext, OrgName) of + not_found -> throw({org_not_found, OrgName}); + {Guid, AuthzId} -> {Guid, AuthzId} + end, + Query = chef_index:add_org_guid_to_query(Query1, Guid1), + case search(Query, ReqId) of + {ok, _Start0, _SolrNumFound, Ids} -> + erlang:length(Ids); + {error, {solr_400, _}=Why} -> + throw({error_getting_nodes, Why}); + {error, {solr_500, _}=Why} -> + throw({error_getting_nodes, Why}) + end. + +search(Query, ReqId) -> + stats_hero:ctime(ReqId, {chef_solr, search}, + fun() -> + solr_search(Query) + end). + +solr_search(Query) -> + try + chef_index:search(Query) + catch + Error:Reason -> + {Error, Reason} + end. + +get_company_name(State) -> + CompanyName = + case sqerl:adhoc_select([<<"email">>], <<"users">>, all) of + {ok, Ids1} -> + Ids = [Id || [{_, Id}] <- Ids1], + Fun = + fun(Email) -> + case re:run(Email, "^[^@]*@\([^.]*\)\..*$") of + {match, [_, {Pos, Len} | _]} -> + {true, binary:part(Email, Pos, Len)}; + _ -> + false + end + end, + CompanyNames = lists:filtermap(Fun, Ids), + case length(CompanyNames) == 0 of + true -> + throw("no valid Email Ids."); + _ -> + get_most_occuring(CompanyNames) + end; + Error -> + throw(Error) + end, + CurrentScan = State#state.current_scan, + State#state{ + current_scan = CurrentScan#current_scan{company_name = CompanyName}}. + +get_most_occuring(List) -> + FirstElement = lists:nth(1, List), + Fun = fun(Element, Map) -> + Count = maps:get(Element, Map, 0), + maps:put(Element, Count + 1, Map) + end, + Map1 = lists:foldl(Fun, #{}, List), + + Fun1 = + fun(Key1, Count1, {Key2, Count2}) -> + case Count1 > Count2 of + true -> + {Key1, Count1}; + _ -> + {Key2, Count2} + end + end, + Res1 = maps:fold(Fun1, {FirstElement, 0}, Map1), + element(1, Res1). + +get_nodes(#state{req_id = ReqId, db_context = DbContext} = State) -> + CurrentScan = State#state.current_scan, + ScanStartTime = CurrentScan#current_scan.scan_start_time, + ScanEndTime = CurrentScan#current_scan.scan_end_time, + QueryString = lists:flatten(io_lib:format("ohai_time:{~p TO ~p}", [ScanStartTime, ScanEndTime])), + Query1 = chef_index:query_from_params("node", QueryString, "0", "100000"), + Count = + case chef_db:count_nodes(DbContext) of + Count1 when is_integer(Count1) -> Count1; + Error -> throw({db_error, Error}) + end, + Orgs = + case chef_db:list(#oc_chef_organization{}, DbContext) of + Orgs1 when is_list(Orgs1) -> Orgs1; + Error1 -> throw({db_error, Error1}) + end, + Stats = [ {Org, get_org_nodes(Org, Query1, ReqId, DbContext)} || Org <- Orgs ], + Fun = fun({_Org, Nodes}, Sum) -> + Sum + Nodes + end, + ActiveNodes = lists:foldl(Fun, 0, Stats), + State#state{ + current_scan = CurrentScan#current_scan{ + total_nodes = Count, + active_nodes = ActiveNodes}}. + +generate_request(ServerVersion, State) -> + CurrentScan = State#state.current_scan, + Res = jiffy:encode({[ + {<<"licenseId">>, <<"Infra-Server-license-Id">>}, + {<<"customerName">>, State#state.current_scan#current_scan.company_name}, + {<<"periods">>, [ + {[ + {<<"version">>, to_binary(ServerVersion)}, + {<<"date">>, to_binary(epoch_to_string(CurrentScan#current_scan.scan_end_time))}, + {<<"period">>, {[ + {<<"start">>, to_binary(epoch_to_string(CurrentScan#current_scan.scan_start_time))}, + {<<"end">>, to_binary(epoch_to_string(CurrentScan#current_scan.scan_end_time))} + ]}}, + {<<"summary">>, {[ + {<<"nodes">>, {[ + {<<"total">>, CurrentScan#current_scan.total_nodes}, + {<<"active">>, CurrentScan#current_scan.active_nodes} + ]}}, + {<<"scans">>, {[ + {<<"total">>, 0}, + {<<"targets">>, 0} + ]}}, + {<<"services">>, {[ + {<<"targets">>, 0} + ]}} + ]}}, + {<<"evidence">>, {[ + {<<"nodes">>, null}, + {<<"scans">>, null}, + {<<"content">>, null} + ]}} + ]} + ]}, + {<<"metadata">>, {[ + {<<"Infra Server">>, {[ + {<<"deploymentType">>, <<"">>}, + {<<"instanceId">>, <<"">>}, + {<<"fqdn">>, State#state.fqdns}, + {<<"config_location">>, to_binary(State#state.running_file)}, + {<<"binary_location">>, to_binary(State#state.ctl_command)} + ]}} + ]}}, + {<<"source">>, <<"Infra Server">>}, + {<<"scannerVersion">>, <<"0.1.0">>}, + {<<"scannedOn">>, to_binary(epoch_to_string(State#state.scan_time))} + ]}), + Res. + +to_binary(String) when is_list(String) -> + list_to_binary(String); + +to_binary(Element) -> + throw({not_a_string, Element}). + +epoch_to_string(Epoch) -> + calendar:system_time_to_rfc3339(Epoch, [{offset, "Z"}]). + +send_data(Req, State) -> + case catch ibrowse:send_req(State#state.reporting_url, [{"Content-Type", "application/json"}], post, Req, [], 5000) of + {ok, _Status, _ResponseHeaders, _ResponseBody} -> ok; + Error -> throw({failed_sending_request, Error}) + end. + +check_send(State) -> + Node = erlang:atom_to_binary(node()), + Now = calendar:system_time_to_universal_time(State#state.scan_time, second), + case sqerl:adhoc_select([<<"event_timestamp">>], <<"telemetry">>, {<<"property">>, equals, <<"last_send">>}) of + {ok, Rows} when is_list(Rows) andalso length(Rows) > 0 -> + LastSend = to_system_time(Rows), + case should_send(LastSend, State) of + true -> + sqerl:adhoc_delete(<<"telemetry">>, {<<"property">>, equals, <<"last_send">>}), + sqerl:adhoc_insert(<<"telemetry">>, [[{<<"property">>, <<"last_send">>}, {<<"value_string">>, Node}, {<<"event_timestamp">>, Now}]]), + true; + false -> + false + end; + {ok, Rows} when is_list(Rows) -> + sqerl:adhoc_insert(<<"telemetry">>, [[{<<"property">>, <<"last_send">>}, {<<"value_string">>, Node}, {<<"event_timestamp">>, Now}]]), + true; + Error -> + throw({not_able_to_read_from_db, Error}) + end. + +to_system_time(Rows) -> + TimeStamps1 = [ proplists:get_value(<<"event_timestamp">>, Row) || Row <- Rows, not (proplists:get_value(<<"event_timestamp">>, Row) == undefined) ], + SystemTimes = [calendar:datetime_to_gregorian_seconds({Date, {H, M, floor(S1)}}) - 62167219200 || {Date, {H, M, S1}} <- TimeStamps1], + MaxFun = + fun(Time, Max) -> + case Time > Max of + true -> + Time; + _ -> + Max + end + end, + lists:foldl( MaxFun, 0, SystemTimes). + +should_send(LastSend, State) -> + LastSend < calendar:datetime_to_gregorian_seconds( + calendar:system_time_to_universal_time(State#state.scan_time, seconds)) - 62167219200. + +insert_fqdn(State) -> + {ok, HostName} = inet:gethostname(), + HostName1 = string:trim(HostName), + Now = calendar:system_time_to_universal_time(State#state.scan_time, second), + %%Hostname = os:cmd('hostname -f'), + HostName2 = to_binary("FQDN:" ++ HostName1), + sqerl:adhoc_delete(<<"telemetry">>, {<<"property">>, equals, HostName2}), + sqerl:adhoc_insert(<<"telemetry">>, [[{<<"property">>, HostName2}, {<<"event_timestamp">>, Now}, {<<"value_string">>, <<"">>}]]). + +mask(FQDNs) -> + Fun = fun(FQDN) -> + case re:run(FQDN, + <<"(?:(.*?):\/\/?)?\/?(?:[^\/\.]+\.)*?([^\/\.]+)\.?([^\/:]*)(?::([^?\/]*)?)?(.*)?">>, + [{capture, all_but_first, binary}]) of + {match, Parts} -> + [Protocall, SubDomain, Domain, Rest1, Rest2] = Parts, + Hash = crypto:hash(md5, SubDomain), + Hash1 = base64:encode(Hash), + Len = binary:longest_common_suffix([Hash1, <<"===">>]), + Hash2 = binary:part(Hash1, {0, size(Hash1) - Len}), + Res1 = + case Protocall /= <<"">> of + true -> + <>; + false -> + <> + end, + Res2 = + case Rest1 /= <<"">> of + true -> + <>; + _ -> + <> + end, + Res2; + _ -> + <<"">> + end + end, + lists:map(Fun, FQDNs). diff --git a/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_test_utils.erl b/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_test_utils.erl new file mode 100644 index 0000000000..8459c8f3a8 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_test_utils.erl @@ -0,0 +1,12 @@ +-module(chef_telemetry_test_utils). + +-export([ + start_server/1 + ]). + +start_server(_Inputs) -> + application:start(inets), + {_Httpd_State, _Httpd_Pid} = + inets:start(httpd, [{port, 9001}, + {server_name, "localhost"}, {document_root, "/tmp"}, + {modules,[mod_esi]},{server_root, "/tmp"}, {erl_script_alias, {"/esi", [payload]}}]). diff --git a/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_worker_test.erl b/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_worker_test.erl new file mode 100644 index 0000000000..07ba07d127 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/test/chef_telemetry_worker_test.erl @@ -0,0 +1,171 @@ +-module(chef_telemetry_worker_test). + +-include_lib("eunit/include/eunit.hrl"). + +-export([ + get_execute/1, + adhoc_select/3, + count_nodes/1, + chef_db_list/2, + org_metadata/2, + index_search/1 +]). +%%-record(oc_chef_organization, { +%% server_api_version, +%% id, +%% authz_id, +%% name, +%% full_name, +%% assigned_at, +%% last_updated_by, +%% created_at, +%% updated_at +%% }). + +-define(DEFAULT_CONFIG, []). + +-record(state, { + fqdn_select, + last_send_timestamp, + user_emails = [], + nodes_count = 0, + organizations = ["org1", "org2"], + index_search = [["node1_org1", "node2_org1"], ["node1_org2", "node2_org2"]] +}). + +-record(expected, { + company_name = <<"">>, + nodes_count = 0, + active_nodes = 0, + fqdn = []}). + +feild_value_test() -> + State = #state{fqdn_select = {ok, [{<<"property">>, <<"FQDN:node1.domain1.com">>}, + {<<"property">>, <<"FQDN:node2.subdomain2.domain2.com">>}, + {<<"property">>, <<"FQDN:node3.subdomain3.domain3.co.uk">>}]}, + last_send_timestamp = {{2024, 8, 7}, {0, 0, 1}}, + user_emails = [[{<<"email">>, <<"test@testorg.com">>}]], + nodes_count = 10 + }, + Expected = #expected{company_name = <<"testorg">>, + nodes_count = 10, + active_nodes = 4, + fqdn = [<<".*\.domain1.com$">>, + <<".*\.subdomain2\.domain2\.com$">>, + <<".*\.subdomain3\.domain3\.co\.uk$">>]}, + execute(State, Expected, []). + +enable_flag_test() -> + State = #state{fqdn_select = {ok, []}, + last_send_timestamp = {{2024, 8, 7}, {0, 0, 1}}, + user_emails = [[{<<"email">>, <<"test@testorg.com">>}]], + nodes_count = 10 + }, + Expected = #expected{company_name = <<"testorg">>, + nodes_count = 10, + active_nodes = 4}, + ?_assertException(error, no_request1, execute(State, Expected, [{chef_telemetry, is_enabled, false}])). + +execute(State, Expected, Env) -> + set_env([{chef_telemetry, reporting_url, "http://127.0.0.1:9001/esi/payload:io"}] ++ Env), + application:start(ibrowse), + put(state, State), + setup(), + chef_telemetry_test_utils:start_server([]), + register(telemetry_mock_consumer, self()), + trigger_send_data(), + Req = get_message(), + Req1 = jiffy:decode(Req), + validate(Req1, Expected), + io:format(user, "json ~p", [jiffy:decode(Req)]). + +setup() -> + meck:new(sqerl, [passthrough]), + meck:new(chef_db, [passthrough]), + meck:new(chef_index, [passthrough]), + meck:new(release_handler, [passthrough]), + meck:new(stats_hero, [passthrough]), + meck:expect(sqerl, adhoc_insert, fun(_Table, _Rows) -> ok end), + meck:expect(sqerl, adhoc_delete, fun(_Table, _Clause) -> ok end), + meck:expect(sqerl, execute, fun get_execute/1 ), + meck:expect(sqerl, adhoc_select, fun adhoc_select/3 ), + meck:expect(chef_db, count_nodes, fun count_nodes/1 ), + meck:expect(chef_db, list, fun chef_db_list/2 ), + meck:expect(chef_db, fetch_org_metadata, fun org_metadata/2 ), + meck:expect(chef_index, search, fun index_search/1), + meck:expect(release_handler, which_releases, fun(_) -> [{"chef_server", "15.9.38", [], []}] end), + meck:expect(stats_hero, ctime, fun(_, _, Fun) -> Fun() end). + +get_execute(<<"select property from telemetry where property like 'FQDN:%'">>) -> + State = get(state), + State#state.fqdn_select; + +get_execute(_) -> + ok. + +adhoc_select([<<"email">>], <<"users">>, all) -> + State = get(state), + {ok, State#state.user_emails}; + +adhoc_select([<<"event_timestamp">>], <<"telemetry">>, {<<"property">>, equals, <<"last_send">>}) -> + State = get(state), + {ok, [[{<<"event_timestamp">>, State#state.last_send_timestamp}]]}. + +count_nodes(_Context) -> + State = get(state), + State#state.nodes_count. + +chef_db_list(Record, _context) -> + RecordName = element(1, Record), + State = get(state), + case RecordName of + oc_chef_organization -> State#state.organizations; + _ -> [] + end. + +org_metadata(_context, OrgName) -> + OrgName1 = list_to_binary(OrgName), + {OrgName1, OrgName1}. + +index_search(_) -> + State = get(state), + [Nodes | Rest] = State#state.index_search, + State1 = State#state{index_search = Rest}, + put(state, State1), + {ok, 0, length(Nodes), Nodes}. + +trigger_send_data() -> + {ok, State} = chef_telemetry_worker:init([]), + chef_telemetry_worker:handle_cast(send_data, State). + +get_message() -> + receive + {http_request, _From, Req} -> + Req + after 5000 -> + throw(no_request) + end. + +validate(Req, Expected) -> + Licence = ej:get({<<"licenseId">>}, Req), + TotalNodes = ej:get({<<"periods">>, 1, <<"summary">>, <<"nodes">>, <<"total">>}, Req), + ActiveNodes = ej:get({<<"periods">>, 1, <<"summary">>, <<"nodes">>, <<"active">>}, Req), + FQDNs = ej:get({<<"metadata">>, <<"Infra Server">>, <<"fqdn">>}, Req), + ?assertEqual(<<"Infra-Server-license-Id">>, Licence), + ?assertEqual(Expected#expected.nodes_count, TotalNodes), + ?assertEqual(Expected#expected.active_nodes, ActiveNodes), + ?assertEqual(true, check_fqdn(FQDNs, Expected#expected.fqdn)). + +set_env(ConfigList) -> + ConfigList1 = ?DEFAULT_CONFIG ++ ConfigList, + [ application:set_env(App, Parameter, Value) || {App, Parameter, Value} <- ConfigList1 ]. + +check_fqdn(ReqFQDNs, Expected) -> + MatchFun = + fun(FQDN) -> + lists:any( + fun(Pattern) -> + match == re:run(FQDN, Pattern, [{capture, none}]) + end, Expected) + end, + lists:all(MatchFun, ReqFQDNs). \ No newline at end of file diff --git a/src/oc_erchef/apps/chef_telemetry/test/payload.erl b/src/oc_erchef/apps/chef_telemetry/test/payload.erl new file mode 100644 index 0000000000..322f118b68 --- /dev/null +++ b/src/oc_erchef/apps/chef_telemetry/test/payload.erl @@ -0,0 +1,11 @@ +-module(payload). + +-export([ + io/3]). + +io(SessionID, Env, Input) -> + io:format(user, "pid ~p", [whereis(telemetry_mock_consumer)]), + telemetry_mock_consumer ! {http_request, self(), Input}, + io:format(user, "input parameters ~p, ~p, ~p", [SessionID, Env, Input]), + mod_esi:deliver(SessionID, "status:201 Created\r\nContent-Type:text/html\r\n\r\n"), + ok. \ No newline at end of file diff --git a/src/oc_erchef/apps/data_collector/src/data_collector.app.src b/src/oc_erchef/apps/data_collector/src/data_collector.app.src index 97e61ab169..218fea9d90 100644 --- a/src/oc_erchef/apps/data_collector/src/data_collector.app.src +++ b/src/oc_erchef/apps/data_collector/src/data_collector.app.src @@ -26,6 +26,8 @@ {vsn, {cmd,"cat ../../VERSION | awk '{print $0}'"}}, {registered, []}, {applications, [ + kernel, + stdlib, lager, chef_secrets, opscoderl_httpc diff --git a/src/oc_erchef/schema/deploy/telemetry.sql b/src/oc_erchef/schema/deploy/telemetry.sql new file mode 100644 index 0000000000..1bb3a48819 --- /dev/null +++ b/src/oc_erchef/schema/deploy/telemetry.sql @@ -0,0 +1,11 @@ +-- Deploy telemetry + +BEGIN; + +CREATE TABLE IF NOT EXISTS telemetry( + property CHAR(32), + value_string CHAR(32) NOT NULL, + event_timestamp TIMESTAMP +); + +COMMIT; \ No newline at end of file diff --git a/src/oc_erchef/schema/revert/telemetry.sql b/src/oc_erchef/schema/revert/telemetry.sql new file mode 100644 index 0000000000..ecacb73bd5 --- /dev/null +++ b/src/oc_erchef/schema/revert/telemetry.sql @@ -0,0 +1,7 @@ +-- Revert telemetry + +BEGIN; + +DROP TABLE IF EXISTS telemetry; + +COMMIT; \ No newline at end of file diff --git a/src/oc_erchef/schema/sqitch.plan b/src/oc_erchef/schema/sqitch.plan index 46f39faf15..9bb8670e05 100644 --- a/src/oc_erchef/schema/sqitch.plan +++ b/src/oc_erchef/schema/sqitch.plan @@ -84,3 +84,5 @@ users_email_functional_index [users] 2017-05-30T12:10:32Z Stephan Renatus # Adding functions to create and update users by inserting a sentinel value in the public_key_columns keys_update_trigger [keys_update_trigger@users_email_functional_index] 2017-08-23T23:12:29Z Prajakta Purohit # The insert and update triggers ignore rows with sentinel value for public_key. @sentinel_public_key_for_users 2017-08-24T14:32:04Z Prajakta Purohit # public_key only updated in the keys table + +telemetry 2024-08-01T08:37:30Z Sreepuram Sudheer # Adding telemetry to chef-server. diff --git a/src/oc_erchef/schema/verify/telemetry.sql b/src/oc_erchef/schema/verify/telemetry.sql new file mode 100644 index 0000000000..9f580de290 --- /dev/null +++ b/src/oc_erchef/schema/verify/telemetry.sql @@ -0,0 +1,7 @@ +-- Verify enterprise_chef:telemetry on pg + +BEGIN; + +-- XXX Add verifications here. + +ROLLBACK; diff --git a/src/oc_erchef/src/oc_erchef.app.src b/src/oc_erchef/src/oc_erchef.app.src index 3a6615781b..4395947ecc 100644 --- a/src/oc_erchef/src/oc_erchef.app.src +++ b/src/oc_erchef/src/oc_erchef.app.src @@ -31,7 +31,8 @@ opscoderl_httpc, oc_chef_authz, oc_chef_wm, - data_collector + data_collector, + chef_telemetry ]}, {applications, [lager, chef_secrets,