From 68d245fa88c2737179dffc9958da384307d11322 Mon Sep 17 00:00:00 2001 From: Oleg Hasjanov Date: Fri, 6 Oct 2023 12:19:26 +0300 Subject: [PATCH 1/2] optimise statistic data calculations --- Gemfile | 1 + Gemfile.lock | 4 +++- app/controllers/repp/v1/stats_controller.rb | 13 ++++++++++- script/benchmarks/static_benchmark.rb | 26 +++++++++++++++++++++ 4 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 script/benchmarks/static_benchmark.rb diff --git a/Gemfile b/Gemfile index 0f2e855ab3..4349c82238 100644 --- a/Gemfile +++ b/Gemfile @@ -106,3 +106,4 @@ gem 'pg_query', '>= 0.9.0' # token gem 'jwt' +gem 'benchmark-ips' diff --git a/Gemfile.lock b/Gemfile.lock index 4fe330f693..80e678f153 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -168,6 +168,7 @@ GEM aws-sigv4 (1.2.4) aws-eventstream (~> 1, >= 1.0.2) bcrypt (3.1.16) + benchmark-ips (2.12.0) bindata (2.4.14) bootsnap (1.9.3) msgpack (~> 1.0) @@ -539,6 +540,7 @@ DEPENDENCIES airbrake apipie-rails (~> 0.6.0) aws-sdk-sesv2 (~> 1.19) + benchmark-ips bootsnap (>= 1.1.0) bootstrap-sass (~> 3.4) cancancan @@ -604,4 +606,4 @@ DEPENDENCIES wkhtmltopdf-binary (~> 0.12.6.1) BUNDLED WITH - 2.4.19 + 2.4.20 diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index 052cdcbc45..cd7f576d4c 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -40,7 +40,7 @@ def market_share_growth_rate end # rubocop:enable Metrics/MethodLength - private + # private def search_params params.permit(:q, q: %i[start_date end_date compare_to_end_date compare_to_start_date]) @@ -103,6 +103,17 @@ def log_domains(event:, date_to:, date_from:) .order(Arel.sql("object ->> 'name', created_at desc")) end + def log_domains2(event:, date_to:, date_from:) + domains = ::Version::DomainVersion.where(event: event) + domains.where!("object_changes ->> 'registrar_id' IS NOT NULL") if event == 'update' + domains.where('created_at > ?', date_to) + .where("object ->> 'created_at' <= ?", date_to) + .where("object ->> 'created_at' >= ?", date_from) + # .select("DISTINCT ON (object ->> 'name') object, created_at") + # .order(Arel.sql("object ->> 'name', created_at desc")) + end + + def group(domains) domains.group_by { |ld| ld.object['registrar_id'].to_s } .transform_values(&:count) diff --git a/script/benchmarks/static_benchmark.rb b/script/benchmarks/static_benchmark.rb new file mode 100644 index 0000000000..91b6bb3a20 --- /dev/null +++ b/script/benchmarks/static_benchmark.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require_relative "../../config/environment" + +# Any benchmarking setup goes here... + + + +Benchmark.ips do |x| + x.report("before") do + date_to = Date.strptime("10.23", '%m.%y').end_of_month + date_from = Date.strptime("01.22", '%m.%y').end_of_month + + res = Repp::V1::StatsController.new.log_domains(event: 'update', date_to: date_to, date_from: date_from) + puts res.size + end + x.report("after") do + date_to = Date.strptime("10.23", '%m.%y').end_of_month + date_from = Date.strptime("01.22", '%m.%y').end_of_month + + res = Repp::V1::StatsController.new.log_domains2(event: 'update', date_to: date_to, date_from: date_from) + puts res.size + end + + x.compare! +end From d9d49739a5d3b4729823c8efaf0183f5e6710b0f Mon Sep 17 00:00:00 2001 From: Oleg Hasjanov Date: Fri, 6 Oct 2023 16:18:28 +0300 Subject: [PATCH 2/2] added test endpoint --- app/controllers/repp/v1/stats_controller.rb | 102 ++++++++++++++++---- config/routes.rb | 1 + script/benchmarks/static_benchmark.rb | 26 ----- script/benchmarks/static_benchmark_ips.rb | 33 +++++++ script/benchmarks/static_benchmark_time.rb | 23 +++++ 5 files changed, 140 insertions(+), 45 deletions(-) delete mode 100644 script/benchmarks/static_benchmark.rb create mode 100644 script/benchmarks/static_benchmark_ips.rb create mode 100644 script/benchmarks/static_benchmark_time.rb diff --git a/app/controllers/repp/v1/stats_controller.rb b/app/controllers/repp/v1/stats_controller.rb index cd7f576d4c..cd4cc1a662 100644 --- a/app/controllers/repp/v1/stats_controller.rb +++ b/app/controllers/repp/v1/stats_controller.rb @@ -14,17 +14,22 @@ def market_share_distribution render_success(data: result) end - # rubocop:disable Metrics/MethodLength - api :get, '/repp/v1/stats/market_share_growth_rate' - desc 'Get market share and growth rate of registrars' - param :q, Hash, required: true, desc: 'Period parameters for data' do - param :end_date, String, required: true, desc: 'Period end date' - param :compare_to_end_date, String, required: true, desc: 'Comparison date' - end + # # rubocop:disable Metrics/MethodLength + # api :get, '/repp/v1/stats/market_share_growth_rate' + # desc 'Get market share and growth rate of registrars' + # param :q, Hash, required: true, desc: 'Period parameters for data' do + # param :end_date, String, required: true, desc: 'Period end date' + # param :compare_to_end_date, String, required: true, desc: 'Comparison date' + # end def market_share_growth_rate domains_by_rar = domains_by_registrar(@date_to, @date_from) prev_domains_by_rar = domains_by_registrar(@date_compare_to, @date_compare_from) + puts '---------' + puts domains_by_rar + puts prev_domains_by_rar + puts '---------' + set_zero_values!(domains_by_rar, prev_domains_by_rar) market_share_by_rar = calculate_market_share(domains_by_rar) @@ -36,7 +41,77 @@ def market_share_growth_rate data: { name: search_params[:end_date], domains: serialize_growth_rate_result(domains_by_rar), market_share: serialize_growth_rate_result(market_share_by_rar) } } - render_success(data: result) + # render_success(data: result) + end + + # rubocop:disable Metrics/MethodLength + # api :get, '/repp/v1/stats/market_share_growth_rate2' + # desc 'Get market share and growth rate of registrars' + # param :q, Hash, required: true, desc: 'Period parameters for data' do + # param :end_date, String, required: true, desc: 'Period end date' + # param :compare_to_end_date, String, required: true, desc: 'Comparison date' + # end + def market_share_growth_rate2 + # domains_by_rar = domains_by_registrar(@date_to, @date_from) + # prev_domains_by_rar = domains_by_registrar(@date_compare_to, @date_compare_from) + + query = ActiveRecord::Base.send(:sanitize_sql_array, [" + SELECT \"object_changes\" -> 'registrar_id' ->> 1 AS \"registrar_id\", COUNT(*) AS \"domain_count\" + FROM \"log_domains\" + WHERE (\"event\" = 'create' OR \"event\" = 'update' OR \"event\" = 'delete') + AND \"created_at\" BETWEEN ? AND ? + AND ( + \"event\" != 'update' OR + (\"event\" = 'update' AND \"object_changes\" ->> 'registrar_id' IS NOT NULL) + ) + GROUP BY \"object_changes\" -> 'registrar_id' ->> 1", @date_from, @date_to]) + + + result = ActiveRecord::Base.connection.execute(query) + + # Преобразование результата в хэш + domains_by_rar = {} + result.each do |row| + domains_by_rar[row['registrar_id']] = row['domain_count'].to_i + end + + query = ActiveRecord::Base.send(:sanitize_sql_array, [" + SELECT \"object_changes\" -> 'registrar_id' ->> 1 AS \"registrar_id\", COUNT(*) AS \"domain_count\" + FROM \"log_domains\" + WHERE (\"event\" = 'create' OR \"event\" = 'update' OR \"event\" = 'delete') + AND \"created_at\" BETWEEN ? AND ? + AND ( + \"event\" != 'update' OR + (\"event\" = 'update' AND \"object_changes\" ->> 'registrar_id' IS NOT NULL) + ) + GROUP BY \"object_changes\" -> 'registrar_id' ->> 1", @date_compare_from, @date_compare_to]) + + + result = ActiveRecord::Base.connection.execute(query) + + # Преобразование результата в хэш + prev_domains_by_rar = {} + result.each do |row| + prev_domains_by_rar[row['registrar_id']] = row['domain_count'].to_i + end + + puts '---------' + puts domains_by_rar + puts prev_domains_by_rar + puts '---------' + + set_zero_values!(domains_by_rar, prev_domains_by_rar) + + market_share_by_rar = calculate_market_share(domains_by_rar) + prev_market_share_by_rar = calculate_market_share(prev_domains_by_rar) + + result = { prev_data: { name: search_params[:compare_to_end_date], + domains: serialize_growth_rate_result(prev_domains_by_rar), + market_share: serialize_growth_rate_result(prev_market_share_by_rar) }, + data: { name: search_params[:end_date], + domains: serialize_growth_rate_result(domains_by_rar), + market_share: serialize_growth_rate_result(market_share_by_rar) } } + # render_success(data: result) end # rubocop:enable Metrics/MethodLength @@ -103,17 +178,6 @@ def log_domains(event:, date_to:, date_from:) .order(Arel.sql("object ->> 'name', created_at desc")) end - def log_domains2(event:, date_to:, date_from:) - domains = ::Version::DomainVersion.where(event: event) - domains.where!("object_changes ->> 'registrar_id' IS NOT NULL") if event == 'update' - domains.where('created_at > ?', date_to) - .where("object ->> 'created_at' <= ?", date_to) - .where("object ->> 'created_at' >= ?", date_from) - # .select("DISTINCT ON (object ->> 'name') object, created_at") - # .order(Arel.sql("object ->> 'name', created_at desc")) - end - - def group(domains) domains.group_by { |ld| ld.object['registrar_id'].to_s } .transform_values(&:count) diff --git a/config/routes.rb b/config/routes.rb index 6652f013ed..59a89e4683 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -106,6 +106,7 @@ collection do get '/market_share_distribution', to: 'stats#market_share_distribution' get '/market_share_growth_rate', to: 'stats#market_share_growth_rate' + get '/market_share_growth_rate2', to: 'stats#market_share_growth_rate2' end end resources :api_users, only: %i[index show update create destroy] do diff --git a/script/benchmarks/static_benchmark.rb b/script/benchmarks/static_benchmark.rb deleted file mode 100644 index 91b6bb3a20..0000000000 --- a/script/benchmarks/static_benchmark.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -require_relative "../../config/environment" - -# Any benchmarking setup goes here... - - - -Benchmark.ips do |x| - x.report("before") do - date_to = Date.strptime("10.23", '%m.%y').end_of_month - date_from = Date.strptime("01.22", '%m.%y').end_of_month - - res = Repp::V1::StatsController.new.log_domains(event: 'update', date_to: date_to, date_from: date_from) - puts res.size - end - x.report("after") do - date_to = Date.strptime("10.23", '%m.%y').end_of_month - date_from = Date.strptime("01.22", '%m.%y').end_of_month - - res = Repp::V1::StatsController.new.log_domains2(event: 'update', date_to: date_to, date_from: date_from) - puts res.size - end - - x.compare! -end diff --git a/script/benchmarks/static_benchmark_ips.rb b/script/benchmarks/static_benchmark_ips.rb new file mode 100644 index 0000000000..660af0ac54 --- /dev/null +++ b/script/benchmarks/static_benchmark_ips.rb @@ -0,0 +1,33 @@ +require_relative "../../config/environment" +require 'benchmark/ips' + +Benchmark.ips do |x| + x.warmup = 2 + x.time = 5 + + controller = Repp::V1::StatsController.new + controller.params = ActionController::Parameters.new( + q: { + end_date: '10.23', + compare_to_end_date: '10.22' + } + ) + # date_to = Date.strptime("10.23", '%m.%y').end_of_month + # date_from = Date.strptime("01.22", '%m.%y').end_of_month + + controller.set_date_params # Вызывает метод, который устанавливает переменные экземпляра + + x.report("before") do + ActiveRecord::Base.uncached do + res = controller.market_share_growth_rate + end + end + + x.report("after") do + ActiveRecord::Base.uncached do + res = controller.market_share_growth_rate2 + end + end + + x.compare! +end \ No newline at end of file diff --git a/script/benchmarks/static_benchmark_time.rb b/script/benchmarks/static_benchmark_time.rb new file mode 100644 index 0000000000..bcbb671ebe --- /dev/null +++ b/script/benchmarks/static_benchmark_time.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require_relative "../../config/environment" +require 'benchmark/ips' # Подразумевается, что gem 'benchmark-ips' установлен + +date_to = Date.strptime("10.23", '%m.%y').end_of_month +date_from = Date.strptime("01.22", '%m.%y').end_of_month +controller = Repp::V1::StatsController.new + +time_before = Benchmark.realtime do + ActiveRecord::Base.uncached do + res = controller.log_domains(event: 'update', date_to: date_to, date_from: date_from) + end +end + +time_after = Benchmark.realtime do + ActiveRecord::Base.uncached do + res = controller.log_domains(event: 'update', date_to: date_to, date_from: date_from) + end +end + +puts "Time for 'before': #{time_before} seconds" +puts "Time for 'after': #{time_after} seconds"