Skip to content

Commit

Permalink
Adds initial algorithm metric reports
Browse files Browse the repository at this point in the history
Why are these changes being introduced:

* Allowing a way to access our historical matches over time will be
  useful

Relevant ticket(s):

* https://mitlibraries.atlassian.net/browse/TCO-63
* https://mitlibraries.atlassian.net/browse/TCO-19

How does this address that need:

* Adds Report index
* Adds Metrics Algorithm report with monthly and aggregate support

Document any side effects to this change:

* Updates ability.rb for SuggestedResources to handle the difference in
  syntax when working in Administrate context versus the rest of our app
  • Loading branch information
JPrevost committed Sep 23, 2024
1 parent c18e50c commit 59f9ac0
Show file tree
Hide file tree
Showing 9 changed files with 129 additions and 6 deletions.
13 changes: 13 additions & 0 deletions app/controllers/report_controller.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# frozen_string_literal: true

class ReportController < ApplicationController
def index; end

def algorithm_metrics
@metrics = if params[:type] == 'aggregate'
Metrics::Algorithms.aggregates
else
Metrics::Algorithms.monthlies
end
end
end
18 changes: 18 additions & 0 deletions app/helpers/metrics_helper.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# frozen_string_literal: true

module MetricsHelper
# Calculate percentage of search events in which we had any Detection match
def percent_match(metric)
(sum_matched(metric) / sum_total(metric) * 100).round(2)
end

# Sums all detection matches for a given Metrics::Algorithms record
def sum_matched(metric)
metric.doi.to_f + metric.issn.to_f + metric.isbn.to_f + metric.pmid.to_f + metric.journal_exact.to_f + metric.suggested_resource_exact.to_f
end

# Calculates total events for a given Metrics::Algorithms record
def sum_total(metric)
sum_matched(metric) + metric.unmatched.to_f
end
end
12 changes: 9 additions & 3 deletions app/models/ability.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,18 @@ class Ability
# See the wiki for details:
# https://github.com/CanCanCommunity/cancancan/blob/develop/docs/define_check_abilities.md
def initialize(user)
# Actions allowed for non-authenticated Users should add `skip_before_action :require_user`

# Start of Rules for all authenticated user with no additional roles required
return if user.blank?
# Rules will go here.

# all authenticated
# can :view, :playground
can :manage, :detector__suggested_resource
can :manage, Detector::SuggestedResource

can :view, :report
# End of Rules for all authenticated user with no additional roles required

# Rules for admins
return unless user.admin?

can :manage, :all
Expand Down
3 changes: 3 additions & 0 deletions app/models/metrics/algorithms.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ module Metrics
class Algorithms < ApplicationRecord
self.table_name = 'metrics_algorithms'

scope :monthlies, -> { where.not(month: nil).order(month: :asc) }
scope :aggregates, -> { where(month: nil).order(month: :asc) }

# generate metrics data about SearchEvents matches
#
# @note This is expected to only be run once per month per type of aggregation (once with no month supplied, once
Expand Down
12 changes: 9 additions & 3 deletions app/views/layouts/_site_nav.html.erb
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@
<% if can? :view, :playground %>
<%= link_to('Playground', '/playground', class: 'nav-item') %>
<% end %>
<% if can? :manage, :detector__suggested_resource %>
<%= link_to('Suggested Resources', admin_detector_suggested_resources_path, class: 'nav-item') %>
<% end %>
<% if can? :view, :report %>
<%= link_to('Reports', report_path, class: 'nav-item') %>
<% end %>
<% end %>
</nav>
<nav class="nav-user" aria-label="User menu">
Expand All @@ -24,11 +30,11 @@
method: :delete) %>
<% else %>
<% if FakeAuthConfig.fake_auth_enabled? %>
<%= button_to("Sign in", user_developer_omniauth_authorize_path, id: "sign_in", class: 'action-auth',
method: :post) %>
<%= button_to("Sign in", user_developer_omniauth_authorize_path, id: "sign_in", class: 'action-auth',
data: { turbo:false }, method: :post) %>
<% else %>
<%= button_to("Sign in", user_openid_connect_omniauth_authorize_path, class: 'action-auth', id: "sign_in",
method: :post) %>
data: { turbo:false }, method: :post) %>
<% end %>
<% end %>
</nav>
Expand Down
40 changes: 40 additions & 0 deletions app/views/report/algorithm_metrics.html.erb
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
<div class="alert alert-banner warning">
<p><i class="fa fa-exclamation-circle fa-lg"></i> Percentage match is not accurate for Terms that match multiple algorithms. The actual percentage match will be lower than the reported value. At this time, the error is not believed to be significant based on the currently low volume of Terms that match multiple algorithms. This may change as we develop new algorithms to a point where we need to address this discrepancy.</p>
</div>

<table class="table">
<tr>
<% if params[:type] == 'aggregate' %>
<th>Aggregation date</th>
<% else %>
<th>Month</th>
<% end %>
<th>DOI</th>
<th>ISSN</th>
<th>ISBN</th>
<th>PMID</th>
<th>Journal</th>
<th>SuggestedResource</th>
<th>Unmatched</th>
<th>% matched</th>
</tr>
<% @metrics.each do |metric| %>
<tr>
<% if params[:type] == 'aggregate' %>
<td><%= metric.updated_at.strftime("%d %B %Y") %></td>
<% else %>
<td><%= metric.month.strftime("%B %Y") %></td>
<% end %>
<td><%= metric.doi %></td>
<td><%= metric.issn %></td>
<td><%= metric.isbn %></td>
<td><%= metric.pmid %></td>
<td><%= metric.journal_exact %></td>
<td><%= metric.suggested_resource_exact %></td>
<td><%= metric.unmatched %></td>
<td>
<%= percent_match(metric) %>
</td>
</tr>
<% end %>
</table>
6 changes: 6 additions & 0 deletions app/views/report/index.html.erb
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
<h3>Report index</h3>

<ul>
<li><%= link_to("Monthly algorithm metrics", report_algorithm_metrics_path) %></li>
<li><%= link_to("Aggregate algorithm metrics", report_algorithm_metrics_path(type: 'aggregate') ) %></li>
</ul>
2 changes: 2 additions & 0 deletions config/routes.rb
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
get 'up' => 'rails/health#show', as: :rails_health_check

get 'playground', to: 'static#playground'
get '/report', to: 'report#index'
get '/report/algorithm_metrics', to: 'report#algorithm_metrics'

# Defines the root path route ("/")
root to: 'static#index'
Expand Down
29 changes: 29 additions & 0 deletions test/helpers/metrics_helper_test.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
require 'test_helper'

class MetricsHelperTest < ActionView::TestCase
def metric
Metrics::Algorithms.new(
month: DateTime.now - 1.month,
doi: 1,
issn: 2,
isbn: 3,
pmid: 4,
unmatched: 79,
journal_exact: 5,
suggested_resource_exact: 6,
created_at: DateTime.now,
updated_at: DateTime.now
)
end
test 'can calculate a percentage of matches for a record' do
assert_in_delta(21.0, percent_match(metric), 0.01)
end

test 'can calculate sum of matches for a record' do
assert_in_delta(21.0, sum_matched(metric), 0.01)
end

test 'can calculate sum of all events for a record' do
assert_in_delta(100.0, sum_total(metric), 0.01)
end
end

0 comments on commit 59f9ac0

Please sign in to comment.