Merge branch 'master' of baltig.sandia.gov:scot/SCOT

.gitignore

@@ -22,3 +22,4 @@ scot-debug.js
 scot-ui/build/static/js/
 scot-ui/build/
 .vscode/
+emailapi.cfg.pl

Dockerfile-Perl

@@ -41,7 +41,7 @@ IPC::Run IO::Prompt  Log::Log4perl Mail::IMAPClient Mail::IMAPClient::BodyStruct
 Alien::GMP Meerkat Mojo MojoX::Log::Log4perl Mojolicious::Plugin::WithCSRFProtection Mojolicious::Plugin::TagHelpers XML::Smart Config::Auto Data::GUID File::LibMagic List::Uniq \ 
 Domain::PublicSuffix Mozilla::PublicSuffix Crypt::PBKDF2 Config::Crontab Math::Int128 GeoIP2 Search::Elasticsearch Term::ANSIColor \ 
 Courriel Statistics::Descriptive Net::SSH::Perl Net::SFTP Lingua::Stem Math::VecStat Class::Exporter Math::HashSum Math::Vector::SortIndexes Lingua::EN::StopWords \ 
-XML::Twig XML::Simple SVG::Sparkline Email::Stuffer HTML::Entities HTML::Scrubber HTML::Strip HTML::StripTags HTML::TreeBuilder HTML::FromText HTML::FormatText 
+XML::Twig XML::Simple SVG::Sparkline Email::Stuffer HTML::Entities HTML::Scrubber HTML::Strip HTML::StripTags HTML::TreeBuilder HTML::FromText HTML::FormatText HTML::Make
 
 #problem modules that should be run last
 RUN cpanm MaxMind::DB::Reader::XS 

Dockerfile-Rfproxy

@@ -0,0 +1,10 @@
+FROM sandialabs/scot_perl 
+
+#Create log directory
+RUN mkdir -p /var/log/scot
+RUN mkdir -p /opt/scot
+
+COPY install/src/scot/ /opt/scot/etc/
+COPY bin/rfproxy.pl /opt/scot/bin/
+
+CMD ["/usr/bin/perl", "/opt/scot/bin/rfproxy.pl"]

bin/ebm.pl

@@ -0,0 +1,44 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+use v5.16;
+
+use lib '../../Scot-Internal-Modules/lib';
+use lib '../lib';
+use lib '/opt/scot/lib';
+use Scot::App::Mail;
+use Scot::Env;
+use Data::Dumper;
+use DateTime;
+
+my $config_file = $ENV{'scot_config_file'} // 
+                    '/opt/scot/etc/scot.cfg.pl';
+
+my $env = Scot::Env->new(
+    config_file => $config_file,
+);
+
+my $mongo       = $env->mongo;
+my $collection  = $mongo->collection('Event');
+my $cursor      = $collection->find();;
+my %results;
+
+while (my $ag = $cursor->next) {
+    my $created = $ag->created;
+    my $dt      = DateTime->from_epoch( epoch => $created );
+    my $year    = $dt->year;
+    my $month   = $dt->month;
+    $results{$year}{$month}++;
+}
+
+say "event counts";
+say "Year, Month, Amount";
+foreach my $y (sort {$a<=>$b} keys %results) {
+    foreach my $m (sort {$a<=>$b} keys %{$results{$y}} ) {
+        say "$y-$m, $results{$y}{$m}";
+    }
+}
+
+
+

bin/reports.pl

@@ -4,8 +4,8 @@
 use strict;
 use warnings;
 use v5.16;
-use Mojo::DOM;
 use Scot::Env;
+use Time::Seconds;
 
 =head1 NAME
 
@@ -97,7 +97,7 @@ =head1 PROGRAM ARGUMENTS
     die "Could not parse datetimes\n";
 }
 
-print "SCOT Stats\n\n";
+print "SCOT Stats $report\n\n";
 print "For period:\n";
 print  "Start                    End\n";
 printf "%17s        %17s\n", $start, $end;
@@ -123,18 +123,22 @@ =head1 PROGRAM ARGUMENTS
 elsif ( $report eq "event_report" ) {
     event_report();
 }
+elsif ( $report eq "incident_close_times" ) {
+    incident_stats(); 
+}
 
 
 $env->log->debug("========= Finished $0 ==========");
 exit 0;
 
 sub list_reports {
     print "
-pyramid             Print totals for Alerts Events and Incidents
-alerts_by_month     Print the total alerts receive by month
-response_times      Print the average response time for alerts by year and month
-compare_days        print totals for the last for days of the week specified by                         the start parameter
-event_report        Print the Event report
+pyramid                 Print totals for Alerts Events and Incidents
+alerts_by_month         Print the total alerts receive by month
+response_times          Print the average response time for alerts by year and month
+compare_days            print totals for the last for days of the week specified by                         the start parameter
+event_report            Print the Event report
+incident_close_times    Print statistics about incident closure
 ";
     exit 1;
 }
@@ -388,3 +392,37 @@ sub event_report {
         }
     }
 }
+
+sub incident_stats {
+    my $query   = {
+        created => {
+            '$gte'  => $thendt->epoch,
+            '$lte'  => $nowdt->epoch,
+        },
+        status  => 'closed',
+        closed => { '$ne' => 0 },
+    };
+
+    my $cursor = $mongo->collection('Incident')->find($query);
+    my @values;
+
+    while (my $incident = $cursor->next) {
+        push @values, $incident->closed - $incident->created
+    }
+
+    my $util = Statistics::Descriptive::Sparse->new();
+    $util->add_data(@values);
+
+    print "Average Close = ".$get_readable_time($util->mean)."\n";
+    print "Minimum Close = ".$get_readable_time($util->min)."\n";
+    print "Maximum Close = ".$get_readable_time($util->max)."\n";
+    print "Std. of Dev.  = ".$util->statndar_deviation."\n";
+    print "Number of incidents = ".$util->count."\n";
+
+}
+
+sub get_readable_time {
+    my $seconds = shift;
+    my $t = Time::Seconds->new($seconds);
+    return $t->pretty;
+}

docker-compose.yml

@@ -99,6 +99,19 @@ services:
    container_name: reflair
    image: sandialabs/scot_reflair
 
+  rfproxy:
+   volumes:
+    - "rfproxy_log_data:/var/log/scot/"
+    - "/etc/localtime:/etc/localtime:ro"
+   depends_on:
+    - mongodb
+    - activemq
+    - scot
+   networks:
+     - scot-docker-net
+   container_name: rfproxy
+   image: sandialabs/scot_rfproxy
+
   stretch:
    image: scot_stretch
    volumes:

install/install_packages.sh

@@ -148,12 +148,10 @@ function install_cent_packages {
         krb5-libs
         krb5-devel
         GeoIP
-        libmaxminddb
-        libmaxminddb-devel
+        libmaxminddb.x86_64
+        libmaxminddb-devel.x86_64
         libgmp3-devel
         policycoreutils-python
-	perl-File-LibMagic
-	expat-devel
     '
 
     for pkg in $YUMPACKAGES; do

install/src/scot/migrate.cfg.pl

@@ -130,13 +130,6 @@
                 suffixfile  => '/opt/scot/etc/effective_tld_names.dat',
             },
         },
-        {
-            attr    => 'regex',
-            class   => 'Scot::Extractor::Regex',
-            config  => {
-                entity_regexes => [],
-            },
-        },
     ],
 );
 

install/src/scot/recfuture.cfg.pl

@@ -1,6 +1,4 @@
 %environment = (
-    api_key => "your_key_goes_here",
-    proxy => "proxy_url_here",
     location    => "snl",
     site_identifier    => "snl",
     time_zone   => 'America/Denver',

lib/Scot.pm

@@ -438,6 +438,12 @@ relies on the browser BasicAuth popup.
             ->to    ('controller-api#recfuture')
             ->name  ('get_recfuture_data');
 
+    $scot   ->route ('/api/v2/lriproxy/:id')
+            ->via   ('get')
+            ->to    ('controller-api#lriproxy')
+            ->name  ('get_lri_data');
+
+
 =pod
 
 @api {post} /scot/api/v2/:thing Create thing

lib/Scot/App/Migrate.pm

@@ -163,7 +163,7 @@ sub migrate {
     my $legacy_cursor       = $legacy_collection->find($findjson);
     $legacy_cursor->immortal(1);
 
-    my $remaining_docs  = $legacy_collection->count($findjson);
+    my $remaining_docs  = $legacy_collection->count_documents($findjson);
     my $migrated_docs   = 0;
     my $total_docs      = $remaining_docs;
     my $total_time      = 0;
@@ -459,7 +459,7 @@ sub xform_alertgroup {
 
     my $legacy_alert_cursor = $leg_alert_col->find({alertgroup => $id});
     $legacy_alert_cursor->immortal(1);
-    $href->{alert_count} = $leg_alert_col->count({alertgroup => $id});
+    $href->{alert_count} = $leg_alert_col->count_documents({alertgroup => $id});
 
     my $entities;           # ... keep track of entities found
     my @alert_promotions;   # ... and promoted alerts

lib/Scot/App/Report.pm

@@ -7,6 +7,7 @@ use warnings;
 use DateTime;
 use Time::Duration;
 use Data::Dumper;
+use Statistics::Descriptive::Sparse;
 
 use Moose;
 extends 'Scot::App';
@@ -72,6 +73,45 @@ sub alertgroup_counts {
     return wantarray ? %result : \%result;
 }
 
+sub incident_avg_time_to_close {
+    my $self    = shift;
+    my $env     = $self->env;
+    my $mongo   = $env->mongo;
+    my $log     = $env->log;
+    my $nowdt   = $self->nowdt;
+    my $thendt  = $self->thendt;
+
+    $log->debug("calculating avg time for incident close");
+
+    my $query   = {
+        created => {
+            '$gte'  => $thendt->epoch,
+            '$lte'  => $nowdt->epoch,
+        },
+        status  => 'closed',
+        closed => { '$ne' => 0 },
+    };
+
+    my $cursor = $mongo->collection('Incident')->find($query);
+    my @values;
+
+    while (my $incident = $cursor->next) {
+        push @values, $incident->closed - $incident->created
+    }
+
+    my $util = Statistics::Descriptive::Sparse->new();
+    $util->add_data(@values);
+
+    my $result  = {
+        avg     => $util->mean,
+        min     => $util->min,
+        max     => $util->max,
+        stddev  => $util->standard_deviation,
+        count   => $util->count,
+    };
+    return $result;
+}
+
 sub event_counts {
     my $self    = shift;
     my $env     = $self->env;