diff --git a/Makefile.PL b/Makefile.PL index fecfe3955..c47c5a469 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -59,11 +59,10 @@ my %WriteMakefileArgs = ( "DBIx::Class::UUIDColumns" => 0, "DBIx::QuickDB" => "0.000020", "Data::Dumper" => 0, - "Data::GUID" => 0, - "Data::UUID" => 0, "DateTime" => 0, "DateTime::Format::MySQL" => 0, "DateTime::Format::Pg" => 0, + "DateTime::Format::SQLite" => 0, "Email::Sender::Simple" => 0, "Email::Simple" => 0, "Email::Simple::Creator" => 0, @@ -114,7 +113,7 @@ my %WriteMakefileArgs = ( "Test2::Event::V2" => "1.302198", "Test2::Formatter" => "1.302198", "Test2::Plugin::MemUsage" => "0.002003", - "Test2::Plugin::UUID" => "0.002001", + "Test2::Plugin::UUID" => "0.002008", "Test2::Tools::AsyncSubtest" => "0.000159", "Test2::Tools::Basic" => 0, "Test2::Tools::Compare" => 0, @@ -128,6 +127,7 @@ my %WriteMakefileArgs = ( "Test2::V0" => "0.000159", "Test::Builder" => "1.302198", "Test::Builder::Formatter" => "1.302198", + "Test::Harness" => "3.49", "Test::More" => "1.302198", "Text::ParseWords" => 0, "Text::Xslate" => 0, @@ -176,11 +176,10 @@ my %FallbackPrereqs = ( "DBIx::Class::UUIDColumns" => 0, "DBIx::QuickDB" => "0.000020", "Data::Dumper" => 0, - "Data::GUID" => 0, - "Data::UUID" => 0, "DateTime" => 0, "DateTime::Format::MySQL" => 0, "DateTime::Format::Pg" => 0, + "DateTime::Format::SQLite" => 0, "Email::Sender::Simple" => 0, "Email::Simple" => 0, "Email::Simple::Creator" => 0, @@ -234,7 +233,7 @@ my %FallbackPrereqs = ( "Test2::Formatter" => "1.302198", "Test2::Plugin::MemUsage" => "0.002003", "Test2::Plugin::NoWarnings" => 0, - "Test2::Plugin::UUID" => "0.002001", + "Test2::Plugin::UUID" => "0.002008", "Test2::Tools::AsyncSubtest" => "0.000159", "Test2::Tools::Basic" => 0, "Test2::Tools::Compare" => 0, @@ -249,6 +248,7 @@ my %FallbackPrereqs = ( "Test2::V0" => "0.000159", "Test::Builder" => "1.302198", "Test::Builder::Formatter" => "1.302198", + "Test::Harness" => "3.49", "Test::More" => "1.302198", "Text::ParseWords" => 0, "Text::Xslate" => 0, diff --git a/cpanfile b/cpanfile index d9d6a7893..cc06208c8 100644 --- a/cpanfile +++ b/cpanfile @@ -18,11 +18,10 @@ requires "DBIx::Class::Tree::AdjacencyList" => "0"; requires "DBIx::Class::UUIDColumns" => "0"; requires "DBIx::QuickDB" => "0.000020"; requires "Data::Dumper" => "0"; -requires "Data::GUID" => "0"; -requires "Data::UUID" => "0"; requires "DateTime" => "0"; requires "DateTime::Format::MySQL" => "0"; requires "DateTime::Format::Pg" => "0"; +requires "DateTime::Format::SQLite" => "0"; requires "Email::Sender::Simple" => "0"; requires "Email::Simple" => "0"; requires "Email::Simple::Creator" => "0"; @@ -73,7 +72,7 @@ requires "Test2::Event" => "1.302198"; requires "Test2::Event::V2" => "1.302198"; requires "Test2::Formatter" => "1.302198"; requires "Test2::Plugin::MemUsage" => "0.002003"; -requires "Test2::Plugin::UUID" => "0.002001"; +requires "Test2::Plugin::UUID" => "0.002008"; requires "Test2::Tools::AsyncSubtest" => "0.000159"; requires "Test2::Tools::Basic" => "0"; requires "Test2::Tools::Compare" => "0"; @@ -87,6 +86,7 @@ requires "Test2::Util::Times" => "0"; requires "Test2::V0" => "0.000159"; requires "Test::Builder" => "1.302198"; requires "Test::Builder::Formatter" => "1.302198"; +requires "Test::Harness" => "3.49"; requires "Test::More" => "1.302198"; requires "Text::ParseWords" => "0"; requires "Text::Xslate" => "0"; @@ -103,7 +103,10 @@ suggests "Class::XSAccessor" => "1.19"; suggests "Cpanel::JSON::XS" => "0"; suggests "DBD::Pg" => "0"; suggests "DBD::mysql" => "0"; +suggests "DBIx::Class::Storage::DBI::MariaDB" => "0"; suggests "DBIx::Class::Storage::DBI::mysql::Retryable" => "0"; +suggests "Data::UUID" => "1.227"; +suggests "Data::UUID::MT" => "1.001"; suggests "Email::Stuffer" => "0.016"; suggests "HTTP::Tiny" => "0.070"; suggests "HTTP::Tiny::Multipart" => "0.08"; @@ -114,6 +117,8 @@ suggests "Term::ANSIColor" => "4.06"; suggests "Test2::Plugin::Cover" => "0.000025"; suggests "Test2::Plugin::DBIProfile" => "0.002002"; suggests "Test2::Plugin::IOEvents" => "0.001001"; +suggests "UUID" => "0.35"; +suggests "UUID::Tiny" => "1.04"; suggests "Win32::Console::ANSI" => "0"; on 'test' => sub { diff --git a/dist.ini b/dist.ini index 464beeea8..4952ee375 100644 --- a/dist.ini +++ b/dist.ini @@ -77,11 +77,10 @@ DBI = 0 DBIx::Class::UUIDColumns = 0 DBIx::QuickDB = 0.000020 Data::Dumper = 0 -Data::GUID = 0 -Data::UUID = 0 DateTime = 0 DateTime::Format::MySQL = 0 DateTime::Format::Pg = 0 +DateTime::Format::SQLite = 0 Email::Sender::Simple = 0 Email::Simple = 0 Email::Simple::Creator = 0 @@ -131,7 +130,7 @@ Test2::Event = 1.302198 Test2::Event::V2 = 1.302198 Test2::Formatter = 1.302198 Test2::Plugin::MemUsage = 0.002003 -Test2::Plugin::UUID = 0.002001 +Test2::Plugin::UUID = 0.002008 Test2::Tools::AsyncSubtest = 0.000159 Test2::Tools::Basic = 0 Test2::Tools::Compare = 0 @@ -200,7 +199,12 @@ System::Info = 0.064 Cpanel::JSON::XS = 0 DBD::Pg = 0 DBD::mysql = 0 +UUID = 0.35 +Data::UUID = 1.227 +UUID::Tiny = 1.04 +Data::UUID::MT = 1.001 +DBIx::Class::Storage::DBI::MariaDB = 0 DBIx::Class::Storage::DBI::mysql::Retryable = 0 [MakeMaker::Awesome] diff --git a/lib/App/Yath.pm b/lib/App/Yath.pm index 640da3abe..0a4ef2a09 100644 --- a/lib/App/Yath.pm +++ b/lib/App/Yath.pm @@ -104,29 +104,40 @@ sub cli_help { my $opts = $options->docs('cli', groups => {':{' => '}:'}, group => $params{group}, settings => $settings, color => $self->use_color); my $script = File::Spec->abs2rel($settings->yath->script // $0); - my $usage = ''; - my $append = ''; - if ($self->use_color) { - $usage = join ' ' => ( - color('bold white') . "USAGE:" . color('reset'), - color('white') . $script, - color('cyan') . "[YATH OPTIONS]", - color('bold green') . $cmd . color('reset'), - color('cyan') . "[OPTIONS FOR COMMAND AND/OR YATH]", - color('yellow') . "[--]", - ); - $append = ($cmd_class && $cmd_class->args_include_tests) ? ' ' . join " " => ( - color('white') . "[ARGUMENTS/TESTS]", - color('green') . "[TEST :{ ARGS TO PASS TO TEST }:]", - color('magenta') . "[:: PASS-THROUGH]" . color('reset') - ) : color('white') . " [ARGUMENTS]"; - } - else { - $usage = "USAGE: $script [YATH OPTIONS] $cmd [OPTIONS FOR COMMAND AND/OR YATH] [--]"; - $append = ($cmd_class && $cmd_class->args_include_tests) ? " [ARGUMENTS/TESTS] [TEST :{ ARGS TO PASS TO TEST }:] [:: PASS-THROUGH]" : " [ARGUMENTS]"; + my $colors = {reset => ''}; + if ($self->use_color) { + $colors = { + reset => color('reset'), + usage => color('bold white'), + script => color('white'), + yath_opts => color('cyan'), + command => color('bold green'), + cmd_opts => color('cyan'), + '--a' => color('yellow'), + '--b' => color('yellow'), + arguments => color('white'), + tests => color('green'), + dot_args => color('magenta'), + }; } + my $parts = { + usage => "USAGE:", + script => $script, + yath_opts => "[YATH OPTIONS]", + command => $cmd, + cmd_opts => "[OPTIONS FOR COMMAND AND/OR YATH]", + ($cmd_class->cli_args || $cmd_class->args_include_tests) ? ('--a' => "[[--]", '--b' => ']') : (), + $cmd_class->args_include_tests ? (tests => "[TEST :{ ARGS TO PASS TO TEST }:]") : (), + $cmd_class->cli_args ? (arguments => $cmd_class->cli_args) : (), + $cmd_class->accepts_dot_args ? (dot_args => $cmd_class->cli_dot || "[:: PASS-THROUGH]") : (), + }; + + my $usage = join " " => map { ($colors->{$_} || '') . $parts->{$_} . $colors->{reset} } grep { $parts->{$_} } qw/usage script yath_opts command cmd_opts --a tests arguments/; + $usage .= ($colors->{'--b'} || '') . $parts->{'--b'} . $colors->{'reset'} if $parts->{'--b'}; + $usage .= " " . ($colors->{'dot_args'} || '') . $parts->{'dot_args'} . $colors->{'reset'} if $parts->{'dot_args'}; + my $end = ""; if ($settings->yath->help && !$params{group}) { $end = $self->_render_groups( @@ -135,7 +146,7 @@ sub cli_help { ); } - return "${usage}${append}\n${help}\n${opts}\n${end}"; + return "${usage}\n${help}\n${opts}\n${end}"; } sub _strip_color { @@ -143,8 +154,8 @@ sub _strip_color { my ($colors, $line) = @_; return $line unless $self->use_color; - my $pattern = join '|' => map { "\Q$_\E"} values %$colors; - $line =~ s/($pattern)//g; + my $pattern = join '|' => map { "\Q$_\E"} grep { $_ } values %$colors; + $line =~ s/($pattern)//g if $pattern; return $line; } @@ -345,6 +356,8 @@ sub load_command { $self->include_options('plugins' => 'App::Yath::Plugin::*') if $cmd_class->load_plugins(); $self->include_options('resource' => 'App::Yath::Resource::*') if $cmd_class->load_resources(); $self->include_options('renderer' => 'App::Yath::Renderer::*') if $cmd_class->load_renderers(); + + return $cmd_class; } sub process_args { @@ -365,7 +378,7 @@ sub process_args { my $state = $self->_process_global_args($argv); - my $cmd; + my ($cmd, $cmd_class); if (my $stop = $state->{stop}) { my @cmd_args; @@ -410,25 +423,26 @@ sub process_args { @{$state->{remains} // []}, ); - $self->load_command($cmd) if $cmd; + $cmd_class = $self->load_command($cmd) if $cmd; $state = $self->_process_command_args(\@cmd_args, cmd => $cmd); } $cmd //= 'do'; + $cmd_class //= 'App::Yath::Command::do'; - my $test_args; + my $dot_args; $argv = [@{$state->{skipped}}]; if (my $stop = $state->{stop}) { if ($stop eq '--') { for my $arg (@{$state->{remains}}) { - if ($test_args) { push @$test_args => $arg } - elsif ($arg eq '::') { $test_args //= [] } + if ($dot_args) { push @$dot_args => $arg } + elsif ($arg eq '::') { $dot_args //= [] } else { push @$argv => $arg } } } elsif ($stop eq '::') { - $test_args = [@{$state->{remains}}]; + push @{$dot_args //= []} => @{$state->{remains}}; } else { push @$argv => ($stop, @{$state->{remains}}); @@ -438,9 +452,9 @@ sub process_args { push @$argv => @{$state->{remains}}; } - if ($test_args) { - die "'::' cannot be used with the '$cmd' command" unless $settings->check_group('tests'); - $settings->tests->option(args => $test_args); + if ($dot_args) { + die "'::' cannot be used with the '$cmd' command" unless $cmd_class->accepts_dot_args; + $cmd_class->set_dot_args($settings, $dot_args); } $self->{argv} = $argv; diff --git a/lib/App/Yath/Command.pm b/lib/App/Yath/Command.pm index 82a9b2e63..5d44654ab 100644 --- a/lib/App/Yath/Command.pm +++ b/lib/App/Yath/Command.pm @@ -10,13 +10,15 @@ use Test2::Harness::Util qw/mod2file/; use Test2::Harness::Util::HashBase qw/get_from_http($project, $count, $user) // die "Could not get data from the server.\n"; +} + +1; + +__END__ + +=head1 POD IS AUTO-GENERATED diff --git a/lib/App/Yath/Command/db.pm b/lib/App/Yath/Command/db.pm new file mode 100644 index 000000000..400137267 --- /dev/null +++ b/lib/App/Yath/Command/db.pm @@ -0,0 +1,581 @@ +package App::Yath::Command::db; +use strict; +use warnings; + +use App::Yath::Server; +use App::Yath::Schema::Util qw/schema_config_from_settings/; + +use parent 'App::Yath::Command'; +use Test2::Harness::Util::HashBase; + +sub summary { "Start a yath database server" } +sub description { "Starts a database that can be used to temporarily store data (data is deleted when server shuts down)" } +sub group { "db" } + +sub cli_args { "" } + +use Getopt::Yath; +include_options( + 'App::Yath::Options::Yath', + 'App::Yath::Options::DB', + 'App::Yath::Options::Server', +); + +sub run { + my $self = shift; + + my $args = $self->args; + my $settings = $self->settings; + + my $daemon = $settings->server->daemon; + + if ($daemon) { + my $pid = fork // die "Could not fork"; + exit(0) if $pid; + + POSIX::setsid(); + setpgrp(0, 0); + + $pid = fork // die "Could not fork"; + exit(0) if $pid; + } + + my $ephemeral = $settings->server->ephemeral; + unless($ephemeral) { + $ephemeral = 'Auto'; + $settings->server->ephemeral($ephemeral); + } + + my $config = schema_config_from_settings($settings, ephemeral => $ephemeral); + + my $qdb_params = { + single_user => $settings->server->single_user // 0, + single_run => $settings->server->single_run // 0, + no_upload => $settings->server->no_upload // 0, + email => $settings->server->email // undef, + }; + + my $server = App::Yath::Server->new(schema_config => $config, qdb_params => $qdb_params); + $server->start_ephemeral_db; + + my $done = 0; + $SIG{TERM} = sub { $done++; print "Caught SIGTERM shutting down...\n" unless $daemon; $SIG{TERM} = 'DEFAULT' }; + $SIG{INT} = sub { $done++; print "Caught SIGINT shutting down...\n" unless $daemon; $SIG{INT} = 'DEFAULT' }; + + if ($settings->server->shell) { + system($ENV{SHELL}); + } + else { + $server->qdb->watcher->detach if $daemon; + sleep 1 until $done; + } + + $server->stop_ephemeral_db if $server->qdb; + + return 0; +} + + + + +# TODO: +# start an ephemeral yath database optionally go to shell + +1; + +__END__ + + +use feature 'state'; + + +use App::Yath::Schema::Util qw/schema_config_from_settings/; +use Test2::Util::UUID qw/gen_uuid/; + +use Test2::Harness::Util qw/clean_path/; + +our $VERSION = '2.000000'; + +use parent 'App::Yath::Command'; +use Test2::Harness::Util::HashBase qw{ + webserver->launcher_args} => @$dot_args; + return; +} + +use Getopt::Yath; +include_options( + 'App::Yath::Options::Term', + 'App::Yath::Options::Yath', + 'App::Yath::Options::DB', + 'App::Yath::Options::WebServer', +); + +option_group {group => 'server', category => "Server Options"} => sub { + option ephemeral => ( + type => 'Auto', + autofill => 'Auto', + long_examples => ['', '=Auto', '=PostgreSQL', '=MySQL', '=MariaDB', '=SQLite', '=Percona' ], + description => "Use a temporary 'ephemeral' database that will be destroyed when the server exits.", + autofill_text => 'If no db type is specified it will use "auto" which will try PostgreSQL first, then MySQL.', + allowed_values => [qw/Auto PostgreSQL MySQL MariaDB Percona SQLite/], + ); + + option shell => ( + type => 'Bool', + default => 0, + description => "Drop into a shell where the server and database env vars are set so that yath commands will use the started server.", + ); + + option daemon => ( + type => 'Bool', + default => 0, + description => "Run the server in the background.", + ); + + option dev => ( + type => 'Bool', + default => 0, + description => 'Launches in "developer mode" which accepts some developer commands while the server is running.', + ); + + option single_user => ( + type => 'Bool', + default => 0, + description => "When using an ephemeral database you can use this to enable single user mode to avoid login and user credentials.", + ); + + option single_run => ( + type => 'Bool', + default => 0, + description => "When using an ephemeral database you can use this to enable single run mode which causes the server to take you directly to the first run.", + ); + + option no_upload => ( + type => 'Bool', + default => 0, + description => "When using an ephemeral database you can use this to enable no-upload mode which removes the upload workflow.", + ); + + option email => ( + type => 'Scalar', + description => "When using an ephemeral database you can use this to set a 'from' email address for email sent from this server.", + ); +}; + + +sub run { + my $self = shift; + my $pid = $$; + + my $args = $self->args; + my $settings = $self->settings; + + my $ephemeral = $settings->server->ephemeral; + + my $config = $self->{+CONFIG} = schema_config_from_settings($settings, ephemeral => $ephemeral); + + my $qdb_params = { + single_user => $settings->server->single_user // 0, + single_run => $settings->server->single_run // 0, + no_upload => $settings->server->no_upload // 0, + email => $settings->server->email // undef, + }; + + my $server = $self->{+SERVER} = App::Yath::Server->new(schema_config => $config, $settings->webserver->all, qdb_params => $qdb_params); + $server->start_server; + + my $done = 0; + $SIG{TERM} = sub { $done++; print "Caught SIGTERM shutting down...\n"; $SIG{TERM} = 'DEFAULT' }; + $SIG{INT} = sub { $done++; print "Caught SIGINT shutting down...\n"; $SIG{INT} = 'DEFAULT' }; + + for my $log (@{$args // []}) { + $self->load_file($config, $log); + } + + SERVER_LOOP: until ($done) { + if ($settings->server->dev) { + $ENV{T2_HARNESS_SERVER_DEV} = 1; + + unless(eval { $done = $self->shell($pid); 1 }) { + warn $@; + $done = 1; + } + } + else { + sleep 1; + } + } + + if ($pid == $$) { + $server->stop_server if $server->pid; + } + else { + die "Scope leak, wrong PID"; + } + + return 0; +} + + +sub load_file { + my $self = shift; + my ($config, $file) = @_; + + $file = clean_path($file); + + state %projects; + + my $project; + if ($file =~ m/moose/i) { + $project = 'Moose'; + } + else { + $project = $1 if $file =~ m/\b([\w\d]+)\./; + } + + $project //= "oops"; + + unless ($projects{$project}) { + my $p = $config->schema->resultset('Project')->find_or_create({name => $project}); + $projects{$project} = $p; + } + + my $logfile = $config->schema->resultset('LogFile')->create({ + name => $file, + local_file => $file =~ m{^/} ? $file : "./demo/$file", + }); + + state $user = $config->schema->resultset('User')->find_or_create({username => 'root', password => 'root', realname => 'root'}); + + my $run = $config->schema->resultset('Run')->create({ + run_id => gen_uuid(), + user_id => $user->user_id, + mode => 'complete', + buffer => 'job', + status => 'pending', + project_id => $projects{$project}->project_id, + + log_file_id => $logfile->log_file_id, + }); + + return $run; +} + +sub shell { + my $self = shift; + my ($pid, $doneref) = @_; + + # Return that we should exit if the PID is wrong. + return 1 unless $pid == $$; + + my $settings = $self->settings; + my $server = $self->{+SERVER}; + my $config = $self->{+CONFIG}; + + $SIG{TERM} = sub { $SIG{TERM} = 'DEFAULT'; die "Cought SIGTERM exiting...\n" }; + $SIG{INT} = sub { $SIG{INT} = 'DEFAULT'; die "Cought SIGINT exiting...\n" }; + + STDERR->autoflush(); + sleep 1; + + my $dsn = $config->dbi_dsn; + + print "DBI_DSN: $dsn\n\n"; + print "\n"; + print "| Yath Server Developer Shell |\n"; + print "| type 'help', 'h', or '?' for help |\n"; + + while(1) { + print "\n> "; + + my $in = ; + return 1 if !defined($in) && eof(STDIN); + chomp($in); + next unless length($in); + + return 1 if $in =~ m/^(q|x|exit|quit)$/; + + if ($in =~ m/^(help|h|\?)(?:\s(.+))?$/) { + $self->shell_help($1); + next; + } + + my ($cmd, $args) = split /\s/, $in, 2; + + my $meth = "shell_$cmd"; + if ($self->can($meth)) { + eval { $self->$meth($args); 1 } or warn $@; + } + else { + print STDERR "Invalid command '$in'\n"; + } + } +} + +sub shell_help_text { "Show command list." } +sub shell_help { + my $self = shift; + my $class = ref($self); + my $stash = do { no strict 'refs'; \%{"$class\::"} }; + + print "\nAvailable commands:\n"; + printf(" %-12s %s\n", "[q]uit", "Quit the program."); + printf(" %-12s %s\n", "e[x]it", "Exit the program."); + printf(" %-12s %s\n", "[h]elp", "Show this help."); + printf(" %-12s %s\n", "?", "Show this help."); + + for my $sym (sort keys %$stash) { + next unless $sym =~ m/^shell_(.*)/; + my $cmd = $1; + next if $cmd eq 'help'; + next if $sym =~ m/_text$/; + next unless $self->can($sym); + + my $text = "${sym}_text"; + $text = $self->can($text) ? $self->$text() : 'No description.'; + printf(" %-12s %s\n", $cmd, $text); + } + print "\n"; +} + +sub shell_reload_text { "Restart web server (does not restart database or importers)." } +sub shell_reload { $_[0]->server->restart_server } + +sub shell_reloaddb_text { "Restart database (data is lost)." } +sub shell_reloaddb { + my $self = shift; + + my $server = $self->server; + $server->stop_server; + $server->stop_importers; + $server->reset_ephemeral_db; + $server->start_server; +} + +sub shell_reloadimp_text { "Restart the importers." } +sub shell_reloadimp { $_[0]->restart_importers() } + +sub shell_db_text { "Open the database." } +sub shell_db { $_[0]->server->qdb->shell } + +sub shell_load_text { "Load a database file (filename given as argument)" } +sub shell_load { die "TODO: fix me" } + +{ + no warnings 'once'; + *shell_r = \*shell_reload; + *shell_r_text = \*shell_reload_text; + *shell_rdb = \*shell_reloaddb; + *shell_rdb_text = \*shell_reloaddb_text; + *shell_ri = \*shell_reloadimp; + *shell_ri_text = \*shell_reloadimp_text; +} + +1; + +__END__ + +use Test2::Util qw/pkg_to_file/; + +use App::Yath::Server::Util qw/share_dir share_file dbd_driver qdb_driver/; + +use App::Yath::Server::Config; +use App::Yath::Schema::Importer; +use App::Yath::Server; + +use Test2::Util::UUID qw/gen_uuid/; + +use DBIx::QuickDB; +use Plack::Builder; +use Plack::App::Directory; +use Plack::App::File; +use Plack::Runner; + +use parent 'App::Yath::Command'; +use Test2::Harness::Util::HashBase qw/ 'ui', group => 'ui', category => "UI Options"} => sub { + option schema => ( + type => 'Scalar', + default => 'PostgreSQL', + long_examples => [' PostgreSQL', ' MySQL', ' MySQL56'], + description => "What type of DB/schema to use", + ); + + option port => ( + type => 'Scalar', + long_examples => [' 8080'], + description => 'Port to use', + ); + + option port_command => ( + type => 'Scalar', + long_examples => [' get_port.sh', ' get_port.sh --pid $$'], + description => 'Use a command to get a port number. "$$" will be replaced with the PID of the yath process', + ); +}; + +sub summary { "Launch a standalone Test2-Harness-UI server for a log file" } + +sub group { 'ui' } + +sub cli_args { "[--] event_log.jsonl[.gz|.bz2]" } + +sub description { + return <<" EOT"; + EOT +} + +sub run { + my $self = shift; + + my $args = $self->args; + my $settings = $self->settings; + + my $schema = $settings->ui->schema; + require(pkg_to_file("App::Yath::Server::Schema::$schema")); + + shift @$args if @$args && $args->[0] eq '--'; + + $self->{+LOG_FILE} = shift @$args or die "You must specify a log file"; + die "'$self->{+LOG_FILE}' is not a valid log file" unless -f $self->{+LOG_FILE}; + die "'$self->{+LOG_FILE}' does not look like a log file" unless $self->{+LOG_FILE} =~ m/\.jsonl(\.(gz|bz2))?$/; + + my $db = DBIx::QuickDB->build_db(harness_ui => {driver => qdb_driver($schema), dbd_driver => dbd_driver($schema)}); + + my $dbh = $db->connect('quickdb', AutoCommit => 1, RaiseError => 1); + $dbh->do('CREATE DATABASE harness_ui') or die "Could not create db " . $dbh->errstr; + $db->load_sql(harness_ui => share_file("schema/$schema.sql")); + my $dsn = $db->connect_string('harness_ui'); + $dbh = undef; + + $ENV{HARNESS_UI_DSN} = $dsn; + + print "DSN: $dsn\n"; + my $config = App::Yath::Server::Config->new( + dbi_dsn => $dsn, + dbi_user => '', + dbi_pass => '', + single_user => 1, + single_run => 1, + ); + + my $user = $config->schema->resultset('User')->create({username => 'root', password => 'root', realname => 'root', user_idx => gen_uuid()}); + my $proj = $config->schema->resultset('Project')->create({name => 'default', project_idx => gen_uuid()}); + + $config->schema->resultset('Run')->create({ + run_id => gen_uuid(), + user_idx => $user->user_idx, + mode => 'complete', + status => 'pending', + project_idx => $proj->project_idx, + + log_file => { + log_file_idx => gen_uuid(), + name => $self->{+LOG_FILE}, + local_file => $self->{+LOG_FILE}, + }, + }); + + App::Yath::Schema::Importer->new(config => $config)->run(1); + + my $app = builder { + mount '/js' => Plack::App::Directory->new({root => share_dir('js')})->to_app; + mount '/css' => Plack::App::Directory->new({root => share_dir('css')})->to_app; + mount '/favicon.ico' => Plack::App::File->new({file => share_dir('img') . '/favicon.ico'})->to_app; + mount '/img' => Plack::App::Directory->new({root => share_dir('img')})->to_app; + + mount '/' => sub { + App::Yath::Server->new(config => $config)->to_app->(@_); + }; + }; + + my $port = $settings->ui->port; + if (my $cmd = $settings->ui->port_command) { + $cmd =~ s/\$\$/$$/; + chomp($port = `$cmd`); + } + + my $r = Plack::Runner->new; + my @options = ("--server", "Starman"); + + push @options => ('--listen' => ":$port") if $port; + + $r->parse_options(@options); + $r->run($app); + + return 0; +} + +1; + +__END__ + +=pod + +=encoding UTF-8 + +=head1 NAME + +App::Yath::Command::ui - FIXME + +=head1 DESCRIPTION + +=head1 SYNOPSIS + +=head1 EXPORTS + +=over 4 + +=back + +=head1 SOURCE + +The source code repository for Test2-Harness can be found at +L. + +=head1 MAINTAINERS + +=over 4 + +=item Chad Granum Eexodist@cpan.orgE + +=back + +=head1 AUTHORS + +=over 4 + +=item Chad Granum Eexodist@cpan.orgE + +=back + +=head1 COPYRIGHT + +Copyright Chad Granum Eexodist7@gmail.comE. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +See L + +=cut + diff --git a/lib/App/Yath/Command/db/importer.pm b/lib/App/Yath/Command/db/importer.pm index 7af9f2d19..22be1fc84 100644 --- a/lib/App/Yath/Command/db/importer.pm +++ b/lib/App/Yath/Command/db/importer.pm @@ -25,7 +25,7 @@ sub run { my $settings = $self->settings; my $config = schema_config_from_settings($settings); - $SIG{INT} = sub { exit 0 }; + $SIG{INT} = sub { exit 0 }; $SIG{TERM} = sub { exit 0 }; App::Yath::Schema::Importer->new(config => $config)->run; diff --git a/lib/App/Yath/Command/db/loader.pm b/lib/App/Yath/Command/db/loader.pm index 230f9277b..6fa57d386 100644 --- a/lib/App/Yath/Command/db/loader.pm +++ b/lib/App/Yath/Command/db/loader.pm @@ -30,7 +30,7 @@ option_group {group => 'loader', category => "Loader Options"} => sub { type => 'Scalar', short => 'j', alt => ['procs'], - description => 'Set the number of processes to use to dump the database', + description => 'Set the number of processes to use to load the database', notes => "If System::Info is installed, this will default to the cpu core count, otherwise the default is 1.", long_examples => [' 5'], short_examples => ['5'], diff --git a/lib/App/Yath/Command/db/publish.pm b/lib/App/Yath/Command/db/publish.pm index ece4eadae..80c87914b 100644 --- a/lib/App/Yath/Command/db/publish.pm +++ b/lib/App/Yath/Command/db/publish.pm @@ -4,10 +4,12 @@ use warnings; our $VERSION = '2.000000'; +use Time::HiRes qw/time/; + use IO::Uncompress::Bunzip2 qw($Bunzip2Error); use IO::Uncompress::Gunzip qw($GunzipError); -use App::Yath::Schema::Util qw/schema_config_from_settings/; +use App::Yath::Schema::Util qw/schema_config_from_settings format_duration/; use Test2::Harness::Util::JSON qw/decode_json/; use App::Yath::Schema::RunProcessor; @@ -18,7 +20,7 @@ use Test2::Harness::Util::HashBase; use Getopt::Yath; include_options( 'App::Yath::Options::DB', - 'App::Yath::Options::Upload', + 'App::Yath::Options::Publish', ); sub summary { "Publish a log file directly to a yath database" } @@ -41,48 +43,98 @@ sub run { die "'$file' is not a valid log file" unless -f $file; die "'$file' does not look like a log file" unless $file =~ m/\.jsonl(\.(gz|bz2))?$/; + my $lines = 0; my $fh; if ($file =~ m/\.bz2$/) { $fh = IO::Uncompress::Bunzip2->new($file) or die "Could not open bz2 file: $Bunzip2Error"; + $lines++ while <$fh>; + $fh = IO::Uncompress::Bunzip2->new($file) or die "Could not open bz2 file: $Bunzip2Error"; } elsif ($file =~ m/\.gz$/) { $fh = IO::Uncompress::Gunzip->new($file) or die "Could not open gz file: $GunzipError"; + $lines++ while <$fh>; + $fh = IO::Uncompress::Gunzip->new($file) or die "Could not open gz file: $GunzipError"; } else { open($fh, '<', $file) or die "Could not open log file: $!"; + $lines++ while <$fh>; + seek($fh, 0, 0); } - my $config = schema_config_from_settings($settings); - - my $ydb = $settings->db; - my $yup = $settings->upload; - my $user = $yup->user || $ENV{USER}; + my $user = $settings->yath->user; my $is_term = -t STDOUT ? 1 : 0; print "\n" if $is_term; - my $cb = App::Yath::Schema::RunProcessor->process_lines($settings); + my $project = $file; + $project =~ s{^.*/}{}g; + $project =~ s{\.jsonl.*$}{}g; + $project =~ s/-\d.*$//g; + $project =~ s/^\s+//g; + $project =~ s/\s+$//g; + + my $start = time; + + my $cb = App::Yath::Schema::RunProcessor->process_lines($settings, project => $project, print_links => 1); + + my $run; + eval { + my $ln = <$fh>; + $run = $cb->($ln); + 1 + } or return $self->fail($@); + + $SIG{INT} = sub { + print STDERR "\nCought SIGINT...\n"; + eval { $run->update({status => 'canceled', error => "SIGINT while importing"}); 1 } or warn $@; + exit 255; + }; + + $SIG{TERM} = sub { + print STDERR "\nCought SIGTERM...\n"; + eval { $run->update({status => 'canceled', error => "SIGTERM while importing"}); 1 } or warn $@; + exit 255; + }; + + my $len = length("" . $lines); local $| = 1; while (my $line = <$fh>) { my $ln = $.; - print "\033[Fprocessing log line: $ln\n" + printf("\033[Fprocessing '%s' line: % ${len}d / %d\n", $file, $ln, $lines) if $is_term; next if $line =~ m/^null$/ims; - $cb->($line); + eval { $cb->($line); 1 } or return $self->fail($@, $run); } $cb->(); - print "Upload Complete\n"; + my $end = time; + + my $dur = format_duration($end - $start); + + print "Published Run. [Status: " . $run->status . ", Duration: $dur]\n"; return 0; } +sub fail { + print STDERR "FAIL!\n\n"; + my $self = shift; + my ($err, $run) = @_; + + $run->update({status => 'broken', error => $err}) if $run; + + print STDERR "\n$err\n"; + + print STDERR "\nPublish Failed.\n"; + return 255; +} + 1; __END__ diff --git a/lib/App/Yath/Command/db/recent.pm b/lib/App/Yath/Command/db/recent.pm new file mode 100644 index 000000000..76a8f1053 --- /dev/null +++ b/lib/App/Yath/Command/db/recent.pm @@ -0,0 +1,29 @@ +package App::Yath::Command::db::recent; +use strict; +use warnings; + +our $VERSION = '2.000000'; + +use parent 'App::Yath::Command::recent'; +use Test2::Harness::Util::HashBase; + +sub summary { "Show a list of recent runs in the database" } + +sub description { + return <<" EOT"; +This command will find the last several runs from a yath database. + EOT +} + +sub get_data { + my $self = shift; + my ($project, $count, $user) = @_; + + return $self->get_from_db($project, $count, $user) // die "Could not get data from the database.\n"; +} + +1; + +__END__ + +=head1 POD IS AUTO-GENERATED diff --git a/lib/App/Yath/Command/do.pm b/lib/App/Yath/Command/do.pm index 4987f4c82..79546be50 100644 --- a/lib/App/Yath/Command/do.pm +++ b/lib/App/Yath/Command/do.pm @@ -4,7 +4,7 @@ use warnings; our $VERSION = '2.000000'; -use parent 'App::Yath::Command'; +use parent 'App::Yath::Command::test'; use Test2::Harness::Util::HashBase; sub group { ' main' } diff --git a/lib/App/Yath/Command/server/recent.pm b/lib/App/Yath/Command/recent.pm similarity index 53% rename from lib/App/Yath/Command/server/recent.pm rename to lib/App/Yath/Command/recent.pm index 05c9799ed..e6363b513 100644 --- a/lib/App/Yath/Command/server/recent.pm +++ b/lib/App/Yath/Command/recent.pm @@ -5,50 +5,40 @@ use warnings; our $VERSION = '2.000000'; use Term::Table; -use App::Yath::Server::Util qw/config_from_settings/; use Test2::Harness::Util::JSON qw/decode_json/; +use App::Yath::Schema::Util qw/schema_config_from_settings/; use parent 'App::Yath::Command'; use Test2::Harness::Util::HashBase; use Getopt::Yath; -option_group {group => 'yathui', prefix => 'yathui', category => "List Options"} => sub { - option max => ( - type => 'Scalar', - long_examples => [' 10'], - default => 10, - description => 'Max number of recent runs to show', - ); -}; +include_options( + 'App::Yath::Options::Yath', + 'App::Yath::Options::Recent', + 'App::Yath::Options::WebClient', + 'App::Yath::Options::DB', +); -sub summary { "Show a list of recent YathUI runs" } +sub summary { "Show a list of recent runs on a yathui server" } -sub group { 'log' } +sub group { 'recent' } sub cli_args { "" } sub description { return <<" EOT"; -This command will find the last several runs from yathui or the yathui -database. - -This command gets the username from the "--yathui-user USER" option (Defaults to \$ENV{USER}). -This command gets the project from the "--yathui-project PROJECT" option. -This command uses the "--max NUM" option to set how many runs to show (Defaults to 10). - +This command will find the last several runs from a yath server EOT } sub run { my $self = shift; - my $args = $self->args; my $settings = $self->settings; - my $yui = $settings->yathui; - my $ydb = $self->settings->prefix('yathui-db'); - my $config = config_from_settings($settings); + my $yath = $settings->yath; + my $recent = $settings->recent; my ($is_term, $use_color) = (0, 0); if (-t STDOUT) { @@ -57,19 +47,15 @@ sub run { $use_color = eval { require Term::ANSIColor; 1 }; } - my $project = $yui->project // die "--yathui-project is a required argument.\n"; - my $user = $yui->user // $ENV{USER}; - my $count = $yui->max || 10; - - print "\nProject: $project\n User: $user\n Count: $count\n\n"; + my $project = $yath->project // die "--project is a required argument.\n"; + my $count = $recent->max || 10; + my $user = $settings->yath->user; - my $data = $config->dbi_dsn - ? $self->get_from_db($settings, $config, $project, $user, $count) - : $self->get_from_http($settings, $yui, $project, $user, $count); + my $data = $self->get_data($project, $count, $user) or die "Could not get any data.\n"; @$data = reverse @$data; - my $url = $yui->url; + my $url = $settings->server->url; $url =~ s{/$}{}g if $url; my $header = [qw/Time Duration Status Pass Fail Retry/, "Run ID"]; @@ -98,7 +84,6 @@ sub run { } } - my $table = Term::Table->new( header => $header, rows => $rows, @@ -109,29 +94,23 @@ sub run { return 0; } -sub get_from_http { +sub get_data { my $self = shift; - my ($settings, $yui, $project, $user, $count) = @_; - - require HTTP::Tiny; - my $ht = HTTP::Tiny->new(); - my $url = $yui->url; - $url =~ s{/$}{}g; - $url .= "/recent/$project/$user"; - my $res = $ht->get($url); - - die "Could not get recent runs from '$url'\n$res->{status}: $res->{reason}\n$res->{content}\n" - unless $res->{success}; + my ($project, $count, $user) = @_; - return decode_json($res->{content}); + return $self->get_from_db($project, $count, $user) + || $self->get_from_http($project, $count, $user); } sub get_from_db { my $self = shift; - my ($settings, $config, $project, $user, $count) = @_; + my ($project, $count, $user) = @_; + + my $settings = $self->settings; + my $config = schema_config_from_settings($settings) or return undef; + my $schema = $config->schema or return undef; - my $schema = $config->schema; - my $runs = $schema->vague_run_search( + my $runs = $schema->vague_run_search( username => $user, project_name => $project, query => {}, @@ -145,62 +124,34 @@ sub get_from_db { push @$data => $run->TO_JSON; } + return undef unless @$data; + return $data; } +sub get_from_http { + my $self = shift; + my ($project, $count, $user) = @_; + my $settings = $self->settings; + my $server = $settings->server; -1; - -__END__ - -=pod - -=encoding UTF-8 - -=head1 NAME - -App::Yath::Command::recent - FIXME - -=head1 DESCRIPTION - -=head1 SYNOPSIS - -=head1 EXPORTS - -=over 4 - -=back - -=head1 SOURCE - -The source code repository for Test2-Harness can be found at -L. - -=head1 MAINTAINERS - -=over 4 - -=item Chad Granum Eexodist@cpan.orgE - -=back - -=head1 AUTHORS - -=over 4 - -=item Chad Granum Eexodist@cpan.orgE - -=back + require HTTP::Tiny; + my $ht = HTTP::Tiny->new(); + my $url = $server->url or return; + $url =~ s{/$}{}g; + $url .= "/recent/$project/$user"; + my $res = $ht->get($url); -=head1 COPYRIGHT + die "Could not get recent runs from '$url'\n$res->{status}: $res->{reason}\n$res->{content}\n" + unless $res->{success}; -Copyright Chad Granum Eexodist7@gmail.comE. + return decode_json($res->{content}); +} -This program is free software; you can redistribute it and/or -modify it under the same terms as Perl itself. +1; -See L +__END__ -=cut +=head1 POD IS AUTO-GENERATED diff --git a/lib/App/Yath/Command/run.pm b/lib/App/Yath/Command/run.pm index a44b31210..7e1b95372 100644 --- a/lib/App/Yath/Command/run.pm +++ b/lib/App/Yath/Command/run.pm @@ -19,7 +19,7 @@ use Test2::Harness::Util::LogFile; use Test2::Harness::Util qw/mod2file write_file_atomic/; use Test2::Harness::Util::JSON qw/encode_json encode_pretty_json/; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use Test2::Harness::IPC::Util qw/set_procname/; use parent 'App::Yath::Command'; @@ -38,8 +38,13 @@ include_options( 'App::Yath::Options::Run', 'App::Yath::Options::Tests', 'App::Yath::Options::Yath', + 'App::Yath::Options::WebClient', + 'App::Yath::Options::DB', ); +use App::Yath::Options::Tests qw/ set_dot_args /; + +sub accepts_dot_args { 1 } sub args_include_tests { 1 } sub load_plugins { 1 } @@ -85,6 +90,7 @@ sub run { aggregator_ipc => $client->connect->callback, test_settings => $ts, jobs => $jobs, + settings => $settings, ); my $res = $client->queue_run($run); diff --git a/lib/App/Yath/Command/server.pm b/lib/App/Yath/Command/server.pm index 1398c9f8a..3de146bc3 100644 --- a/lib/App/Yath/Command/server.pm +++ b/lib/App/Yath/Command/server.pm @@ -1,9 +1,374 @@ -package App::Yath::Command::ui; +package App::Yath::Command::server; use strict; use warnings; +use feature 'state'; + +use App::Yath::Server; + +use App::Yath::Schema::Util qw/schema_config_from_settings/; +use Test2::Util::UUID qw/gen_uuid/; +use App::Yath::Schema::ImportModes qw/is_mode/; + +use Test2::Harness::Util qw/clean_path/; + our $VERSION = '2.000000'; +use parent 'App::Yath::Command'; +use Test2::Harness::Util::HashBase qw{ + webserver->launcher_args} => @$dot_args; + return; +} + +use Getopt::Yath; +include_options( + 'App::Yath::Options::Term', + 'App::Yath::Options::Yath', + 'App::Yath::Options::DB', + 'App::Yath::Options::WebServer', + 'App::Yath::Options::Server', +); + +option_group {group => 'server', category => "Server Options"} => sub { + option dev => ( + type => 'Bool', + default => 0, + description => 'Launches in "developer mode" which accepts some developer commands while the server is running.', + ); +}; + + +sub run { + my $self = shift; + my $pid = $$; + + $0 = "yath-server"; + + my $args = $self->args; + my $settings = $self->settings; + + my $dev = $settings->server->dev; + my $shell = $settings->server->shell; + my $daemon = $settings->server->daemon; + my $ephemeral = $settings->server->ephemeral; + + die "Cannot combine --dev, --shell, and/or --daemon.\n" if ($dev && $daemon) || ($dev && $shell) || ($shell && $daemon); + + if ($daemon) { + my $pid = fork // die "Could not fork"; + exit(0) if $pid; + + POSIX::setsid(); + setpgrp(0, 0); + + $pid = fork // die "Could not fork"; + exit(0) if $pid; + + open(STDOUT, '>>', '/dev/null'); + open(STDERR, '>>', '/dev/null'); + } + + my $config = $self->{+CONFIG} = schema_config_from_settings($settings, ephemeral => $ephemeral); + + my $qdb_params = { + single_user => $settings->server->single_user // 0, + single_run => $settings->server->single_run // 0, + no_upload => $settings->server->no_upload // 0, + email => $settings->server->email // undef, + }; + + my $server = $self->{+SERVER} = App::Yath::Server->new(schema_config => $config, $settings->webserver->all, qdb_params => $qdb_params); + $server->start_server; + + my $done = 0; + $SIG{TERM} = sub { $done++; print "Caught SIGTERM shutting down...\n" unless $daemon; $SIG{TERM} = 'DEFAULT' }; + $SIG{INT} = sub { $done++; print "Caught SIGINT shutting down...\n" unless $daemon; $SIG{INT} = 'DEFAULT' }; + + for my $log (@{$args // []}) { + $self->load_file($log); + } + + sleep 1; + + $ENV{YATH_URL} = "http://" . $settings->webserver->host . ":" . $settings->webserver->port . "/"; + print "\nYath URL: $ENV{YATH_URL}\n\n"; + + if ($shell) { + system($ENV{SHELL}); + } + else { + SERVER_LOOP: until ($done) { + if ($dev && !$daemon) { + $ENV{T2_HARNESS_SERVER_DEV} = 1; + + unless(eval { $done = $self->shell($pid); 1 }) { + warn $@; + $done = 1; + } + } + else { + sleep 1; + } + } + } + + if ($pid == $$) { + $server->stop_server if $server->pid; + } + else { + die "Scope leak, wrong PID"; + } + + return 0; +} + + +sub load_file { + my $self = shift; + my ($file, $mode, $project) = @_; + + my $config = $self->{+CONFIG}; + + die "No .jsonl[.*] log file provided.\n" unless $file; + die "Invalid log file '$file': File not found, or not a normal file.\n" unless -f $file; + $file = clean_path($file); + + $mode //= 'complete'; + + state %projects; + + unless($project) { + my $base = $file; + $base =~ s{^.*/}{}g; + $base =~ s{\.jsonl.*$}{}g; + $base =~ s/-\d.*$//g; + $project = $base || "devshell"; + } + + unless ($projects{$project}) { + my $p = $config->schema->resultset('Project')->find_or_create({name => $project}); + $projects{$project} = $p; + } + + my $logfile = $config->schema->resultset('LogFile')->create({ + name => $file, + local_file => $file =~ m{^/} ? $file : "./demo/$file", + }); + + state $user = $config->schema->resultset('User')->find_or_create({username => 'root', password => 'root', realname => 'root'}); + + my $run = $config->schema->resultset('Run')->create({ + user_id => $user->user_id, + mode => $mode, + buffer => 'job', + status => 'pending', + project_id => $projects{$project}->project_id, + + log_file_id => $logfile->log_file_id, + }); + + return $run; +} + +sub shell { + my $self = shift; + my ($pid) = @_; + + # Return that we should exit if the PID is wrong. + return 1 unless $pid == $$; + + my $settings = $self->settings; + my $server = $self->{+SERVER}; + my $config = $self->{+CONFIG}; + + $SIG{TERM} = sub { $SIG{TERM} = 'DEFAULT'; die "Cought SIGTERM exiting...\n" }; + $SIG{INT} = sub { $SIG{INT} = 'DEFAULT'; die "Cought SIGINT exiting...\n" }; + + STDERR->autoflush(); + + my $dsn = $config->dbi_dsn; + + print "DBI_DSN: $dsn\n\n"; + print "\n"; + print "| Yath Server Developer Shell |\n"; + print "| type 'help', 'h', or '?' for help |\n"; + + use Term::ReadLine; + my $term = Term::ReadLine->new('Yath dev console'); + my $OUT = $term->OUT || \*STDOUT; + + my $cmds = $self->command_list(); + $term->Attribs->{'attempted_completion_function'} = sub { + my ($text, $start, $end) = @_; + + if ($start !~ m/\s/) { + my @found; + for my $set (@$cmds) { + next unless $set->[0] =~ m/^\Q$text\E/; + push @found => $set->[0]; + } + + return @found; + } + + my ($fname) = reverse(split m/\s+/, $text); + + return Term::ReadLine::Gnu->filename_completion_function($fname // '', 0); + }; + + my $prompt = "\n> "; + while (1) { + my $in = $term->readline($prompt); + + return 1 if !defined($in); + chomp($in); + next unless length($in); + + return 1 if $in =~ m/^(q|x|exit|quit)$/; + + $term->addhistory($in); + + if ($in =~ m/^(help|h|\?)(?:\s(.+))?$/) { + $self->shell_help($1); + next; + } + + my ($cmd, $args) = split /\s/, $in, 2; + + my $meth = "shell_$cmd"; + if ($self->can($meth)) { + eval { $self->$meth($args); 1 } or warn $@; + } + else { + print STDERR "Invalid command '$in'\n"; + } + } +} + +sub shell_help_text { "Show command list." } +sub shell_help { + my $self = shift; + my $class = ref($self); + + print "\nAvailable commands:\n"; + printf(" %-12s %s\n", "[q]uit", "Quit the program."); + printf(" %-12s %s\n", "e[x]it", "Exit the program."); + printf(" %-12s %s\n", "[h]elp", "Show this help."); + printf(" %-12s %s\n", "?", "Show this help."); + + my $cmds = $self->command_list(); + for my $set (@$cmds) { + my ($cmd, $text) = @$set; + next if $cmd eq 'help'; + printf(" %-12s %s\n", $cmd, $text); + } + + print "\n"; +} + +sub command_list { + my $self = shift; + my $class = ref($self) || $self; + + my @out; + + my $stash = do { no strict 'refs'; \%{"$class\::"} }; + for my $sym (sort keys %$stash) { + next unless $sym =~ m/^shell_(.*)/; + my $cmd = $1; + next if $sym =~ m/_text$/; + next unless $self->can($sym); + + my $text = "${sym}_text"; + $text = $self->can($text) ? $self->$text() : 'No description.'; + + push @out => [$cmd, $text]; + } + + return \@out; +} + +sub shell_reload_text { "Restart web server (does not restart database or importers)." } +sub shell_reload { $_[0]->server->restart_server } + +sub shell_reloaddb_text { "Restart database (data is lost)." } +sub shell_reloaddb { + my $self = shift; + + my $server = $self->server; + $server->stop_server; + $server->stop_importers; + $server->reset_ephemeral_db; + $server->start_server; +} + +sub shell_reloadimp_text { "Restart the importers." } +sub shell_reloadimp { $_[0]->restart_importers() } + +sub shell_db_text { "Open the database." } +sub shell_db { $_[0]->server->qdb->shell('harness_ui') } + +sub shell_shell_text { "Open a shell" } +sub shell_shell { system($ENV{SHELL}) } + +sub shell_load_text { "Load a database file (filename given as argument)" } +sub shell_load { + my $self = shift; + my ($args) = @_; + + my ($file, $mode, $project); + for my $part (split /\s+/, $args) { + if (is_mode($part)) { + die "Multiple modes provided: $mode and $part.\n" if $mode; + $mode = $part; + } + elsif ($part =~ m/\.jsonl/) { + die "Multiple files provided: $file and $part.\n" if $file; + $file = $part; + } + else { + die "Multiple projects provided: $project and $part.\n" if $project; + $project = $part; + } + } + + $self->load_file($file, $mode, $project); +} + +{ + no warnings 'once'; + *shell_r = \*shell_reload; + *shell_r_text = \*shell_reload_text; + *shell_rdb = \*shell_reloaddb; + *shell_rdb_text = \*shell_reloaddb_text; + *shell_ri = \*shell_reloadimp; + *shell_ri_text = \*shell_reloadimp_text; + *shell_l = \*shell_load; + *shell_l_text = \*shell_load_text; + *shell_s = \*shell_shell; + *shell_s_text = \*shell_shell_text; +} + +1; + +__END__ + use Test2::Util qw/pkg_to_file/; use App::Yath::Server::Util qw/share_dir share_file dbd_driver qdb_driver/; @@ -12,7 +377,7 @@ use App::Yath::Server::Config; use App::Yath::Schema::Importer; use App::Yath::Server; -use App::Yath::Schema::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use DBIx::QuickDB; use Plack::Builder; @@ -91,8 +456,8 @@ sub run { single_run => 1, ); - my $user = $config->schema->resultset('User')->create({username => 'root', password => 'root', realname => 'root', user_id => gen_uuid()}); - my $proj = $config->schema->resultset('Project')->create({name => 'default', project_id => gen_uuid()}); + my $user = $config->schema->resultset('User')->create({username => 'root', password => 'root', realname => 'root'}); + my $proj = $config->schema->resultset('Project')->create({name => 'default'}); $config->schema->resultset('Run')->create({ run_id => gen_uuid(), @@ -102,7 +467,6 @@ sub run { project_id => $proj->project_id, log_file => { - log_file_id => gen_uuid(), name => $self->{+LOG_FILE}, local_file => $self->{+LOG_FILE}, }, diff --git a/lib/App/Yath/Command/start.pm b/lib/App/Yath/Command/start.pm index 1a4f669c3..274555d94 100644 --- a/lib/App/Yath/Command/start.pm +++ b/lib/App/Yath/Command/start.pm @@ -41,12 +41,16 @@ sub option_modules { 'App::Yath::Options::Yath', 'App::Yath::Options::Renderer', 'App::Yath::Options::Tests', + 'App::Yath::Options::DB', + 'App::Yath::Options::WebClient', ); } use Getopt::Yath; include_options(__PACKAGE__->option_modules); +use App::Yath::Options::Tests qw/ set_dot_args /; + option_group {group => 'start', category => "Start Options"} => sub { option foreground => ( short => 'f', @@ -62,6 +66,7 @@ sub load_plugins { 1 } sub load_resources { 1 } sub load_renderers { 1 } +sub accepts_dot_args { 1 } sub args_include_tests { 0 } sub group { 'daemon' } diff --git a/lib/App/Yath/Command/test.pm b/lib/App/Yath/Command/test.pm index 83d9fd0bd..5495af149 100644 --- a/lib/App/Yath/Command/test.pm +++ b/lib/App/Yath/Command/test.pm @@ -24,6 +24,8 @@ include_options( 'App::Yath::Command::run', ); +use App::Yath::Options::Tests qw/ set_dot_args /; + option_group {group => 'runner', category => "Runner Options"} => sub { option preload_threshold => ( type => 'Scalar', @@ -35,6 +37,7 @@ option_group {group => 'runner', category => "Runner Options"} => sub { ); }; +sub accepts_dot_args { 1 } sub args_include_tests { 1 } sub group { ' main' } @@ -120,6 +123,7 @@ sub become_instance { instance_ipc => $instance->ipc->[0]->callback, test_settings => $ts, jobs => $jobs, + settings => $settings, ); $instance->scheduler->queue_run($run); diff --git a/lib/App/Yath/Finder.pm b/lib/App/Yath/Finder.pm index bbb3eac63..11b14835f 100644 --- a/lib/App/Yath/Finder.pm +++ b/lib/App/Yath/Finder.pm @@ -5,7 +5,7 @@ use warnings; our $VERSION = '2.000000'; use Test2::Harness::Util qw/clean_path mod2file/; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use Test2::Harness::Util::JSON qw/decode_json encode_json/; use List::Util qw/first/; use Cwd qw/getcwd/; diff --git a/lib/App/Yath/Options/DB.pm b/lib/App/Yath/Options/DB.pm index ff7d6613b..701387d83 100644 --- a/lib/App/Yath/Options/DB.pm +++ b/lib/App/Yath/Options/DB.pm @@ -10,47 +10,56 @@ option_group {group => 'db', prefix => 'db', category => "Database Options"} => option config => ( type => 'Scalar', description => "Module that implements 'MODULE->yath_db_config(%params)' which should return a App::Yath::Schema::Config instance.", + from_env_vars => [qw/YATH_DB_CONFIG/], ); option driver => ( type => 'Scalar', description => "DBI Driver to use", - long_examples => [' Pg', 'mysql', 'MariaDB'], + long_examples => [' Pg', ' PostgreSQL', ' MySQL', ' MariaDB', ' Percona', ' SQLite'], + from_env_vars => [qw/YATH_DB_DRIVER/], ); option name => ( type => 'Scalar', description => 'Name of the database to use', + from_env_vars => [qw/YATH_DB_NAME/], ); option user => ( type => 'Scalar', description => 'Username to use when connecting to the db', + from_env_vars => [qw/YATH_DB_USER USER/], ); option pass => ( type => 'Scalar', description => 'Password to use when connecting to the db', + from_env_vars => [qw/YATH_DB_PASS/], ); option dsn => ( type => 'Scalar', description => 'DSN to use when connecting to the db', + from_env_vars => [qw/YATH_DB_DSN/], ); option host => ( type => 'Scalar', description => 'hostname to use when connecting to the db', + from_env_vars => [qw/YATH_DB_HOST/], ); option port => ( type => 'Scalar', description => 'port to use when connecting to the db', + from_env_vars => [qw/YATH_DB_PORT/], ); option socket => ( type => 'Scalar', description => 'socket to use when connecting to the db', + from_env_vars => [qw/YATH_DB_SOCKET/], ); }; diff --git a/lib/App/Yath/Options/Harness.pm b/lib/App/Yath/Options/Harness.pm index 97134f12b..650392e09 100644 --- a/lib/App/Yath/Options/Harness.pm +++ b/lib/App/Yath/Options/Harness.pm @@ -48,14 +48,14 @@ option_group {group => 'harness', category => 'Harness Options'} => sub { option keep_dirs => ( type => 'Bool', short => 'k', - alt => ['keep_dir'], + alt => ['keep-dir'], description => 'Do not delete directories when done. This is useful if you want to inspect the directories used for various commands.', default => 0, ); option tmpdir => ( type => 'Scalar', - alt => ['tmp_dir'], + alt => ['tmp-dir'], description => 'Use a specific temp directory (Default: use system temp dir)', from_env_vars => [qw/T2_HARNESS_TEMP_DIR YATH_TEMP_DIR TMPDIR TEMPDIR TMP_DIR TEMP_DIR/], clear_env_vars => [qw/T2_HARNESS_TEMP_DIR YATH_TEMP_DIR/], diff --git a/lib/App/Yath/Options/Upload.pm b/lib/App/Yath/Options/Publish.pm similarity index 65% rename from lib/App/Yath/Options/Upload.pm rename to lib/App/Yath/Options/Publish.pm index c56579e50..d87aea3f2 100644 --- a/lib/App/Yath/Options/Upload.pm +++ b/lib/App/Yath/Options/Publish.pm @@ -1,4 +1,4 @@ -package App::Yath::Options::Upload; +package App::Yath::Options::Publish; use strict; use warnings; @@ -6,20 +6,7 @@ our $VERSION = '2.000000'; use Getopt::Yath; -option_group {group => 'upload', prefix => 'upload', category => "DB Upload Options"} => sub { - option flush_interval => ( - type => 'Scalar', - long_examples => [' 2', ' 1.5'], - description => 'When buffering DB writes, force a flush when an event is recieved at least N seconds after the last flush.', - ); - - option buffering => ( - type => 'Scalar', - long_examples => [ ' none', ' job', ' diag', ' run' ], - description => 'Type of buffering to use, if "none" then events are written to the db one at a time, which is SLOW', - default => 'diag', - ); - +option_group {group => 'publish', prefix => 'publish', category => "Publish Options"} => sub { option mode => ( type => 'Scalar', default => 'qvfd', @@ -32,15 +19,28 @@ option_group {group => 'upload', prefix => 'upload', category => "DB Upload Opti ], ); - option user => ( + option flush_interval => ( type => 'Scalar', - default => sub { $ENV{USER} }, - description => "Username to be associated with runs stored in the database. Defaults to your shell username.", + long_examples => [' 2', ' 1.5'], + description => 'When buffering DB writes, force a flush when an event is recieved at least N seconds after the last flush.', + ); + + option buffer_size => ( + type => 'Scalar', + long_examples => [ ' 100' ], + description => 'Maximum number of events, coverage, or reporting items to buffer before flushing them (each has its own buffer of this size, and each job has its own event buffer of this size)', + default => 100, ); option retry => ( type => 'Count', - description => "How many times to try an operation before giving up", + description => "How many times to retry an operation before giving up", + default => 0, + ); + + option force => ( + type => 'Bool', + description => 'If the run has already been published, override it. (Delete it, and publish again)', default => 0, ); }; @@ -55,7 +55,7 @@ __END__ =head1 NAME -App::Yath::Options::Upload - FIXME +App::Yath::Options::Publish - FIXME =head1 DESCRIPTION diff --git a/lib/App/Yath/Options/Recent.pm b/lib/App/Yath/Options/Recent.pm new file mode 100644 index 000000000..220a0de34 --- /dev/null +++ b/lib/App/Yath/Options/Recent.pm @@ -0,0 +1,18 @@ +package App::Yath::Options::Recent; +use strict; +use warnings; + +our $VERSION = '2.000000'; + +use Getopt::Yath; + +option_group {group => 'recent', prefix => 'recent', category => "Recent Options"} => sub { + option max => ( + type => 'Scalar', + long_examples => [' 10'], + default => 10, + description => 'Max number of recent runs to show', + ); +}; + +1; diff --git a/lib/App/Yath/Options/Resource.pm b/lib/App/Yath/Options/Resource.pm index 691354238..7994825ec 100644 --- a/lib/App/Yath/Options/Resource.pm +++ b/lib/App/Yath/Options/Resource.pm @@ -29,7 +29,7 @@ option_group {group => 'resource', category => "Resource Options"} => sub { option slots => ( type => 'Scalar', short => 'j', - alt => ['jobs', 'job_count'], + alt => ['jobs', 'job-count'], description => 'Set the number of concurrent jobs to run. Add a :# if you also wish to designate multiple slots per test. 8:2 means 8 slots, but each test gets 2 slots, so 4 tests run concurrently. Tests can find their concurrency assignemnt in the "T2_HARNESS_MY_JOB_CONCURRENCY" environment variable.', notes => "If System::Info is installed, this will default to half the cpu core count, otherwise the default is 2.", long_examples => [' 4', ' 8:2'], @@ -71,7 +71,7 @@ option_group {group => 'resource', category => "Resource Options"} => sub { option job_slots => ( type => 'Scalar', - alt => ['slots_per_job'], + alt => ['slots-per-job'], short => 'x', description => "This sets the number of slots each job will use (default 1). This is normally set by the ':#' in '-j#:#'.", diff --git a/lib/App/Yath/Options/Run.pm b/lib/App/Yath/Options/Run.pm index 2ae9ebef9..f89286128 100644 --- a/lib/App/Yath/Options/Run.pm +++ b/lib/App/Yath/Options/Run.pm @@ -5,7 +5,7 @@ use warnings; our $VERSION = '2.000000'; use Test2::Harness::Util::JSON qw/decode_json/; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use Test2::Harness::Util qw/fqmod/; use List::Util qw/mesh/; diff --git a/lib/App/Yath/Options/Server.pm b/lib/App/Yath/Options/Server.pm index 165c5bd76..4a19bca92 100644 --- a/lib/App/Yath/Options/Server.pm +++ b/lib/App/Yath/Options/Server.pm @@ -2,64 +2,51 @@ package App::Yath::Options::Server; use strict; use warnings; -our $VERSION = '2.000000'; - use Getopt::Yath; -option_group {group => 'server', prefix => 'server', category => "Server Options"} => sub { - option url => ( - type => 'Scalar', - alt => ['uri'], - description => "Yath-UI url", - long_examples => [" http://my-yath-ui.com/..."], +option_group {group => 'server', category => "Server Options"} => sub { + option ephemeral => ( + type => 'Auto', + autofill => 'Auto', + long_examples => ['', '=Auto', '=PostgreSQL', '=MySQL', '=MariaDB', '=SQLite', '=Percona' ], + description => "Use a temporary 'ephemeral' database that will be destroyed when the server exits.", + autofill_text => 'If no db type is specified it will use "auto" which will try PostgreSQL first, then MySQL.', + allowed_values => [qw/Auto PostgreSQL MySQL MariaDB Percona SQLite/], ); -}; - -1; - -__END__ - -=pod - -=encoding UTF-8 - -=head1 NAME - -App::Yath::Options::Server - FIXME - -=head1 DESCRIPTION - -=head1 PROVIDED OPTIONS POD IS AUTO-GENERATED - -=head1 SOURCE - -The source code repository for Test2-Harness can be found at -L. - -=head1 MAINTAINERS -=over 4 - -=item Chad Granum Eexodist@cpan.orgE - -=back - -=head1 AUTHORS - -=over 4 - -=item Chad Granum Eexodist@cpan.orgE - -=back + option shell => ( + type => 'Bool', + default => 0, + description => "Drop into a shell where the server and database env vars are set so that yath commands will use the started server.", + ); -=head1 COPYRIGHT + option daemon => ( + type => 'Bool', + default => 0, + description => "Run the server in the background.", + ); -Copyright Chad Granum Eexodist7@gmail.comE. + option single_user => ( + type => 'Bool', + default => 0, + description => "When using an ephemeral database you can use this to enable single user mode to avoid login and user credentials.", + ); -This program is free software; you can redistribute it and/or -modify it under the same terms as Perl itself. + option single_run => ( + type => 'Bool', + default => 0, + description => "When using an ephemeral database you can use this to enable single run mode which causes the server to take you directly to the first run.", + ); -See L + option no_upload => ( + type => 'Bool', + default => 0, + description => "When using an ephemeral database you can use this to enable no-upload mode which removes the upload workflow.", + ); -=cut + option email => ( + type => 'Scalar', + description => "When using an ephemeral database you can use this to set a 'from' email address for email sent from this server.", + ); +}; diff --git a/lib/App/Yath/Options/Tests.pm b/lib/App/Yath/Options/Tests.pm index 1b022bedc..d318d39c9 100644 --- a/lib/App/Yath/Options/Tests.pm +++ b/lib/App/Yath/Options/Tests.pm @@ -4,8 +4,11 @@ use warnings; our $VERSION = '2.000000'; +use Importer Importer => 'import'; use Getopt::Yath; +our @EXPORT_OK = qw/ set_dot_args /; + use Test2::Harness::TestSettings; my $DEFAULT_COVER_ARGS = Test2::Harness::TestSettings->default_cover_args; @@ -133,7 +136,7 @@ option_group {group => 'tests', category => 'Test Options', maybe => 1} => sub { option stream => ( type => 'Bool', - alt => ['use_stream'], + alt => ['use-stream'], alt_no => ['TAP'], description => "The TAP format is lossy and clunky. Test2::Harness normally uses a newer streaming format to receive test results. There are old/legacy tests where this causes problems, in which case setting --TAP or --no-stream can help.", @@ -141,7 +144,7 @@ option_group {group => 'tests', category => 'Test Options', maybe => 1} => sub { option test_args => ( type => 'List', - alt => ['test_arg'], + alt => ['test-arg'], field => 'args', description => 'Arguments to pass in as @ARGV for all tests that are run. These can be provided easier using the \'::\' argument separator.' @@ -205,6 +208,19 @@ option_group {group => 'tests', category => 'Test Options', maybe => 1} => sub { ); }; +sub set_dot_args { + my $class = shift; + my ($settings, $dot_args) = @_; + + my $oldvals = $settings->tests->args; + unshift @$dot_args => @$oldvals; + $settings->tests->option(args => $dot_args); + + return; +} + + + 1; __END__ diff --git a/lib/App/Yath/Options/WebClient.pm b/lib/App/Yath/Options/WebClient.pm new file mode 100644 index 000000000..ac5c2d0d4 --- /dev/null +++ b/lib/App/Yath/Options/WebClient.pm @@ -0,0 +1,84 @@ +package App::Yath::Options::WebClient; +use strict; +use warnings; + +our $VERSION = '2.000000'; + +use Getopt::Yath; + +option_group {group => 'webclient', category => "Web Client Options"} => sub { + option url => ( + type => 'Scalar', + alt => ['uri'], + description => "Yath server url", + long_examples => [" http://my-yath-server.com/..."], + from_env_vars => [qw/YATH_URL/], + ); + + option api_key => ( + type => 'Scalar', + description => "Yath server API key. This is not necessary if your Yath server instance is set to single-user", + from_env_vars => [qw/YATH_API_KEY/], + ); + + option grace => ( + type => 'Bool', + description => "If yath cannot connect to yath-ui it normally throws an error, use this to make it fail gracefully. You get a warning, but things keep going.", + default => 0, + ); + + option request_retry => ( + type => 'Count', + description => "How many times to try an operation before giving up", + default => 0, + ); +}; + +1; + +__END__ + +=pod + +=encoding UTF-8 + +=head1 NAME + +App::Yath::Options::WebClient - FIXME + +=head1 DESCRIPTION + +=head1 PROVIDED OPTIONS POD IS AUTO-GENERATED + +=head1 SOURCE + +The source code repository for Test2-Harness can be found at +L. + +=head1 MAINTAINERS + +=over 4 + +=item Chad Granum Eexodist@cpan.orgE + +=back + +=head1 AUTHORS + +=over 4 + +=item Chad Granum Eexodist@cpan.orgE + +=back + +=head1 COPYRIGHT + +Copyright Chad Granum Eexodist7@gmail.comE. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +See L + +=cut + diff --git a/lib/App/Yath/Options/WebServer.pm b/lib/App/Yath/Options/WebServer.pm new file mode 100644 index 000000000..a33d9d617 --- /dev/null +++ b/lib/App/Yath/Options/WebServer.pm @@ -0,0 +1,124 @@ +package App::Yath::Options::WebServer; +use strict; +use warnings; + +our $VERSION = '2.000000'; + +use Getopt::Yath; + +include_options( + 'App::Yath::Options::DB', +); + +option_group {group => 'webserver', category => "Web Server Options"} => sub { + option launcher => ( + type => 'Scalar', + default => sub { eval { require Starman; 1 } ? 'Starman' : undef }, + description => 'Command to use to launch the server (--server argument to Plack::Runner) ', + notes => "You can pass custom args to the launcher after a '::' like `yath server [ARGS] [LOG FILES(s)] :: [LAUNCHER ARGS]`", + default_text => "Will use 'Starman' if it installed otherwise whatever Plack::Runner uses by default.", + ); + + option port_command => ( + type => 'Scalar', + description => 'Command to run that returns a port number.', + ); + + option port => ( + type => 'Scalar', + description => 'Port to listen on.', + notes => 'This is passed to the launcher via `launcher --port PORT`', + default => sub { + my ($option, $settings) = @_; + + if (my $cmd = $settings->webserver->port_command) { + local $?; + my $port = `$cmd`; + die "Port command `$cmd` exited with error code $?.\n" if $?; + die "Port command `$cmd` did not return a valid port.\n" unless $port; + chomp($port); + die "Port command `$cmd` did not return a valid port: $port.\n" unless $port =~ m/^\d+$/; + return $port; + } + + return 8080; + }, + ); + + option host => ( + type => 'Scalar', + default => 'localhost', + description => "Host/Address to bind to, default 'localhost'.", + ); + + option workers => ( + type => 'Scalar', + default => sub { eval { require System::Info; System::Info->new->ncore } || 5 }, + default_text => "5, or number of cores if System::Info is installed.", + description => 'Number of workers. Defaults to the number of cores, or 5 if System::Info is not installed.', + notes => 'This is passed to the launcher via `launcher --workers WORKERS`', + ); + + option importers => ( + type => 'Scalar', + default => 2, + description => 'Number of log importer processes.', + ); + + option launcher_args => ( + type => 'List', + initialize => sub { [] }, + description => "Set additional options for the loader.", + notes => "It is better to put loader arguments after '::' at the end of the command line.", + long_examples => [' "--reload"', '="--reload"'], + ); +}; + +1; + +__END__ + +=pod + +=encoding UTF-8 + +=head1 NAME + +App::Yath::Options::WebServer - FIXME + +=head1 DESCRIPTION + +=head1 PROVIDED OPTIONS POD IS AUTO-GENERATED + +=head1 SOURCE + +The source code repository for Test2-Harness can be found at +L. + +=head1 MAINTAINERS + +=over 4 + +=item Chad Granum Eexodist@cpan.orgE + +=back + +=head1 AUTHORS + +=over 4 + +=item Chad Granum Eexodist@cpan.orgE + +=back + +=head1 COPYRIGHT + +Copyright Chad Granum Eexodist7@gmail.comE. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +See L + +=cut + diff --git a/lib/App/Yath/Options/Yath.pm b/lib/App/Yath/Options/Yath.pm index e5731b36c..2e3fc3bca 100644 --- a/lib/App/Yath/Options/Yath.pm +++ b/lib/App/Yath/Options/Yath.pm @@ -22,6 +22,12 @@ option_group {group => 'yath', category => 'Yath Options'} => sub { description => 'This lets you provide a label for your current project/codebase. This is best used in a .yath.rc file.', ); + option user => ( + type => 'Scalar', + description => 'Username to associate with logs, database entries, and yath servers.', + from_env_vars => [qw/YATH_USER USER/], + ); + option base_dir => ( type => 'Scalar', description => "Root directory for the project being tested (usually where .yath.rc lives)", diff --git a/lib/App/Yath/Plugin/Cover.pm b/lib/App/Yath/Plugin/Cover.pm index a3b7970e7..885d41ded 100644 --- a/lib/App/Yath/Plugin/Cover.pm +++ b/lib/App/Yath/Plugin/Cover.pm @@ -6,7 +6,7 @@ our $VERSION = '2.000000'; use Test2::Harness::Util qw/clean_path mod2file fqmod/; use Test2::Harness::Util::JSON qw/encode_json stream_json_l/; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use parent 'App::Yath::Plugin'; use Test2::Harness::Util::HashBase qw/-aggregator -no_aggregate +metrics +outfile/; diff --git a/lib/App/Yath/Plugin/DB.pm b/lib/App/Yath/Plugin/DB.pm index dafc71490..adf549424 100644 --- a/lib/App/Yath/Plugin/DB.pm +++ b/lib/App/Yath/Plugin/DB.pm @@ -5,7 +5,8 @@ use warnings; our $VERSION = '2.000000'; use App::Yath::Schema::Util qw/schema_config_from_settings/; -use Test2::Harness::Util qw/mod2file looks_like_uuid/; +use Test2::Harness::Util qw/mod2file/; +use Test2::Util::UUID qw/looks_like_uuid/; use Getopt::Yath; use parent 'App::Yath::Plugin'; @@ -63,9 +64,9 @@ sub duration_data { my $ydb = $settings->prefix('yathui-db') or return; return unless $ydb->durations; - my $config = schema_config_from_settings($settings); - my $schema = $config->schema; - my $pname = $settings->yathui->project or die "yathui-project is required.\n"; + my $config = schema_config_from_settings($settings); + my $schema = $config->schema; + my $pname = $settings->yath->project or die "--project is required.\n"; my $project = $schema->resultset('Project')->find({name => $pname}) or die "Invalid project '$pname'.\n"; my %args = (user => $ydb->publisher, limit => $ydb->duration_limit); @@ -94,8 +95,8 @@ sub grab_rerun { my ($ok, $err, $run); if ($rerun eq '1') { - my $project_name = $settings->yathui->project; - my $username = $settings->yathui->user // $ENV{USER}; + my $project_name = $settings->yath->project; + my $username = $settings->yath->user // $ENV{USER}; $ok = eval { $run = $schema->vague_run_search(query => {}, project_name => $project_name, username => $username); 1 }; $err = $@; } @@ -163,9 +164,9 @@ sub get_coverage_rows { my $config = schema_config_from_settings($settings); my $schema = $config->schema; - my $pname = $settings->yathui->project or die "yathui-project is required.\n"; + my $pname = $settings->yath->project or die "--project is required.\n"; my $project = $schema->resultset('Project')->find({name => $pname}) or die "Invalid project '$pname'.\n"; - my $run = $project->last_covered_run(user => $ydb->publisher) or return; + my $run = $project->last_covered_run(user => $ydb->publisher) or return; my @searches = $plugin->get_coverage_searches($settings, $changes) or return; return $run->expanded_coverages({'-or' => \@searches}); diff --git a/lib/App/Yath/Plugin/Git.pm b/lib/App/Yath/Plugin/Git.pm index 8fa2b6c09..83f9a4e6f 100644 --- a/lib/App/Yath/Plugin/Git.pm +++ b/lib/App/Yath/Plugin/Git.pm @@ -75,11 +75,8 @@ sub run_fields { if ($branch) { $data{branch} = $branch; - - my $short = length($branch) > 20 ? substr($branch, 0, 20) : $branch; - - $field->{details} = $short; - $field->{raw} = $branch; + $field->{details} = $branch; + $field->{raw} = $long_sha; } else { $short_sha ||= substr($long_sha, 0, 16); diff --git a/lib/App/Yath/Plugin/Server.pm b/lib/App/Yath/Plugin/Server.pm index b33c92714..afb80dace 100644 --- a/lib/App/Yath/Plugin/Server.pm +++ b/lib/App/Yath/Plugin/Server.pm @@ -1,4 +1,14 @@ -package App::Yath::Plugin::YathUI; + +# Move cover stuff to cover plugin +# +# move duration to finder +# +# move publish to client-publish +# +# [POST] server/publish/project + +__END__ +package App::Yath::Plugin::Server; use strict; use warnings; @@ -11,12 +21,9 @@ use Test2::Harness::Util::JSON qw/decode_json/; use Getopt::Yath; use parent 'App::Yath::Plugin'; -sub can_log { - my ($option, $options) = @_; - - return 1 if $options->included->{'App::Yath::Options::Logging'}; - return 0; -} +include_options( + 'App::Yath::Options::Server', +); sub can_finder { my ($option, $options) = @_; @@ -25,69 +32,28 @@ sub can_finder { return 0; } -option_group {prefix => 'yathui', group => 'yathui', category => "YathUI Options"} => sub { - option url => ( - type => 'Scalar', - alt => ['uri'], - description => "Yath-UI url", - long_examples => [" http://my-yath-ui.com/..."], - ); - - option api_key => ( - type => 'Scalar', - description => "Yath-UI API key. This is not necessary if your Yath-UI instance is set to single-user" - ); - - option project => ( - type => 'Scalar', - description => "The Yath-UI project for your test results", - ); - - option mode => ( - type => 'Scalar', - default => 'qvfd', - description => "Set the upload mode (default 'qvfd')", - long_examples => [ - ' summary', - ' qvf', - ' qvfd', - ' complete', - ], - ); - - option retry => ( - type => 'Count', - description => "How many times to try an operation before giving up", - default => 0, - ); +sub can_render { + my ($option, $options) = @_; - option grace => ( - type => 'Bool', - description => "If yath cannot connect to yath-ui it normally throws an error, use this to make it fail gracefully. You get a warning, but things keep going.", - default => 0, - ); + return 1 if $options->included->{'App::Yath::Options::Render'}; + return 0; +} +option_group {prefix => 'server', group => 'server', category => "Server Options"} => sub { option durations => ( type => 'Bool', - description => "Poll duration data from Yath-UI to help order tests efficiently", + description => "Poll duration data from your server to help order tests efficiently", default => 0, applicable => \&can_finder, ); option coverage => ( type => 'Bool', - description => "Poll coverage data from Yath-UI to determine what tests should be run for changed files", + description => "Poll coverage data from your server to determine what tests should be run for changed files", default => 0, applicable => \&can_finder, ); -# TODO -# option median_durations => ( -# type => 'b', -# description => "Get median duration data", -# default => 0, -# ); - option medium_duration => ( type => 'Scalar', description => "Minimum duration length (seconds) before a test goes from SHORT to MEDIUM", @@ -102,11 +68,12 @@ option_group {prefix => 'yathui', group => 'yathui', category => "YathUI Options default => 10, ); - option upload => ( + option publish => ( type => 'Bool', - description => "Upload the log to Yath-UI", + description => 'Publish the log to the server when the run is complete', + notes => 'This will enable logging if it is not already enabled', default => 0, - applicable => \&can_log, + applicable => \&can_render, ); option_post_process -1 => sub { @@ -115,21 +82,22 @@ option_group {prefix => 'yathui', group => 'yathui', category => "YathUI Options my $settings = $state->{settings}; my $has_finder = $options->included->{'App::Yath::Options::Finder'}; - my $has_logger = $options->included->{'App::Yath::Options::Logging'}; + my $has_render = $options->included->{'App::Yath::Options::Renderer'}; - my $has_durations = $has_finder && $settings->yathui->durations; - my $has_upload = $has_logger && $settings->yathui->upload; - my $has_coverage = $has_finder && $settings->yathui->coverage; + my $has_durations = $has_finder && $settings->server->durations; + my $has_publish = $has_render && $settings->server->publish; + my $has_coverage = $has_finder && $settings->server->coverage; - return unless $has_durations || $has_upload || $has_coverage; + return unless $has_durations || $has_publish || $has_coverage; - my $url = $settings->yathui->url or die "'--yathui-url URL' is required to use durations, coverage, or upload a log"; - my $project = $settings->yathui->project or die "'--yathui-project NAME' is required to use durations, coverage, or upload a log"; - my $grace = $settings->yathui->grace; + my $project = $settings->yath->project or die "'--project NAME' is required to use durations, coverage, or upload a log"; + my $url = $settings->server->url or die "'--server-url URL' is required to use durations, coverage, or upload a log"; + my $grace = $settings->server->grace; $url =~ s{/+$}{}g; - if ($has_upload) { + if ($has_render) { + die "fixme"; $settings->logging->option(log => 1); $settings->logging->option(bzip2 => 1); } @@ -162,8 +130,8 @@ sub grab_rerun { my $path; if ($rerun eq '1') { - my $project = $settings->yathui->project or return (0); - my $user = $settings->yathui->user // $ENV{USER}; + my $project = $settings->yath->project or return (0); + my $user = $settings->yath->user // $ENV{USER}; $path = "$project/$user"; diff --git a/lib/App/Yath/Renderer/DB.pm b/lib/App/Yath/Renderer/DB.pm index 215f93835..f59ca3534 100644 --- a/lib/App/Yath/Renderer/DB.pm +++ b/lib/App/Yath/Renderer/DB.pm @@ -16,17 +16,13 @@ use Time::HiRes qw/time/; use Test2::Harness::Util qw/clean_path/; use Test2::Harness::IPC::Util qw/start_process/; use Test2::Harness::Util::JSON qw/encode_ascii_json/; -use App::Yath::Schema::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use parent 'App::Yath::Renderer'; use Test2::Harness::Util::HashBase qw{ 'db', prefix => 'db', category => "Database Options"} => sub { - option resources => ( - type => 'Auto', - description => 'Send resource info (for supported resources) to the database at the specified interval in seconds (5 if not specified)', - long_examples => ['', '=5'], - autofill => 5, - ); -}; - sub start { my $self = shift; @@ -110,12 +97,6 @@ sub render_event { return; } -sub step { - my $self = shift; - - $self->send_resources(); -} - sub signal { my $self = shift; my ($sig) = @_; @@ -163,74 +144,6 @@ sub finish { return; } -sub resource_interval { - my $self = shift; - return $self->{+RESOURCE_INTERVAL} //= $self->settings->db->resources; -} - -sub resources { - my $self = shift; - return $self->{+RESOURCES} if $self->{+RESOURCES}; - - die "FIXME"; - - my $state = $self->state; - $state->poll; - - return $self->{+RESOURCES} = [grep { $_ && $_->can('status_data') } @{$state->resources}]; -} - -sub send_resources { - my $self = shift; - - my $interval = $self->resource_interval or return; - my $resources = $self->resources or return; - return unless @$resources; - - my $stamp = time; - - if (my $last = $self->{+LAST_RESOURCE_STAMP}) { - my $delta = $stamp - $last; - return unless $delta >= $interval; - } - - unless(eval { $self->_send_resources($stamp => $resources); 1 }) { - my $err = $@; - warn "Non fatal error, could not send resource info to YathUI: $err"; - return; - } - - return $self->{+LAST_RESOURCE_STAMP} = $stamp; -} - -sub _send_resources { - my $self = shift; - my ($stamp, $resources) = @_; - - my $batch_id = gen_uuid(); - my $ord = 0; - my @items; - for my $res (@$resources) { - my $data = $res->status_data or next; - - my $item = { - resource_id => gen_uuid(), - resource_batch_id => $batch_id, - batch_ord => $ord++, - module => ref($res) || $res, - data => encode_ascii_json($data), - }; - - push @items => $item; - } - - return unless @items; - - $self->render_event({facet_data => {db_resources => {stamp => $stamp, batch_id => $batch_id, items => \@items}}}); - - return; -} - 1; __END__ diff --git a/lib/App/Yath/Renderer/Logger.pm b/lib/App/Yath/Renderer/Logger.pm index d19a68825..1cab743e6 100644 --- a/lib/App/Yath/Renderer/Logger.pm +++ b/lib/App/Yath/Renderer/Logger.pm @@ -20,6 +20,11 @@ use Test2::Harness::Util::HashBase qw{ }; use Getopt::Yath; + +include_options( + 'App::Yath::Options::WebClient', +); + option_group {group => 'logging', category => "Logging Options", applicable => \&applicable} => sub { option dir => ( prefix => 'log', @@ -58,6 +63,19 @@ option_group {group => 'logging', category => "Logging Options", applicable => \ } }; + option publish => ( + type => 'Auto', + description => 'Publish the log to a yath server, will use the --url if one is provided, otherwise use =URL to specify a url', + long_examples => ['', '=URL'], + short_examples => ['', '=URL'], + notes => 'This should not be used in combination with the DB renderer if the url uses the database.', + autofill => sub { + my ($option, $settings) = @_; + my $url = $settings->webclient->url or die "No --url specified, either provide one or give a value to the --publish option.\n(NOTE: the --url option must come before the --publish option on the command line)\n"; + return $url; + }, + ); + option log => ( type => 'Auto', short => 'L', @@ -202,6 +220,8 @@ sub finish { close($self->{+FH}); print "\nWrote log file: $self->{+FILE}\n"; + + warn "FIXME: publish should send log to server\n"; } sub weight { -100 } diff --git a/lib/App/Yath/Renderer/Notify.pm b/lib/App/Yath/Renderer/Notify.pm index 2cce4fb2b..df664dedc 100644 --- a/lib/App/Yath/Renderer/Notify.pm +++ b/lib/App/Yath/Renderer/Notify.pm @@ -108,7 +108,7 @@ option_group {prefix => 'notify', group => 'notify', category => "Notification O option text_module => ( type => 'Scalar', - alt => ['message_module'], + alt => ['message-module'], description => "Use the specified module to generate messages for emails and/or slack.", ); diff --git a/lib/App/Yath/Renderer/Server.pm b/lib/App/Yath/Renderer/Server.pm index ea267b736..7b7a2f84c 100644 --- a/lib/App/Yath/Renderer/Server.pm +++ b/lib/App/Yath/Renderer/Server.pm @@ -11,7 +11,7 @@ use App::Yath::Schema::RunProcessor; use Test2::Util qw/pkg_to_file/; use Test2::Harness::Util qw/mod2file/; use App::Yath::Server::Util qw/share_dir share_file dbd_driver qdb_driver/; -use App::Yath::Schema::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use DBIx::QuickDB; use Plack::Builder; @@ -153,10 +153,10 @@ sub init { ); $self->{+USER} = 'root'; - my $user = $config->schema->resultset('User')->create({username => 'root', password => 'root', realname => 'root', user_id => gen_uuid()}); + my $user = $config->schema->resultset('User')->create({username => 'root', password => 'root', realname => 'root'}); $self->{+PROJECT} = 'default'; - my $proj = $config->schema->resultset('Project')->create({name => 'default', project_id => gen_uuid()}); + my $proj = $config->schema->resultset('Project')->create({name => 'default'}); $self->{+CONFIG} = $config; diff --git a/lib/App/Yath/Resource/SharedJobSlots.pm b/lib/App/Yath/Resource/SharedJobSlots.pm index fce335f1c..412ad0967 100644 --- a/lib/App/Yath/Resource/SharedJobSlots.pm +++ b/lib/App/Yath/Resource/SharedJobSlots.pm @@ -18,7 +18,7 @@ use Test2::Harness::Util qw/find_in_updir/; use Getopt::Yath; -option_group {group => 'shared_slots', category => "Shared Slot Options"} => sub { +option_group {group => 'resource', category => "Resource Options"} => sub { option shared_jobs => ( type => 'Bool', maybe => 1, @@ -41,16 +41,15 @@ sub shared_post_process { my $settings = $state->{settings}; return unless $settings->check_group('resource'); - my $resource = $settings->resource; - my $shared_slots = $settings->shared_slots; + my $resource = $settings->resource; my $required = 0; - if (defined($shared_slots->shared_jobs)) { - return unless $shared_slots->shared_jobs; + if (defined($resource->shared_jobs)) { + return unless $resource->shared_jobs; $required = 1; } - my $base_name = $shared_slots->shared_jobs_config; + my $base_name = $resource->shared_jobs_config; unless ($base_name && (-e $base_name || find_in_updir($base_name))) { return unless $required; diff --git a/lib/App/Yath/Schema.pm b/lib/App/Yath/Schema.pm index ed2b6a720..15ef644e5 100644 --- a/lib/App/Yath/Schema.pm +++ b/lib/App/Yath/Schema.pm @@ -3,25 +3,80 @@ use utf8; use strict; use warnings; use Carp qw/confess/; - -use App::Yath::Schema::UUID qw/uuid_inflate/; +use Carp::Always; our $VERSION = '2.000000'; use base 'DBIx::Class::Schema'; +use Test2::Util::UUID qw/uuid2bin bin2uuid/; + confess "You must first load a App::Yath::Schema::NAME module" unless $App::Yath::Schema::LOADED; -if ($App::Yath::Schema::LOADED =~ m/MySQL/ && eval { require DBIx::Class::Storage::DBI::mysql::Retryable; 1 }) { - __PACKAGE__->storage_type('::DBI::mysql::Retryable'); -} +#if ($App::Yath::Schema::LOADED =~ m/(MySQL|Percona|MariaDB)/i && eval { require DBIx::Class::Storage::DBI::mysql::Retryable; 1 }) { +# __PACKAGE__->storage_type('::DBI::mysql::Retryable'); +#} require App::Yath::Schema::ResultSet; __PACKAGE__->load_namespaces( default_resultset_class => 'ResultSet', ); +sub is_mysql { + return 1 if is_mariadb(); + return 1 if is_percona(); + return 1 if $App::Yath::Schema::LOADED =~ m/MySQL/; + return 0; +} + +sub is_postgresql { + return 1 if $App::Yath::Schema::LOADED =~ m/PostgreSQL/; + return 0; +} + +sub is_sqlite { + return 1 if $App::Yath::Schema::LOADED =~ m/SQLite/; + return 0; +} + +sub is_percona { + return 1 if $App::Yath::Schema::LOADED =~ m/Percona/; + return 0; +} + +sub is_mariadb { + return 1 if $App::Yath::Schema::LOADED =~ m/MariaDB/; + return 0; +} + +sub format_uuid_for_db { + my $class = shift; + my ($uuid) = @_; + + return $uuid unless is_percona(); + return uuid2bin($uuid); +} + +sub format_uuid_for_app { + my $class = shift; + my ($uuid_bin) = @_; + + return $uuid_bin unless is_percona(); + return bin2uuid($uuid_bin); +} + +sub config { + my $self = shift; + my ($setting, @val) = @_; + + my $conf = $self->resultset('Config')->find_or_create({setting => $setting, @val ? (value => $val[0]) : ()}); + + $conf->update({value => $val[0]}) if @val; + + return $conf->value; +} + sub vague_run_search { my $self = shift; my (%params) = @_; @@ -29,7 +84,7 @@ sub vague_run_search { my ($project, $run, $user); my $query = $params{query} // {status => 'complete'}; - my $attrs = $params{attrs} // {order_by => {'-desc' => 'run_ord'}, rows => 1}; + my $attrs = $params{attrs} // {order_by => {'-desc' => 'run_id'}, rows => 1}; $attrs->{offset} = $params{idx} if $params{idx}; @@ -44,27 +99,22 @@ sub vague_run_search { } if (my $source = $params{source}) { - my $uuid = uuid_inflate($source); - - if ($uuid) { - $run = $self->resultset('Run')->find({%$query, run_id => $uuid}, $attrs); - return $run if $run; - } + my $run = $self->resultset('Run')->find_by_id_or_uuid($source, $query, $attrs); + return $run if $run; - if (my $p = $self->resultset('Project')->find($uuid ? {project_id => $uuid} : {name => $source})) { + if (my $p = $self->resultset('Project')->find({name => $source})) { die "Project mismatch ($source)" if $project && $project->project_id ne $p->project_id; $query->{project_id} = $p->project_id; } - elsif (my $u = $self->resultset('User')->find($uuid ? {user_id => $uuid} : {username => $source})) { + elsif (my $u = $self->resultset('User')->find({username => $source})) { die "User mismatch ($source)" if $user && $user->user_id ne $u->user_id; $query->{user_id} = $u->user_id; } else { - die "No UUID match in runs, users, or projects ($uuid)" if $uuid; die "No match for source ($source)"; } } diff --git a/lib/App/Yath/Server.pm b/lib/App/Yath/Server.pm index a9a1fde92..729255c3f 100644 --- a/lib/App/Yath/Server.pm +++ b/lib/App/Yath/Server.pm @@ -2,245 +2,318 @@ package App::Yath::Server; use strict; use warnings; +use Carp qw/croak confess/; +use Test2::Harness::Util qw/parse_exit mod2file/; +use Test2::Util::UUID qw/gen_uuid/; +use Test2::Harness::Util::JSON qw/encode_json decode_json/; + +use Plack::Runner; +use DBIx::QuickDB; + +use App::Yath::Server::Plack; +use App::Yath::Schema::Importer; + +use App::Yath::Util qw/share_file/; +use App::Yath::Schema::Util qw/qdb_driver dbd_driver/; + our $VERSION = '2.000000'; -use Router::Simple; -use Text::Xslate(qw/mark_raw/); -use Scalar::Util qw/blessed/; -use DateTime; - -use App::Yath::Server::Request; -use App::Yath::Server::Controller::Upload; -use App::Yath::Server::Controller::Recent; -use App::Yath::Server::Controller::User; -use App::Yath::Server::Controller::Run; -use App::Yath::Server::Controller::RunField; -use App::Yath::Server::Controller::Job; -use App::Yath::Server::Controller::JobField; -use App::Yath::Server::Controller::Download; -use App::Yath::Server::Controller::Sweeper; -use App::Yath::Server::Controller::Project; -use App::Yath::Server::Controller::Resources; - -use App::Yath::Server::Controller::Stream; -use App::Yath::Server::Controller::View; -use App::Yath::Server::Controller::Lookup; - -use App::Yath::Server::Controller::Query; -use App::Yath::Server::Controller::Events; - -use App::Yath::Server::Controller::Durations; -use App::Yath::Server::Controller::Coverage; -use App::Yath::Server::Controller::Files; -use App::Yath::Server::Controller::ReRun; - -use App::Yath::Server::Controller::Interactions; -use App::Yath::Server::Controller::Binary; - -use App::Yath::Server::Util qw/share_dir/; -use App::Yath::Server::Response qw/resp error/; +use Test2::Harness::Util::HashBase qw{ + {+ROUTER} ||= Router::Simple->new; - my $config = $self->{+CONFIG}; + croak "'schema_config' is a required attribute" unless $self->{+SCHEMA_CONFIG}; - $router->connect('/' => {controller => 'App::Yath::Server::Controller::View'}); + $self->{+QDB_PARAMS} //= {}; +} - $router->connect('/upload' => {controller => 'App::Yath::Server::Controller::Upload'}) - unless $config->single_run; +sub restart_server { + my $self = shift; + my ($sig) = @_; - $router->connect('/user' => {controller => 'App::Yath::Server::Controller::User'}) - unless $config->single_user; + my $exit = $self->stop_server($sig); + $self->start_server(); - $router->connect('/resources/data/:id' => {controller => 'App::Yath::Server::Controller::Resources', data => 1}); - $router->connect('/resources/data/:id/' => {controller => 'App::Yath::Server::Controller::Resources', data => 1}); - $router->connect('/resources/data/:id/:batch' => {controller => 'App::Yath::Server::Controller::Resources', data => 1}); - $router->connect('/resources/:id' => {controller => 'App::Yath::Server::Controller::Resources'}); - $router->connect('/resources/:id/' => {controller => 'App::Yath::Server::Controller::Resources'}); - $router->connect('/resources/:id/:batch' => {controller => 'App::Yath::Server::Controller::Resources'}); + return $exit; +} - $router->connect('/interactions/:id' => {controller => 'App::Yath::Server::Controller::Interactions'}); - $router->connect('/interactions/:id/:context' => {controller => 'App::Yath::Server::Controller::Interactions'}); - $router->connect('/interactions/data/:id' => {controller => 'App::Yath::Server::Controller::Interactions', data => 1}); - $router->connect('/interactions/data/:id/:context' => {controller => 'App::Yath::Server::Controller::Interactions', data => 1}); +sub stop_server { + my $self = shift; + my ($sig) = @_; - $router->connect('/project/:id' => {controller => 'App::Yath::Server::Controller::Project'}); - $router->connect('/project/:id/stats' => {controller => 'App::Yath::Server::Controller::Project', stats => 1}); - $router->connect('/project/:id/:n' => {controller => 'App::Yath::Server::Controller::Project'}); - $router->connect('/project/:id/:n/:count' => {controller => 'App::Yath::Server::Controller::Project'}); + $self->_root_proc_check(); - $router->connect('/recent/:project/:user/:count' => {controller => 'App::Yath::Server::Controller::Recent'}); - $router->connect('/recent/:project/:user' => {controller => 'App::Yath::Server::Controller::Recent'}); + my $pid = delete $self->{+PID} or croak "No server running"; - $router->connect('/query/:name' => {controller => 'App::Yath::Server::Controller::Query'}); - $router->connect('/query/:name/:arg' => {controller => 'App::Yath::Server::Controller::Query'}); + return $self->stop_proc($pid, $sig); +} + +sub stop_proc { + my $self = shift; + my ($pid, $sig) = @_; - $router->connect('/run/:id' => {controller => 'App::Yath::Server::Controller::Run'}); - $router->connect('/run/:id/pin' => {controller => 'App::Yath::Server::Controller::Run', action => 'pin_toggle'}); - $router->connect('/run/:id/delete' => {controller => 'App::Yath::Server::Controller::Run', action => 'delete'}); - $router->connect('/run/:id/cancel' => {controller => 'App::Yath::Server::Controller::Run', action => 'cancel'}); - $router->connect('/run/:id/parameters' => {controller => 'App::Yath::Server::Controller::Run', action => 'parameters'}); + $self->_root_proc_check(); - $router->connect('/run/field/:id' => {controller => 'App::Yath::Server::Controller::RunField'}); - $router->connect('/run/field/:id/delete' => {controller => 'App::Yath::Server::Controller::RunField', action => 'delete'}); + croak "'pid' is required" unless $pid; + $sig //= 'TERM'; - $router->connect('/job/field/:id' => {controller => 'App::Yath::Server::Controller::JobField'}); - $router->connect('/job/field/:id/delete' => {controller => 'App::Yath::Server::Controller::JobField', action => 'delete'}); + local $?; + kill($sig, $pid); + my $got = waitpid($pid, 0); + my $exit = $?; + + croak "waitpid returned '$got', expected '$pid'" unless $got == $pid; + return parse_exit($exit); +} - $router->connect('/job/:job' => {controller => 'App::Yath::Server::Controller::Job'}); - $router->connect('/job/:job/:try' => {controller => 'App::Yath::Server::Controller::Job'}); - $router->connect('/event/:id' => {controller => 'App::Yath::Server::Controller::Events', from => 'single_event'}); - $router->connect('/event/:id/events' => {controller => 'App::Yath::Server::Controller::Events', from => 'event'}); +sub reset_ephemeral_db { + my $self = shift; + my ($sig) = @_; - $router->connect('/durations/:project' => {controller => 'App::Yath::Server::Controller::Durations'}); - $router->connect('/durations/:project/median' => {controller => 'App::Yath::Server::Controller::Durations', median => 1}); - $router->connect('/durations/:project/median/:user' => {controller => 'App::Yath::Server::Controller::Durations', median => 1}); - $router->connect('/durations/:project/:short/:medium' => {controller => 'App::Yath::Server::Controller::Durations'}); + my $exit = $self->stop_ephemeral_db($sig); + $self->start_ephemeral_db(); - $router->connect('/coverage/:source' => {controller => 'App::Yath::Server::Controller::Coverage'}); - $router->connect('/coverage/:source/:user' => {controller => 'App::Yath::Server::Controller::Coverage'}); - $router->connect('/coverage/:source/delete' => {controller => 'App::Yath::Server::Controller::Coverage', delete => 1}); + return $exit; +} - $router->connect('/failed/:source' => {controller => 'App::Yath::Server::Controller::Files', failed => 1}); - $router->connect('/failed/:source/json' => {controller => 'App::Yath::Server::Controller::Files', failed => 1, json => 1}); - $router->connect('/failed/:project/:idx' => {controller => 'App::Yath::Server::Controller::Files', failed => 1, json => 1}); - $router->connect('/failed/:project/:username/:idx' => {controller => 'App::Yath::Server::Controller::Files', failed => 1, json => 1}); +sub stop_ephemeral_db { + my $self = shift; + my ($sig) = @_; - $router->connect('/files/:source' => {controller => 'App::Yath::Server::Controller::Files', failed => 0}); - $router->connect('/files/:source/json' => {controller => 'App::Yath::Server::Controller::Files', failed => 0, json => 1}); - $router->connect('/files/:project/:idx' => {controller => 'App::Yath::Server::Controller::Files', failed => 0, json => 1}); - $router->connect('/files/:project/:username/:idx' => {controller => 'App::Yath::Server::Controller::Files', failed => 0, json => 1}); + $self->_root_proc_check(); + $self->stop_server if $self->{+PID}; + $self->stop_importers if $self->{+IMPORTER_PIDS}; - $router->connect('/rerun/:run_id' => {controller => 'App::Yath::Server::Controller::ReRun'}); - $router->connect('/rerun/:project/:username' => {controller => 'App::Yath::Server::Controller::ReRun'}); + my $db = delete $self->{+QDB} or croak "No ephemeral db running"; - $router->connect('/binary/:binary_id' => {controller => 'App::Yath::Server::Controller::Binary'}); + $db->stop; +} - $router->connect('/download/:id' => {controller => 'App::Yath::Server::Controller::Download'}); +sub start_ephemeral_db { + my $self = shift; - $router->connect('/lookup' => {controller => 'App::Yath::Server::Controller::Lookup'}); - $router->connect('/lookup/:lookup' => {controller => 'App::Yath::Server::Controller::Lookup'}); - $router->connect('/lookup/data/:lookup' => {controller => 'App::Yath::Server::Controller::Lookup', data => 1}); + croak "Ephemeral DB already started" if $self->{+QDB}; - $router->connect('/view' => {controller => 'App::Yath::Server::Controller::View'}); - $router->connect('/view/:id' => {controller => 'App::Yath::Server::Controller::View'}); - $router->connect('/view/:run_id/:job' => {controller => 'App::Yath::Server::Controller::View'}); - $router->connect('/view/:run_id/:job/:try' => {controller => 'App::Yath::Server::Controller::View'}); + $self->{+ROOT_PID} //= $$; + $self->_root_proc_check(); - $router->connect('/stream/run/:run_id' => {controller => 'App::Yath::Server::Controller::Stream', run_only => 1}); - $router->connect('/stream' => {controller => 'App::Yath::Server::Controller::Stream'}); - $router->connect('/stream/:id' => {controller => 'App::Yath::Server::Controller::Stream'}); - $router->connect('/stream/:run_id/:job' => {controller => 'App::Yath::Server::Controller::Stream'}); - $router->connect('/stream/:run_id/:job/:try' => {controller => 'App::Yath::Server::Controller::Stream'}); + my $config = $self->{+SCHEMA_CONFIG}; + my $schema_type = $config->ephemeral // 'Auto'; - $router->connect('/sweeper/:count/days' => {controller => 'App::Yath::Server::Controller::Sweeper', units => 'day'}); - $router->connect('/sweeper/:count/hours' => {controller => 'App::Yath::Server::Controller::Sweeper', units => 'hour'}); - $router->connect('/sweeper/:count/minutes' => {controller => 'App::Yath::Server::Controller::Sweeper', units => 'minute'}); - $router->connect('/sweeper/:count/seconds' => {controller => 'App::Yath::Server::Controller::Sweeper', units => 'second'}); + my $qdb_args; + if ($schema_type eq 'Auto') { + $qdb_args = {drivers => [qdb_driver('PostgreSQL'), qdb_driver('MySQL')]}; + $schema_type = undef; + } + else { + $qdb_args = {driver => qdb_driver($schema_type), dbd_driver => dbd_driver($schema_type)} + } + + my $db = DBIx::QuickDB->build_db(harness_ui => $qdb_args); + unless($schema_type) { + if (ref($db) =~ m/::(PostgreSQL|MySQL)$/) { + $schema_type = $1; + } + else { + die "$db does not look like PostgreSQL or MySQL"; + } + } + + my $dbh; + if ($schema_type =~ m/SQLite/i) { + $dbh = $db->connect('harness_ui', AutoCommit => 1, RaiseError => 1); + } + else { + $dbh = $db->connect('quickdb', AutoCommit => 1, RaiseError => 1); + $dbh->do('CREATE DATABASE harness_ui') or die "Could not create db " . $dbh->errstr; + } + + $db->load_sql(harness_ui => share_file("schema/$schema_type.sql")); + my $dsn = $db->connect_string('harness_ui'); + + $config->push_ephemeral_credentials(dbi_dsn => $dsn, dbi_user => '', dbi_pass => '', schema_type => $schema_type); + $ENV{YATH_DB_DSN} = $dsn; + + require(mod2file("App::Yath::Schema::$schema_type")); + + my $schema = $config->schema; + + $schema->resultset('User')->create({username => 'root', password => 'root', realname => 'root'}); + + my $qdb_params = $self->{+QDB_PARAMS} // {}; + $schema->config(single_user => $qdb_params->{single_user} // 0); + $schema->config(single_run => $qdb_params->{single_run} // 0); + $schema->config(no_upload => $qdb_params->{no_upload} // 0); + $schema->config(email => $qdb_params->{email}) if $qdb_params->{email}; + + return $self->{+QDB} = $db; } -sub to_app { +sub start_server { my $self = shift; + my %params = @_; + + croak "Server already started with pid $self->{+PID}" if $self->{+PID}; + + $self->{+ROOT_PID} //= $$; + $self->_root_proc_check(); + + if ($self->{+SCHEMA_CONFIG}->ephemeral && !$params{no_db} && !$self->{+QDB}) { + $self->start_ephemeral_db(); + } + + unless ($self->{+IMPORTER_PIDS} || $params{no_importers}) { + $self->start_importers(); + } - my $router = $self->{+ROUTER}; + my $pid = fork // die "Could not fork: $!"; - return sub { - my $env = shift; + return $self->{+PID} = $pid if $pid; - my $req = App::Yath::Server::Request->new(env => $env, config => $self->{+CONFIG}); + $0 = "yath-web-server"; - my $r = $router->match($env) || {}; + my $ok = eval { $self->_do_server_exec(); 1 }; + my $err = $@; - $self->wrap($r->{controller}, $req, $r); - }; + unless ($ok) { + eval { warn $err }; + exit 255; + } + + exit(0); } -sub wrap { +sub _do_server_exec { my $self = shift; - my ($class, $req, $r) = @_; - my ($controller, $res, $session); - my $ok = eval { - die error(404) unless $class; + my @options; + push @options => @{$self->{+LAUNCHER_ARGS} // []}; + push @options => ("--server" => $self->{+LAUNCHER}) if $self->{+LAUNCHER}; + push @options => ('--listen' => "$self->{+HOST}:$self->{+PORT}") if $self->{+PORT}; + push @options => ('--workers' => "$self->{+WORKERS}") if $self->{+WORKERS}; + + exec( + $^X, + (map {"-I$_"} @INC), + "-m${ \__PACKAGE__ }", + '-e' => "${ \__PACKAGE__ }->_do_server_post_exec(\$ARGV[0])", + encode_json({ + schema_config => $self->{+SCHEMA_CONFIG}, + launcher_options => \@options, + }), + ); +} - if ($class->uses_session) { - $session = $req->session; - $req->session_host; # vivify this - } +sub _do_server_post_exec { + my $class = shift; + my ($json) = @_; - $controller = $class->new(request => $req, config => $self->{+CONFIG}); - $res = $controller->handle($r); + $0 = "yath-web-server"; - 1; - }; - my $err = $@ || 'Internal Error'; + my $data = decode_json($json); - unless ($ok && $res) { - if (blessed($err) && $err->isa('App::Yath::Server::Response')) { - $res = $err; - } - else { - warn $err; - my $msg = ($ENV{T2_HARNESS_UI_ENV} || '') eq 'dev' ? "$err\n" : undef; - $res = error(500 => $msg); - } - } + my $r = Plack::Runner->new; + $r->parse_options(@{$data->{launcher_options}}); - my $ct = $r->{json} ? 'application/json' : blessed($res) ? $res->content_type() : 'text/html'; - $ct ||= 'text/html'; - $ct = lc($ct); - $res->content_type($ct) if blessed($res); + my $app = App::Yath::Server::Plack->new( + schema_config => bless($data->{+SCHEMA_CONFIG}, 'App::Yath::Schema::Config'), + ); - if (my $stream = $res->stream) { - return $stream; - } + $r->run($app->to_app()); + + exit(0); +} - if ($ct eq 'text/html') { - my $dt = DateTime->now(time_zone => 'local'); +sub restart_importers { + my $self = shift; + $self->stop_importers(); + $self->start_importers(); +} - my $tx = Text::Xslate->new(path => [share_dir('templates')]); - my $wrapped = $tx->render( - 'main.tx', - { - config => $self->{+CONFIG}, +sub start_importers { + my $self = shift; - user => $req->user || undef, - errors => $res->errors || [], - messages => $res->messages || [], - add_css => $res->css || [], - add_js => $res->js || [], - title => $res->title || ($controller ? $controller->title : 'Test2-Harness-UI'), + local $0 = 'yath-importer'; - time_zone => $dt->strftime("%Z"), + croak "Importers already started" if $self->{+IMPORTER_PIDS}; - base_uri => $req->base->as_string || '', - content => mark_raw($res->raw_body) || '', - } - ); + $self->{+ROOT_PID} //= $$; + $self->_root_proc_check(); - $res->body($wrapped); + # Gen uuids here before forking + my @pids; + for (1 .. $self->{+IMPORTERS} // 2) { + push @pids => App::Yath::Schema::Importer->new(config => $self->{+SCHEMA_CONFIG})->spawn(); } - elsif($ct eq 'application/json') { - if (my $data = $res->raw_body) { - $res->body(ref($data) ? encode_json($data) : $data); - } - elsif (my $errors = $res->errors) { - $res->body(encode_json({errors => $errors})); - } + + $self->{+IMPORTER_PIDS} = \@pids; +} + +sub stop_importers { + my $self = shift; + + my $pids = delete $self->{+IMPORTER_PIDS} or croak "Importers not started"; + $self->_root_proc_check(); + + kill('TERM', @$pids); + + for my $pid (@$pids) { + local $?; + my $got = waitpid($pid, 0); + my $exit = $?; + + warn "waitpid returned '$got' expected '$pid'" unless $got == $pid; + warn "importer process exited with $exit" if $exit; } - $res->cookies->{id} = {value => $session->session_id, httponly => 1, expires => '+1M'} - if $session; + return; +} + +sub _root_proc_check { + my $self = shift; + confess "root_pid is not set, did you start any servers?" unless $self->{+ROOT_PID}; + return if $$ == $self->{+ROOT_PID}; + confess "Attempt to manage processes from the wrong process"; +} + +sub shutdown { + my $self = shift; + + $self->_root_proc_check(); - return $res->finalize; + $self->stop_importers() if $self->importer_pids; + $self->stop_ephemeral_db() if $self->qdb; + $self->stop_server() if $self->pid; } +sub DESTROY { + my $self = shift; + + local $?; + + return unless $self->{+ROOT_PID}; + return unless $self->{+ROOT_PID} == $$; + + $self->shutdown(); +} __END__ @@ -250,22 +323,13 @@ __END__ =head1 NAME -App::Yath::Server - Web interface for viewing and inspecting yath test logs - -=head1 EARLY VERSION WARNING - -This program is still in early development. There are many bugs, missing -features, and things that will change. +App::Yath::Server - FIXME =head1 DESCRIPTION -This package provides a web UI for yath logs. =head1 SYNOPSIS -The easiest thing to do is use the C command, which -will create a temporary postgresql db, load your log into it, then launch the -app in starman on a local port that you can visit in your browser. =head1 SOURCE diff --git a/lib/App/Yath/Server/Config.pm b/lib/App/Yath/Server/Config.pm deleted file mode 100644 index 3dc18aead..000000000 --- a/lib/App/Yath/Server/Config.pm +++ /dev/null @@ -1,118 +0,0 @@ -package App::Yath::Server::Config; -use strict; -use warnings; - -our $VERSION = '2.000000'; - -use Test2::Util qw/get_tid pkg_to_file/; - -use Carp qw/croak/; - -use Test2::Harness::Util::HashBase qw{ - -_schema - -dbi_dsn -dbi_user -dbi_pass - -single_user -single_run -no_upload - -show_user - -email -}; - -sub disconnect { shift->schema->storage->disconnect } -sub connect { shift->schema->storage->dbh } - -sub init { - my $self = shift; - - croak "'dbi_dsn' is a required attribute" - unless defined $self->{+DBI_DSN}; - - croak "'dbi_user' is a required attribute" - unless defined $self->{+DBI_USER}; - - croak "'dbi_pass' is a required attribute" - unless defined $self->{+DBI_PASS}; - - $self->{+SHOW_USER} //= 0; -} - -sub guess_db_driver { - my $self = shift; - - return 'MySQL' if $self->{+DBI_DSN} =~ m/(mysql|maria|percona)/i; - return 'PostgreSQL' if $self->{+DBI_DSN} =~ m/(pg|postgre)/i; - return 'PostgreSQL'; # Default -} - -sub db_driver { - my $self = shift; - return $ENV{YATH_UI_SCHEMA} //= $self->guess_db_driver; -} - -sub schema { - my $self = shift; - - return $self->{+_SCHEMA} if $self->{+_SCHEMA}; - - unless ($App::Yath::Server::Schema::LOADED) { - my $schema = $ENV{YATH_UI_SCHEMA} //= $self->guess_db_driver; - require(pkg_to_file("App::Yath::Server::Schema::$schema")); - } - - require App::Yath::Server::Schema; - - return $self->{+_SCHEMA} = App::Yath::Server::Schema->connect( - $self->dbi_dsn, - $self->dbi_user, - $self->dbi_pass, - {AutoCommit => 1, RaiseError => 1}, - ); -} - -1; - -__END__ - -=pod - -=encoding UTF-8 - -=head1 NAME - -App::Yath::Server::Config - UI configuration - -=head1 DESCRIPTION - -=head1 SYNOPSIS - -TODO - -=head1 SOURCE - -The source code repository for Test2-Harness-UI can be found at -F. - -=head1 MAINTAINERS - -=over 4 - -=item Chad Granum Eexodist@cpan.orgE - -=back - -=head1 AUTHORS - -=over 4 - -=item Chad Granum Eexodist@cpan.orgE - -=back - -=head1 COPYRIGHT - -Copyright Chad Granum Eexodist7@gmail.comE. - -This program is free software; you can redistribute it and/or -modify it under the same terms as Perl itself. - -See F - -=cut diff --git a/lib/App/Yath/Server/Controller.pm b/lib/App/Yath/Server/Controller.pm index dc281c1f2..d124d89f4 100644 --- a/lib/App/Yath/Server/Controller.pm +++ b/lib/App/Yath/Server/Controller.pm @@ -8,21 +8,49 @@ use Carp qw/croak/; use App::Yath::Server::Response qw/error/; -use Test2::Harness::Util::HashBase qw/-request -config/; - -sub uses_session { 1 } +use Test2::Harness::Util::HashBase qw{ + {+REQUEST}; - croak "'config' is a required attribute" unless $self->{+CONFIG}; + croak "'request' is a required attribute" unless $self->{+REQUEST}; + croak "'schema_config' is a required attribute" unless $self->{+SCHEMA_CONFIG}; + + croak "'single_user' must be defined" unless defined $self->{+SINGLE_USER}; + croak "'single_run' must be defined" unless defined $self->{+SINGLE_RUN}; } -sub title { 'Test2-Harness-UI' } -sub handle { error(501) } +sub schema { $_[0]->{+SCHEMA} //= $_[0]->{+SCHEMA_CONFIG}->schema } + +sub title { 'Yath-Server' } + +sub handle { error(501 => "Controller '" . ref($_[0]) . "' did not implement handle()") } + +sub requires_user { 0 } + +sub auth_check { + my $self = shift; + + return unless $self->requires_user; + + return error(501 => "Controller '" . ref($_[0]) . "' did not implement verify_user_credentials()") + unless $self->can('verify_user_credentials'); + + return error(401) unless $self->verify_user_credentials(); + + return; +} -sub schema { $_[0]->{+CONFIG}->schema } 1; diff --git a/lib/App/Yath/Server/Controller/Binary.pm b/lib/App/Yath/Server/Controller/Binary.pm index 9d7d027d5..bdc4a7673 100644 --- a/lib/App/Yath/Server/Controller/Binary.pm +++ b/lib/App/Yath/Server/Controller/Binary.pm @@ -8,7 +8,7 @@ use App::Yath::Server::Response qw/resp error/; use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase; -use App::Yath::Schema::UUID qw/uuid_inflate/; + sub title { 'Binary' } @@ -20,11 +20,11 @@ sub handle { my $res = resp(200); die error(404 => 'Missing route') unless $route; - my $binary_id = uuid_inflate($route->{binary_id}) or die error(404 => "Invalid Route"); + my $binary_id = $route->{binary_id} or die error(404 => "Invalid Route"); error(404 => 'No id') unless $binary_id; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $binary = $schema->resultset('Binary')->find({binary_id => $binary_id}); error(404 => 'No such binary file') unless $binary_id; diff --git a/lib/App/Yath/Server/Controller/Coverage.pm b/lib/App/Yath/Server/Controller/Coverage.pm index ed52b0342..d0acba348 100644 --- a/lib/App/Yath/Server/Controller/Coverage.pm +++ b/lib/App/Yath/Server/Controller/Coverage.pm @@ -4,11 +4,10 @@ use warnings; our $VERSION = '2.000000'; -use Data::GUID; use List::Util qw/max/; use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json encode_pretty_json decode_json/; -use App::Yath::Schema::UUID qw/uuid_inflate/; + use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase; @@ -22,7 +21,7 @@ sub handle { my $req = $self->{+REQUEST}; my $res = resp(200); - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; die error(404 => 'Missing route') unless $route; my $source = $route->{source} or die error(404 => 'No source'); @@ -40,15 +39,13 @@ sub handle { $run = $project->last_covered_run(user => $username); } else { - $source = uuid_inflate($source) or die error(404 => 'Invalid Run'); - $run = eval { $schema->resultset('Run')->find({run_id => $source}) } or warn $@; - die error(405) unless $run; + $run = $schema->resultset('Run')->find_by_id_or_uuid($source) or die error(405); } die error(404) unless $run; if ($delete) { - $run->coverages->delete; + $run->coverage->delete; $run->update({has_coverage => 0}); } else { diff --git a/lib/App/Yath/Server/Controller/Download.pm b/lib/App/Yath/Server/Controller/Download.pm index 610fd5571..3b82d3472 100644 --- a/lib/App/Yath/Server/Controller/Download.pm +++ b/lib/App/Yath/Server/Controller/Download.pm @@ -4,13 +4,12 @@ use warnings; our $VERSION = '2.000000'; -use Data::GUID; use List::Util qw/max/; use Text::Xslate(qw/mark_raw/); -use App::Yath::Server::Util qw/share_dir/; +use App::Yath::Util qw/share_dir/; use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json decode_json/; -use App::Yath::Schema::UUID qw/uuid_inflate/; + use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase qw/-title/; @@ -29,13 +28,12 @@ sub handle { my $run; - if ($self->{+CONFIG}->single_run) { + if ($self->single_run) { $run = $user->runs->first or die error(404 => 'Invalid run'); } else { my $it = $route->{id} or die error(404 => 'No id'); - $it = uuid_inflate($it) or die error(404 => 'Invalid Run'); - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; $run = $schema->resultset('Run')->find({run_id => $it}) or die error(404 => 'Invalid Run'); } diff --git a/lib/App/Yath/Server/Controller/Durations.pm b/lib/App/Yath/Server/Controller/Durations.pm index 21eb5ec25..1b2a5c6b1 100644 --- a/lib/App/Yath/Server/Controller/Durations.pm +++ b/lib/App/Yath/Server/Controller/Durations.pm @@ -4,7 +4,6 @@ use warnings; our $VERSION = '2.000000'; -use Data::GUID; use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json encode_pretty_json/; @@ -30,7 +29,7 @@ sub handle { my $median = $route->{median} || 0; my $username = $route->{user}; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $project = $schema->resultset('Project')->find({name => $project_name}); my $data = {}; diff --git a/lib/App/Yath/Server/Controller/Events.pm b/lib/App/Yath/Server/Controller/Events.pm index 4b02e1230..017993ae2 100644 --- a/lib/App/Yath/Server/Controller/Events.pm +++ b/lib/App/Yath/Server/Controller/Events.pm @@ -7,7 +7,7 @@ our $VERSION = '2.000000'; use List::Util qw/max/; use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json decode_json/; -use App::Yath::Schema::UUID qw/uuid_inflate/; + use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase; @@ -21,7 +21,7 @@ sub handle { my $req = $self->{+REQUEST}; my $res = resp(200); my $user = $req->user; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; die error(404 => 'Missing route') unless $route; my $it = $route->{id} or die error(404 => 'No name or id'); @@ -29,18 +29,16 @@ sub handle { my $p = $req->parameters; my (%query, %attrs, $rs, $meth, $event); - my $event_id = uuid_inflate($it) or die error(404 => "Invalid event id"); - if ($route->{from} eq 'single_event') { - $event = $schema->resultset('Event')->find({event_id => $event_id}, {remove_columns => [qw/orphan/]}) + $event = $schema->resultset('Event')->find_by_id_or_uuid($it, {remove_columns => [qw/orphan/]}) or die error(404 => 'Invalid Event'); } else { - $event = $schema->resultset('Event')->find({event_id => $event_id}, {remove_columns => [qw/orphan facets/]}) + $event = $schema->resultset('Event')->find_by_id_or_uuid($it, {remove_columns => [qw/orphan facets/]}) or die error(404 => 'Invalid Event'); } - $attrs{order_by} = {-asc => 'event_ord'}; + $attrs{order_by} = {-asc => ['event_idx', 'event_sdx', 'event_id']}; if ($route->{from} eq 'single_event') { $res->content_type('application/json'); @@ -49,40 +47,18 @@ sub handle { } if ($p->{load_subtests}) { - # If we are loading subtests then we want ALL descendants, so here - # we take the parent event and find the next event of the same - # nesting level, then we want all events with an event_ord between - # them (in the same job); - my $end_at = $schema->resultset('Event')->find( - {%query, nested => $event->nested, event_ord => {'>' => $event->event_ord}}, - { - columns => [qw/event_ord/], - %attrs, - }, - ); - - $query{event_ord} = {'>' => $event->event_ord, '<' => $end_at->event_ord}; + $query{job_try_id} = $event->job_try_id; # Same job try + $query{event_idx} = $event->event_idx; # Same subtest + $query{event_id} = {'!=' => $event->event_id}; # Not this event } else { # We want direct descendants only - $query{'parent_id'} = $event_id; + $query{'parent_id'} = $event->event_id; } $rs = $schema->resultset('Event')->search( \%query, - { - remove_columns => ['orphan'], - '+select' => [ - 'facets IS NOT NULL AS has_facets', - 'orphan IS NOT NULL AS has_orphan', - ], - '+as' => [ - 'has_facets', - 'has_orphan', - ], - - %attrs - }, + \%attrs ); $res->stream( diff --git a/lib/App/Yath/Server/Controller/Files.pm b/lib/App/Yath/Server/Controller/Files.pm index 07977c9b8..c9e721c04 100644 --- a/lib/App/Yath/Server/Controller/Files.pm +++ b/lib/App/Yath/Server/Controller/Files.pm @@ -4,11 +4,10 @@ use warnings; our $VERSION = '2.000000'; -use Data::GUID; use List::Util qw/max/; use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json encode_pretty_json decode_json/; -use App::Yath::Schema::UUID qw/uuid_inflate/; + use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase; @@ -32,10 +31,10 @@ sub handle { my $failed = $route->{failed}; error(404 => 'No source') unless $source || $project_name; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $query = {status => 'complete'}; - my $attrs = {order_by => {'-desc' => 'run_ord'}, rows => 1}; + my $attrs = {order_by => {'-desc' => 'run_id'}, rows => 1}; $attrs->{offset} = $idx if $idx; @@ -53,12 +52,21 @@ sub handle { die error(400 => "Invalid Request: $err") unless $ok; die error(404 => 'No Data') unless $run; - my $search = {retry => 0}; - $search->{fail} = 1 if $failed; + my $search = {is_harness_out => 0}; + if ($failed) { + $search->{fail} = 1; + $search->{retry} = 0; + } my $files = $run->jobs->search( $search, - {join => 'test_file', order_by => 'test_file.filename'}, + { + join => ['jobs_tries', 'test_file'], + order_by => 'test_file.filename', + group_by => ['me.job_id', 'test_file.filename'], + '+select' => [{max => 'jobs_tries.job_try_id'}, {max => 'jobs_tries.job_try_ord'}, {bool_and => 'jobs_tries.fail'}], + '+as' => ['job_try_id', 'job_try_ord', 'fail'], + }, ); unless($json) { @@ -68,8 +76,8 @@ sub handle { return $res; } - my $run_id = $run->run_id; - my $run_uri = $req->base . "view/$run_id"; + my $run_uuid = $run->run_uuid; + my $run_uri = $req->base . "view/$run_uuid"; my $field_exclusions = { -and => [ @@ -81,8 +89,8 @@ sub handle { my $data = { last_run_stamp => $run->added->epoch, - run_id => $run_id, - run_uri => $req->base . "view/" . $run->run_id, + run_uuid => $run_uuid, + run_uri => $req->base . "view/" . $run->run_uuid, fields => [$run->run_fields->search($field_exclusions)->all], failures => [], passes => [], @@ -92,21 +100,23 @@ sub handle { my $passes = $data->{passes}; while (my $file = $files->next) { - my $job_key = $file->job_key; - my $job_id = $file->job_id; + my $job_uuid = $file->get_column('job_uuid'); + my $try_id = $file->get_column('job_try_id'); + my $try_ord = $file->get_column('job_try_ord'); + my $fail = $file->get_column('fail'); my $row = { file => $file->file, - fields => [$file->job_fields->search($field_exclusions)->all], - job_id => $job_id, - job_key => $job_key, - uri => "$run_uri/$job_key", + fields => [$schema->resultset('JobTryField')->search({job_try_id => $try_id, %$field_exclusions})->all], + job_uuid => $job_uuid, + job_try => $try_id, + uri => "$run_uri/$job_uuid/$try_ord", }; - if ($file->fail) { + if ($fail) { my $subtests = {}; - my $event_rs = $file->events({nested => 0, is_subtest => 1}); + my $event_rs = $schema->resultset('JobTry')->find({job_try_id => $try_id})->events({nested => 0, is_subtest => 1}); while (my $event = $event_rs->next) { my $f = $event->facets; next unless $f->{assert}; diff --git a/lib/App/Yath/Server/Controller/Interactions.pm b/lib/App/Yath/Server/Controller/Interactions.pm index 5077d20c2..4b2983d5c 100644 --- a/lib/App/Yath/Server/Controller/Interactions.pm +++ b/lib/App/Yath/Server/Controller/Interactions.pm @@ -5,12 +5,11 @@ use warnings; our $VERSION = '2.000000'; use DateTime; -use Data::GUID; use Scalar::Util qw/blessed/; use App::Yath::Server::Response qw/resp error/; -use App::Yath::Server::Util qw/share_dir find_job/; +use App::Yath::Util qw/share_dir/; +use App::Yath::Schema::Util qw/find_job is_mysql/; use Test2::Harness::Util::JSON qw/encode_json/; -use App::Yath::Schema::UUID qw/uuid_inflate/; use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase qw/-title/; @@ -23,7 +22,12 @@ sub handle { my $req = $self->{+REQUEST}; - my $id = uuid_inflate($route->{id}) or die error(404 => 'No event id provided'); + my $schema = $self->schema; + my $id = $route->{id} or die error(404 => 'No event id provided'); + + my $event = $schema->resultset('Event')->find_by_id_or_uuid($id) + or die error(404 => 'Invalid Event'); + my $context = $route->{context} // 1; return $self->data($id, $context) if $route->{data}; @@ -43,10 +47,10 @@ sub handle { my $content = $tx->render( 'interactions.tx', { - base_uri => $req->base->as_string, - event_id => $id, - user => $req->user, - data_uri => $data_uri, + base_uri => $req->base->as_string, + event_id => $event->event_uuid, + user => $req->user, + data_uri => $data_uri, context_count => $context, } ); @@ -59,30 +63,33 @@ sub data { my $self = shift; my ($id, $context) = @_; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; + # Get event - my $event = $schema->resultset('Event')->find({event_id => $id}) + my $event = $schema->resultset('Event')->find_by_id_or_uuid($id) or die error(404 => 'Invalid Event'); - my $stamp = $event->get_column('stamp') or die "No stamp?!"; + my $stamp = $event->get_column('stamp') or die error(500 => "Requested event does not have a timestamp"); - # Get job - my $job = $event->job_key or die error(500 => "Could not find job"); - - # Get run from event - my $run = $job->run or die error(500 => "Could not find run"); + # Get job id + my $try = $event->job_try; + my $job = $try->job; + my $run = $job->run; # Get tests from run where the start and end surround the event - my $job_rs = $run->jobs( + my $try_rs = $schema->resultset('JobTry')->search( { - job_key => {'!=' => $job->job_key}, - ended => {'>=' => $stamp}, + 'job.job_id' => {'!=' => $job->job_id}, + 'me.ended' => {'>=' => $stamp }, '-or' => [ - {launch => {'<=' => $stamp}}, - {start => {'<=' => $stamp}}, + {'me.launch' => {'<=' => $stamp}}, + {'me.start' => {'<=' => $stamp}}, ], }, - {order_by => 'job_ord'}, + { + join => 'job', + order_by => 'job_try_id', + }, ); my $req = $self->{+REQUEST}; @@ -91,9 +98,9 @@ sub data { my ($event_rs, %seen_events); my @out = ( {type => 'run', data => $run}, - {type => 'job', data => $job->glance_data}, + {type => 'job', data => $job->glance_data(try_id => $try->job_try_id)}, {type => 'event', data => $event->line_data}, - {type => 'count', data => $job_rs->count}, + {type => 'count', data => $try_rs->count}, ); my $advance = sub { @@ -117,10 +124,11 @@ sub data { $event_rs = undef; } - if (my $job = $job_rs->next) { - push @out => {type => 'job', data => $job->glance_data}; + if (my $try = $try_rs->next) { + my $job = $try->job; + push @out => {type => 'job', data => $job->glance_data(try_id => $try->job_try_id)}; - $event_rs = $job->events( + $event_rs = $try->events( { '-or' => [ {is_subtest => 1, nested => 0}, @@ -132,7 +140,7 @@ sub data { }, ], }, - {order_by => 'event_ord'}, + {order_by => ['event_idx', 'event_sdx']}, ); return 0; @@ -179,9 +187,8 @@ sub interval { my $self = shift; my ($stamp, $op, $context) = @_; - my $driver = $self->{+CONFIG}->db_driver; - - return \"timestamp '$stamp' $op INTERVAL '$context' seconds" if $driver eq 'PostgreSQL'; + return \"timestamp '$stamp' $op INTERVAL '$context' second" + unless is_mysql(); # *Sigh* MySQL return \"DATE_ADD('$stamp', INTERVAL $context second)" if $op eq '+'; diff --git a/lib/App/Yath/Server/Controller/Job.pm b/lib/App/Yath/Server/Controller/Job.pm index 568ed6d58..5be51c3e8 100644 --- a/lib/App/Yath/Server/Controller/Job.pm +++ b/lib/App/Yath/Server/Controller/Job.pm @@ -4,10 +4,10 @@ use warnings; our $VERSION = '2.000000'; -use Data::GUID; use List::Util qw/max/; use Text::Xslate(qw/mark_raw/); -use App::Yath::Server::Util qw/share_dir find_job/; +use App::Yath::Util qw/share_dir/; +use App::Yath::Schema::Util qw/find_job_and_try/; use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json decode_json/; @@ -22,19 +22,20 @@ sub handle { my $res = resp(200); - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $user = $req->user; die error(404 => 'Missing route') unless $route; my $it = $route->{job} or die error(404 => 'No job uuid'); - my $try = $route->{try}; + my $ord = $route->{try}; - my $job = find_job($schema, $it, $try) // die error(404 => 'Invalid Job'); + my ($job, $try) = find_job_and_try($schema, $it, $ord); + die error(404 => 'Invalid Job') unless $job && $try; - $self->{+TITLE} = 'Job: ' . ($job->file || $job->name) . ' - ' . $job->job_id . '+' . $job->job_try; + $self->{+TITLE} = 'Job: ' . ($job->file || $job->name) . ' - ' . $job->job_id . '+' . $try->job_try_ord; $res->content_type('application/json'); - $res->raw_body($job); + $res->raw_body({job => $job, try => $try}); return $res; } diff --git a/lib/App/Yath/Server/Controller/JobField.pm b/lib/App/Yath/Server/Controller/JobTryField.pm similarity index 79% rename from lib/App/Yath/Server/Controller/JobField.pm rename to lib/App/Yath/Server/Controller/JobTryField.pm index 37aaf0bb2..a667c8bec 100644 --- a/lib/App/Yath/Server/Controller/JobField.pm +++ b/lib/App/Yath/Server/Controller/JobTryField.pm @@ -1,16 +1,15 @@ -package App::Yath::Server::Controller::JobField; +package App::Yath::Server::Controller::JobTryField; use strict; use warnings; our $VERSION = '2.000000'; -use Data::GUID; use List::Util qw/max/; use Text::Xslate(qw/mark_raw/); -use App::Yath::Server::Util qw/share_dir/; +use App::Yath::Util qw/share_dir/; use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json decode_json/; -use App::Yath::Schema::UUID qw/uuid_inflate/; + use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase qw/-title/; @@ -27,9 +26,8 @@ sub handle { die error(404 => 'Missing route') unless $route; my $it = $route->{id} or die error(404 => 'No id'); - $it = uuid_inflate($it) or die error(404 => "Invalid id"); - my $schema = $self->{+CONFIG}->schema; - my $field = $schema->resultset('JobField')->find({job_field_id => $it}) or die error(404 => 'Invalid Field'); + my $schema = $self->schema; + my $field = $schema->resultset('JobTryField')->find({job_try_field_id => $it}) or die error(404 => 'Invalid Field'); if (my $act = $route->{action}) { if ($act eq 'delete') { diff --git a/lib/App/Yath/Server/Controller/Lookup.pm b/lib/App/Yath/Server/Controller/Lookup.pm index ef35b90ef..5a29dc576 100644 --- a/lib/App/Yath/Server/Controller/Lookup.pm +++ b/lib/App/Yath/Server/Controller/Lookup.pm @@ -4,12 +4,12 @@ use warnings; our $VERSION = '2.000000'; -use Data::GUID; -use Scalar::Util qw/blessed/; use App::Yath::Server::Response qw/resp error/; -use App::Yath::Server::Util qw/share_dir find_job/; +use App::Yath::Util qw/share_dir/; +use App::Yath::Schema::Util qw/find_job/; use Test2::Harness::Util::JSON qw/encode_json/; + use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase qw/-title/; @@ -56,7 +56,7 @@ sub data { my $req = $self->{+REQUEST}; my $res = resp(200); - my @sources = qw/run jobs event/; + my @sources = qw/run job event/; my @out; @@ -90,19 +90,13 @@ sub lookup_run { my ($lookup, $state) = @_; return unless $lookup; - if (blessed($lookup)) { - $lookup = $lookup->run_id; - } - else { - $lookup = uuid_inflate($lookup); - } return if $state->{run}->{$lookup}++; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $rs = $schema->resultset('Run'); - my $run = eval { $rs->find({run_id => $lookup}) }; + my $run = $rs->find_by_id_or_uuid($lookup); return () unless $run; return ( @@ -110,36 +104,44 @@ sub lookup_run { ); } -sub lookup_jobs { +sub lookup_job { my $self = shift; - my ($lookup, $state) = @_; + my ($lookup, $state, $try_id) = @_; return unless $lookup; - if (blessed($lookup)) { - $lookup = $lookup->job_key; - } - else { - $lookup = uuid_inflate($lookup); - } return if $state->{job}->{$lookup}++; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $rs = $schema->resultset('Job'); + my $job = $rs->find_by_id_or_uuid($lookup); + return () unless $job; - my @out; + # FIXME: Make sure getitng only a specific job_try_id works + return ( + $self->lookup_run($job->run_id, $state), + encode_json({type => 'job', data => $job->glance_data(try_id => $try_id)}) . "\n", + ); +} - for my $key (qw/job_id job_key/) { - my $jobs = eval { $rs->search({$key => $lookup}) }; +sub lookup_job_try { + my $self = shift; + my ($lookup, $state) = @_; - while (my $job = eval { $jobs->next }) { - push @out => $self->lookup_run($job->run_id, $state); - push @out => encode_json({type => 'job', data => $job->glance_data }) . "\n"; - } - } + return unless $lookup; + + return if $state->{job_try}->{$lookup}++; + + my $schema = $self->schema; - return @out; + my $rs = $schema->resultset('JobTry'); + my $try = $rs->find({job_try_id => $lookup}); + return () unless $try; + + return ( + $self->lookup_job($try->job_id, $state, try => $try->job_try_id), + ); } sub lookup_event { @@ -147,24 +149,18 @@ sub lookup_event { my ($lookup, $state) = @_; return unless $lookup; - if (blessed($lookup)) { - $lookup = $lookup->event_id; - } - else { - $lookup = uuid_inflate($lookup); - } return if $state->{event}->{$lookup}++; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $rs = $schema->resultset('Event'); - my $event = eval { $rs->find({event_id => $lookup}) }; + my $event = $rs->find_by_id_or_uuid($lookup); return () unless $event; return ( - $self->lookup_jobs($event->job_key, $state), + $self->lookup_job_try($event->job_try_id, $state), encode_json({type => 'event', data => $event->line_data }) . "\n" ); } diff --git a/lib/App/Yath/Server/Controller/Project.pm b/lib/App/Yath/Server/Controller/Project.pm index 12aa7dddc..7b4992456 100644 --- a/lib/App/Yath/Server/Controller/Project.pm +++ b/lib/App/Yath/Server/Controller/Project.pm @@ -7,10 +7,10 @@ our $VERSION = '2.000000'; use Time::Elapsed qw/elapsed/; use List::Util qw/sum/; use Text::Xslate(); -use App::Yath::Server::Util qw/share_dir format_duration parse_duration is_invalid_subtest_name/; +use App::Yath::Util qw/share_dir/; +use App::Yath::Schema::Util qw/format_duration parse_duration is_invalid_subtest_name/; use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json decode_json/; -use App::Yath::Schema::UUID qw/uuid_deflate uuid_inflate/; use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase; @@ -26,7 +26,7 @@ sub users { my $self = shift; my ($project) = @_; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $dbh = $schema->storage->dbh; my $query = <<" EOT"; @@ -38,13 +38,12 @@ sub users { EOT my $sth = $dbh->prepare($query); - $sth->execute(uuid_deflate($project->project_id)) or die $sth->errstr; + $sth->execute($project->project_id) or die $sth->errstr; my $owner = $project->owner; my @out; for my $row (@{$sth->fetchall_arrayref // []}) { my ($user_id, $username) = @$row; - $user_id = uuid_inflate($user_id); my $is_owner = ($owner && $user_id eq $owner->user_id) ? 1 : 0; push @out => {user_id => $user_id, username => $username, owner => $is_owner}; } @@ -65,13 +64,10 @@ sub handle { my $n = $route->{n} // 25; my $stats = $route->{stats} // 0; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $project; $project = $schema->resultset('Project')->single({name => $it}); - if (my $uuid = uuid_inflate($it)) { - $project //= $schema->resultset('Project')->single({project_id => $uuid}); - } error(404 => 'Invalid Project') unless $project; return $self->html($req, $project, $n) @@ -186,22 +182,22 @@ sub get_add_query { return ('') unless $n || @$users || $range; - return ("AND run_ord > (SELECT MAX(run_ord) - ? FROM runs)\n", $n) + return ("AND run_id > (SELECT MAX(run_id) - ? FROM runs)\n", $n) unless @$users || $range; my @add_vals; my $user_query = 'user_id in (' . join(',' => map { '?' } @$users) . ')'; - push @add_vals => map { uuid_deflate($_) } @$users; + push @add_vals => @$users; return ("AND $user_query\n", @add_vals) unless $n || $range; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $dbh = $schema->storage->dbh; if ($range) { my $query = <<" EOT"; - SELECT min(run_ord) AS min, max(run_ord) AS max + SELECT min(run_id) AS min, max(run_id) AS max FROM runs WHERE project_id = ? AND added >= ? @@ -212,28 +208,28 @@ sub get_add_query { $end = parse_date($end); my $sth = $dbh->prepare($query); - $sth->execute(uuid_deflate($project->project_id), $start, $end) or die $sth->errstr; + $sth->execute($project->project_id, $start, $end) or die $sth->errstr; my $ords = $sth->fetchrow_hashref; - my $ord_query = "run_ord >= ? AND run_ord <= ?"; + my $ord_query = "run_id >= ? AND run_id <= ?"; push @add_vals => ($ords->{min}, $ords->{max}); return ("AND $user_query AND $ord_query", @add_vals) if @$users; return ("AND $ord_query", @add_vals); } my $query = <<" EOT"; - SELECT run_ord, run_id + SELECT run_id FROM reporting WHERE project_id = ? AND $user_query - GROUP BY run_ord, run_id - ORDER BY run_ord DESC + GROUP BY run_id + ORDER BY run_id DESC LIMIT ? EOT my $sth = $dbh->prepare($query); - $sth->execute(uuid_deflate($project->project_id), @add_vals, $n) or die $sth->errstr; + $sth->execute($project->project_id, @add_vals, $n) or die $sth->errstr; my @ids = map { $_->[1] } @{$sth->fetchall_arrayref}; return ('') unless @ids; @@ -246,7 +242,7 @@ sub _build_stat_run_list { my $self = shift; my ($project, $stat) = @_; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $dbh = $schema->storage->dbh; my ($add_query, @add_vals) = $self->get_add_query($project, $stat); @@ -260,11 +256,11 @@ sub _build_stat_run_list { EOT my $sth = $dbh->prepare($query); - $sth->execute(uuid_deflate($project->project_id), @add_vals) or die $sth->errstr; + $sth->execute($project->project_id, @add_vals) or die $sth->errstr; - my @ids = map { uuid_inflate($_->[0]) } @{$sth->fetchall_arrayref}; + my @ids = map { $_->[0] } @{$sth->fetchall_arrayref}; - my @items = map { $_->TO_JSON } $schema->resultset('Run')->search({run_id => {'-in' => \@ids}}, {order_by => {'-DESC' => 'run_ord'}})->all; + my @items = map { $_->TO_JSON } $schema->resultset('Run')->search({run_id => {'-in' => \@ids}}, {order_by => {'-DESC' => 'run_id'}})->all; $stat->{runs} = \@items; } @@ -273,7 +269,7 @@ sub _build_stat_expensive_files { my $self = shift; my ($project, $stat) = @_; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $dbh = $schema->storage->dbh; my ($add_query, @add_vals) = $self->get_add_query($project, $stat); @@ -282,7 +278,7 @@ sub _build_stat_expensive_files { SELECT test_files.filename AS filename, SUM(duration) AS total_duration, AVG(duration) AS average_duration, - COUNT(DISTINCT(run_ord)) AS runs, + COUNT(DISTINCT(run_id)) AS runs, COUNT(duration) AS tries, COUNT(DISTINCT(user_id)) AS users, SUM(pass) AS pass, @@ -299,7 +295,7 @@ sub _build_stat_expensive_files { EOT my $sth = $dbh->prepare($query); - $sth->execute(uuid_deflate($project->project_id), @add_vals) or die $sth->errstr; + $sth->execute($project->project_id, @add_vals) or die $sth->errstr; my @rows; for my $row (sort { $b->[1] <=> $a->[1] } @{$sth->fetchall_arrayref}) { @@ -326,7 +322,7 @@ sub _build_stat_expensive_subtests { my $self = shift; my ($project, $stat) = @_; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $dbh = $schema->storage->dbh; my ($add_query, @add_vals) = $self->get_add_query($project, $stat); @@ -336,7 +332,7 @@ sub _build_stat_expensive_subtests { subtest AS subtest, SUM(duration) AS total_duration, AVG(duration) AS average_duration, - COUNT(DISTINCT(run_ord)) AS runs, + COUNT(DISTINCT(run_id)) AS runs, COUNT(duration) AS tries, COUNT(DISTINCT(user_id)) AS users, SUM(pass) AS pass, @@ -344,7 +340,7 @@ sub _build_stat_expensive_subtests { SUM(abort) AS abort FROM reporting LEFT JOIN test_files USING(test_file_id) - WHERE project_id = ? + WHERE project_id = ? AND subtest IS NOT NULL AND test_file_id IS NOT NULL $add_query @@ -352,7 +348,7 @@ sub _build_stat_expensive_subtests { EOT my $sth = $dbh->prepare($query); - $sth->execute(uuid_deflate($project->project_id), @add_vals) or die $sth->errstr; + $sth->execute($project->project_id, @add_vals) or die $sth->errstr; my @rows; for my $row (sort { $b->[2] <=> $a->[2] } @{$sth->fetchall_arrayref}) { @@ -379,7 +375,7 @@ sub _build_stat_expensive_users { my $self = shift; my ($project, $stat) = @_; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $dbh = $schema->storage->dbh; my ($add_query, @add_vals) = $self->get_add_query($project, $stat); @@ -395,14 +391,14 @@ sub _build_stat_expensive_users { FROM reporting LEFT JOIN users USING(user_id) WHERE project_id = ? - AND job_key IS NULL + AND job_try_id IS NULL AND subtest IS NULL $add_query GROUP BY username EOT my $sth = $dbh->prepare($query); - $sth->execute(uuid_deflate($project->project_id), @add_vals) or die $sth->errstr; + $sth->execute($project->project_id, @add_vals) or die $sth->errstr; my @rows; for my $row (sort { $b->[1] <=> $a->[1] } @{$sth->fetchall_arrayref}) { @@ -429,15 +425,17 @@ sub _build_stat_user_summary { my $self = shift; my ($project, $stat) = @_; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $dbh = $schema->storage->dbh; my ($add_query, @add_vals) = $self->get_add_query($project, $stat); + print "HERE!\n"; + my $query = <<" EOT"; SELECT SUM(duration) AS total_duration, AVG(duration) AS average_duration, - COUNT(DISTINCT(run_ord)) AS runs, + COUNT(DISTINCT(run_id)) AS runs, COUNT(DISTINCT(user_id)) AS users, SUM(pass) AS pass, SUM(fail) AS fail, @@ -453,19 +451,27 @@ sub _build_stat_user_summary { WHERE project_id = ? $add_query GROUP BY has_file, has_subtest - ORDER BY has_File, has_subtest + ORDER BY has_file, has_subtest EOT my $sth = $dbh->prepare($query); - $sth->execute(uuid_deflate($project->project_id), @add_vals) or die $sth->errstr; + $sth->execute($project->project_id, @add_vals) or die $sth->errstr; - my $runs = $sth->fetchrow_hashref; + my ($runs, $jobs, $subs); + while (my $row = $sth->fetchrow_hashref) { + if ($row->{has_file} && $row->{has_subtest}) { + $subs = $row; + } + elsif ($row->{has_file}) { + $jobs = $row; + } + else { + $runs = $row; + } + } return $stat->{text} = "No run data." unless $runs->{runs}; - my $jobs = $sth->fetchrow_hashref; - my $subs = $sth->fetchrow_hashref; - $stat->{pair_sets} = []; push @{$stat->{pair_sets}} => [ @@ -510,7 +516,7 @@ sub _build_stat_uncovered { my $self = shift; my ($project, $stat) = @_; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $users = $stat->{users}; my $field = $schema->resultset('RunField')->search( @@ -519,7 +525,7 @@ sub _build_stat_uncovered { 'me.data' => { '!=' => undef }, 'run.project_id' => $project->project_id, 'run.has_coverage' => 1, - @$users ? (user_id => {'-in' => [map { uuid_deflate($_) } @$users]}) : () + @$users ? (user_id => {'-in' => $users}) : () }, { join => 'run', @@ -552,7 +558,7 @@ sub _build_stat_coverage { my $n = $stat->{n}; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $users = $stat->{users}; my @items = reverse $schema->resultset('RunField')->search( @@ -561,11 +567,11 @@ sub _build_stat_coverage { 'me.data' => { '!=' => undef }, 'run.project_id' => $project->project_id, 'run.has_coverage' => 1, - @$users ? (user_id => {'-in' => [map { uuid_deflate($_) } @$users]}) : () + @$users ? (user_id => {'-in' => $users}) : () }, { join => 'run', - order_by => {'-DESC' => 'run.added'}, + order_by => {'-DESC' => 'run_id'}, $n ? (rows => $n) : (), }, )->all; diff --git a/lib/App/Yath/Server/Controller/Query.pm b/lib/App/Yath/Server/Controller/Query.pm index f748ee16b..e0aafa94a 100644 --- a/lib/App/Yath/Server/Controller/Query.pm +++ b/lib/App/Yath/Server/Controller/Query.pm @@ -27,7 +27,7 @@ sub handle { my $req = $self->{+REQUEST}; my $res = resp(200); my $user = $req->user; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; die error(404 => 'Missing route') unless $route; my $it = $route->{name} or die error(400 => 'No query specified'); @@ -35,7 +35,7 @@ sub handle { my $arg = $route->{arg}; die error(400 => 'Missing Argument') if $spec->{args} && !defined($arg); - my $q = App::Yath::Schema::Queries->new(config => $self->{+CONFIG}); + my $q = App::Yath::Schema::Queries->new(config => $self->{+SCHEMA_CONFIG}); my $data = $q->$it($arg); $res->stream( diff --git a/lib/App/Yath/Server/Controller/ReRun.pm b/lib/App/Yath/Server/Controller/ReRun.pm index b82fc8d6d..a47bdc6db 100644 --- a/lib/App/Yath/Server/Controller/ReRun.pm +++ b/lib/App/Yath/Server/Controller/ReRun.pm @@ -6,7 +6,7 @@ our $VERSION = '2.000000'; use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json encode_pretty_json decode_json/; -use App::Yath::Schema::UUID qw/uuid_inflate/; + use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase; @@ -26,15 +26,11 @@ sub handle { my $project_name = $route->{project}; my $username = $route->{username}; - if ($run_id) { - $run_id = uuid_inflate($run_id) or die error(404 => "Invalid run id"); - } - error(404 => 'No source') unless $run_id || ($project_name && $username); - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $query = {}; - my $attrs = {order_by => {'-desc' => 'run_ord'}, rows => 1}; + my $attrs = {order_by => {'-desc' => 'run_id'}, rows => 1}; my $run; my $ok = eval { diff --git a/lib/App/Yath/Server/Controller/Recent.pm b/lib/App/Yath/Server/Controller/Recent.pm index 9c5756fd9..3aa2024bf 100644 --- a/lib/App/Yath/Server/Controller/Recent.pm +++ b/lib/App/Yath/Server/Controller/Recent.pm @@ -4,7 +4,6 @@ use warnings; our $VERSION = '2.000000'; -use Data::GUID; use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json/; @@ -25,7 +24,7 @@ sub handle { my $user_name = $route->{user}; my $count = $route->{count} || 10; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $runs = $schema->vague_run_search( username => $user_name, project_name => $project_name, diff --git a/lib/App/Yath/Server/Controller/Resources.pm b/lib/App/Yath/Server/Controller/Resources.pm index d24b39eb1..f5326760a 100644 --- a/lib/App/Yath/Server/Controller/Resources.pm +++ b/lib/App/Yath/Server/Controller/Resources.pm @@ -7,11 +7,11 @@ our $VERSION = '2.000000'; use DateTime; use Scalar::Util qw/blessed/; use App::Yath::Server::Response qw/resp error/; -use App::Yath::Server::Util qw/share_dir find_job/; +use App::Yath::Util qw/share_dir/; +use App::Yath::Schema::Util qw/find_job/; use App::Yath::Schema::DateTimeFormat qw/DTF/; use Test2::Harness::Util::JSON qw/encode_json decode_json/; use Test2::Util::Times qw/render_duration/; -use App::Yath::Schema::UUID qw/uuid_inflate uuid_deflate/; use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase qw/-title/; @@ -20,19 +20,25 @@ sub handle { my $self = shift; my ($route) = @_; - $self->{+TITLE} = 'YathUI'; + $self->{+TITLE} = 'Yath Run Resources'; my $req = $self->{+REQUEST}; - # Test run, Host, or resource instance - my $id = $route->{id} or die error(404 => 'No id provided'); + my $schema = $self->schema; - # Specific instant - my $batch = uuid_inflate($route->{batch}); + # Test run + my $run_id = $route->{run_id} or die error(404 => 'No run ID or UUID provided'); + my $run = $schema->resultset('Run')->find_by_id_or_uuid($run_id) or die error(404 => "Invalid Run"); + + die error(400 => "This run does not have any resource data.") unless $run->has_resources; + + my $res_ord = $route->{ord}; if ($route->{data}) { - return $self->data_stamps($req, $id) unless $batch; - return $self->data($req, $id, $batch); + return $self->res_max($run) if $route->{max}; + return $self->res_min($run) if $route->{min}; + return $self->res_ord($run, $res_ord) if $res_ord; + return $self->res_stream($req, $run); } my $res = resp(200); @@ -43,21 +49,21 @@ sub handle { my $tx = Text::Xslate->new(path => [share_dir('templates')]); - my $base_uri = $req->base->as_string; - my $stamp_uri = join '/' => $base_uri . 'resources', 'data', $id; - my $res_uri = join '/' => $base_uri . 'resources', $id; - $stamp_uri =~ s{/$}{}g; - $res_uri =~ s{/$}{}g; + my $base_uri = $req->base->as_string; + my $res_uri = join '/' => $base_uri . 'resources', $run_id; + my $data_uri = join '/' => $base_uri . 'resources', $run_id, 'data'; + $res_uri =~ s{/$}{}g; + $data_uri =~ s{/$}{}g; my $content = $tx->render( 'resources.tx', { - user => $req->user, - base_uri => $req->base->as_string, - stamp_uri => $stamp_uri, - res_uri => $res_uri, - tailing => $batch ? 0 : 1, - selected => $batch ? $batch : undef, + user => $req->user, + base_uri => $base_uri, + res_uri => $res_uri, + data_uri => $data_uri, + selected => $res_ord, + tailing => $res_ord ? 0 : 1, } ); @@ -65,205 +71,135 @@ sub handle { return $res; } -sub get_thing { +sub get_min_ord { my $self = shift; - my ($id) = @_; - - my $schema = $self->{+CONFIG}->schema; - - my ($thing, $stamp_start, $done_check); - my $search_args = {}; - my $stamp_args = {start => \$stamp_start}; - - my $host_rs = $schema->resultset('Host'); - my $res_rs = $schema->resultset('Resource'); - my $run_rs = $schema->resultset('Run'); + my ($run) = @_; - if (!$id || lc($id) eq 'global') { - $thing = undef; - $search_args->{global} = 1; - } - else { - my $uuid = uuid_inflate($id); - if ($uuid && eval { $thing = $run_rs->find({run_id => $uuid}) }) { - $search_args->{run_id} = $uuid; - $done_check = sub { - return 1 if $thing->complete; - return 0; - }; - } - elsif (($uuid && eval { $thing = $host_rs->find({host_id => $uuid}) }) || eval { $thing = $host_rs->find({hostname => $id}) }) { - $search_args->{host_id} = $thing->host_id; - } - else { - die error(404 => 'Invalid Job ID or Host ID'); - } - } + my $schema = $self->schema; + my $dbh = $schema->storage->dbh; - return ($thing, $search_args, $stamp_args, $done_check); + my $sth = $dbh->prepare("SELECT MIN(resource_ord) FROM resources WHERE run_id = ?"); + $sth->execute($run->run_id); + my $row = $sth->fetchrow_arrayref() or return 0; + return $row->[0] // 0; } -sub get_stamps { +sub get_max_ord { my $self = shift; - my %params = @_; - - my $search_args = $params{search_args} || {}; - my $start = $params{start}; + my ($run) = @_; - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $dbh = $schema->storage->dbh; - my $fields = ""; - my @vals; - if ($search_args->{run_id}) { - $fields = "run_id = ?"; - push @vals => uuid_deflate($search_args->{run_id}); - } - elsif ($search_args->{host_id}) { - $fields = "host_id = ?"; - push @vals => uuid_deflate($search_args->{host_id}); - } - - if ($$start) { - $fields .= " AND stamp > ?"; - push @vals => $$start; - } - - my $sth = $dbh->prepare("SELECT resource_batch_id, stamp FROM resource_batch WHERE " . $fields . " ORDER BY stamp ASC"); - $sth->execute(@vals) or die $sth->errstr; - my $rows = $sth->fetchall_arrayref; - - return unless @$rows; - - $_->[0] = uuid_inflate($_->[0]) for @$rows; - - $$start = $rows->[-1]->[1]; - - return $rows; + my $sth = $dbh->prepare("SELECT MAX(resource_ord) FROM resources WHERE run_id = ?"); + $sth->execute($run->run_id); + my $row = $sth->fetchrow_arrayref() or return 0; + return $row->[0] // 0; } -sub data_stamps { +sub res_max { my $self = shift; - my ($req, $id) = @_; + my ($run) = @_; my $res = resp(200); - my ($thing, $search_args, $stamp_args, $done_check) = $self->get_thing($id); - - my ($complete, @out); - - if (my $run_id = $search_args->{run_id}) { - push @out => { run_id => $run_id }; - } - if (my $host_id = $search_args->{host_id}) { - push @out => { host_id => $host_id }; - } - - my $start = time; - my $advance = sub { - return 0 if @out; - return 1 if $complete; - return 1 if (time - $start) > 600; - - if ($thing) { - if (my $stamps = $self->get_stamps(%$stamp_args, search_args => $search_args)) { - push @out => {stamps => $stamps}; - } - - # Finish if run is done - if ($done_check && $done_check->()) { - push @out => {complete => 1}; - } - - return 0; - } - - push @out => {complete => 1}; - return 1; - }; - - $res->stream( - env => $req->env, - content_type => 'application/x-jsonl; charset=utf-8', - done => $advance, + $res->content_type('text/plain'); + $res->body($self->get_max_ord($run)); + return $res; +} - fetch => sub { - return () if $complete; +sub res_min { + my $self = shift; + my ($run) = @_; - $advance->() unless @out; + my $res = resp(200); + $res->content_type('text/plain'); + $res->body($self->get_min_ord($run)); + return $res; +} - my $item = shift @out or return (); - $complete = 1 if $item->{complete}; +sub get_up_to { + my $self = shift; + my ($run, $ord) = @_; - return encode_json($item) . "\n"; - }, - ); + my $schema = $self->schema; + my $dbh = $schema->storage->dbh; - return $res; + my $sth = $dbh->prepare(<<' EOT'); + SELECT name, stamp, resource_ord, data + FROM resources + JOIN resource_types USING(resource_type_id) + WHERE resource_id IN ( + SELECT MAX(resource_id) + FROM resources + WHERE run_id = ? + AND resource_ord <= ? + GROUP BY resource_type_id + ); + EOT + + $sth->execute($run->run_id, $ord); + + return {ord => $ord, resources => [map { $_->{data} = decode_json($_->{data}); $_ } @{$sth->fetchall_arrayref({})}]}; } -sub data { +sub res_ord { my $self = shift; - my ($req, $id, $batch) = @_; + my ($run, $ord) = @_; - my $res = resp(200); - my ($thing, $search_args, $stamp_args, $done_check) = $self->get_thing($id); + my $data = $self->get_up_to($run, $ord); + my $res = resp(200); $res->content_type('application/json'); - $res->raw_body({ - resources => $self->render_stamp_resources(search_args => $search_args, batch => $batch), - }); - + $res->raw_body($data); return $res; } -sub render_stamp_resources { +sub res_stream { my $self = shift; - my %params = @_; - - my $search_args = $params{search_args}; - my $batch_id = uuid_inflate($params{batch}); + my ($req, $run) = @_; - my $schema = $self->{+CONFIG}->schema; - my $res_rs = $schema->resultset('Resource'); + my $current; + my $complete = 0; - my @res_list; - my $resources = $res_rs->search({resource_batch_id => $batch_id}, {order_by => {'-asc' => 'batch_ord'}}); - while (my $res = $resources->next) { - push @res_list => $self->render_resource($res); - } + my $min = $self->get_min_ord($run); - return \@res_list; -} + my $run_uuid = $run->run_uuid; + my $run_sent = 0; -sub render_resource { - my $self = shift; - my ($r) = @_; + my $res = resp(200); + $res->stream( + env => $req->env, + content_type => 'application/x-jsonl; charset=utf-8', + done => sub { $complete }, - my $data = $r->data; + fetch => sub { + return () if $complete; - for my $group (@{$data || []}) { - for my $table (@{$group->{tables} || []}) { - for my $row (@{$table->{rows} || []}) { - my @formats = @{$table->{format} || []}; + unless ($run_sent) { + $run_sent = 1; + return encode_json({run_uuid => $run_uuid}) . "\n"; + } - for my $item (@{$row || []}) { - my $format = shift @formats or next; + $run->discard_changes; + my $run_complete = $run->complete; + my $max = $self->get_max_ord($run); + if (defined $current && $max <= $current) { + $complete = 1 if $run_complete; + return unless $complete; + return encode_json({min => $min, max => $max, complete => $complete, data => undef}) . "\n"; + } - unless ($format eq 'duration') { - $item = "$item (unsupported format '$format')"; - next; - } + $min //= $self->get_min_ord($run); + my $data = $self->get_up_to($run, $max); + $current = $max; - $item = render_duration($item); - } - } - } - } + return encode_json({min => $min, max => $max, complete => $complete, data => $data}) . "\n"; + }, + ); - return {resource => $r->module, groups => $r->data}; + return $res; } - 1; __END__ diff --git a/lib/App/Yath/Server/Controller/Run.pm b/lib/App/Yath/Server/Controller/Run.pm index d7e19076e..bb74a7655 100644 --- a/lib/App/Yath/Server/Controller/Run.pm +++ b/lib/App/Yath/Server/Controller/Run.pm @@ -4,13 +4,12 @@ use warnings; our $VERSION = '2.000000'; -use Data::GUID; use List::Util qw/max/; use Text::Xslate(qw/mark_raw/); -use App::Yath::Server::Util qw/share_dir/; +use App::Yath::Util qw/share_dir/; use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json decode_json/; -use App::Yath::Schema::UUID qw/uuid_inflate/; + use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase qw/-title/; @@ -28,14 +27,13 @@ sub handle { my $run; - if ($self->{+CONFIG}->single_run) { + if ($self->single_run) { $run = $user->runs->first or die error(404 => 'Invalid run'); } else { my $it = $route->{id} or die error(404 => 'No id'); - $it = uuid_inflate($it) or die error(404 => "Invalid run id"); - my $schema = $self->{+CONFIG}->schema; - $run = $schema->resultset('Run')->find({run_id => $it}) or die error(404 => 'Invalid Run'); + my $schema = $self->schema; + $run = $schema->resultset('Run')->find_by_id_or_uuid($it) or die error(404 => 'Invalid Run'); } if (my $act = $route->{action}) { @@ -52,32 +50,6 @@ sub handle { } elsif ($act eq 'delete') { die error(400 => "Cannot delete a pinned run") if $run->pinned; - - $run->coverages->delete; - $run->reportings->delete; - - my $batches = $run->resource_batches; - while (my $batch = $batches->next) { - $batch->resources->delete; - $batch->delete; - } - - my $jobs = $run->jobs; - - while (my $job = $jobs->next()) { - my $has_binary = $job->events->search({has_binary => 1}); - while (my $e = $has_binary->next()) { - $has_binary->binaries->delete; - $e->delete; - } - - $job->events->delete; - $job->job_fields->delete; - $job->delete; - } - - $run->run_fields->delete; - $run->sweeps->delete; $run->delete; } } diff --git a/lib/App/Yath/Server/Controller/RunField.pm b/lib/App/Yath/Server/Controller/RunField.pm index fc3413ef1..19c5cfa80 100644 --- a/lib/App/Yath/Server/Controller/RunField.pm +++ b/lib/App/Yath/Server/Controller/RunField.pm @@ -4,13 +4,12 @@ use warnings; our $VERSION = '2.000000'; -use Data::GUID; use List::Util qw/max/; use Text::Xslate(qw/mark_raw/); -use App::Yath::Server::Util qw/share_dir/; +use App::Yath::Util qw/share_dir/; use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json decode_json/; -use App::Yath::Schema::UUID qw/uuid_inflate/; + use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase qw/-title/; @@ -26,8 +25,8 @@ sub handle { die error(404 => 'Missing route') unless $route; - my $it = uuid_inflate($route->{id}) or die error(404 => 'No id'); - my $schema = $self->{+CONFIG}->schema; + my $it = $route->{id} or die error(404 => 'No id'); + my $schema = $self->schema; my $field = $schema->resultset('RunField')->find({run_field_id => $it}) or die error(404 => 'Invalid Field'); if (my $act = $route->{action}) { diff --git a/lib/App/Yath/Server/Controller/Stream.pm b/lib/App/Yath/Server/Controller/Stream.pm index 46c302746..abce5ef9c 100644 --- a/lib/App/Yath/Server/Controller/Stream.pm +++ b/lib/App/Yath/Server/Controller/Stream.pm @@ -4,11 +4,11 @@ use warnings; our $VERSION = '2.000000'; -use Data::GUID; use List::Util qw/max/; use Scalar::Util qw/blessed/; -use App::Yath::Server::Util qw/find_job/; -use App::Yath::Schema::UUID qw/uuid_inflate/; +use App::Yath::Schema::Util qw/find_job_and_try format_uuid_for_db/; +use Test2::Util::UUID qw/looks_like_uuid/; + use App::Yath::Server::Response qw/resp error/; use Test2::Harness::Util::JSON qw/encode_json/; use JSON::PP(); @@ -18,6 +18,7 @@ use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase qw{ 100; @@ -46,15 +47,16 @@ sub handle { } $res->stream( - env => $req->env, - content_type => 'application/x-jsonl; charset=utf-8', + env => $req->env, cache => $cache, + content_type => 'application/x-jsonl; charset=utf-8', + done => sub { my @keep; while (my $set = shift @sets) { my ($check) = @$set; - next if $check->(); # Next if done + next if $check->(); # Next if done # Not done, keep it push @keep => $set; @@ -65,7 +67,9 @@ sub handle { return @sets ? 0 : 1; }, - fetch => sub { map { $_->[1]->() } @sets }, + fetch => sub { + map { $_->[1]->() } @sets; + }, ); return $res; @@ -75,25 +79,9 @@ sub stream_runs { my $self = shift; my ($req, $route) = @_; - my $schema = $self->{+CONFIG}->schema; - - my $opts = { - remove_columns => [qw/log_data run_fields.data parameters/], - - join => [qw/user_join project run_fields/], - '+columns' => { - 'prefetched_fields' => \'1', - 'run_fields.run_field_id' => 'run_fields.run_field_id', - 'run_fields.name' => 'run_fields.name', - 'run_fields.details' => 'run_fields.details', - 'run_fields.raw' => 'run_fields.raw', - 'run_fields.link' => 'run_fields.link', - 'run_fields.data', => \"run_fields.data IS NOT NULL", - 'user' => \'user_join.username', - 'project' => \'project.name', - }, - }; + my $schema = $self->schema; + my $opts = {remove_columns => [qw/parameters/]}; my %params = ( type => 'run', @@ -101,8 +89,8 @@ sub stream_runs { track_status => 1, id_field => 'run_id', - ord_field => 'added', - sort_field => 'added', + ord_field => 'run_id', + sort_field => 'run_id', search_base => $schema->resultset('Run'), initial_limit => RUN_LIMIT, @@ -111,39 +99,29 @@ sub stream_runs { timeout => 60 * 30, # 30 min. ); - my $id = $route->{id}; - my $uuid = uuid_inflate($id); - my $run_id = $route->{run_id}; - my ($project, $user); + my $run_id = $route->{run_id}; + my $user_id = $route->{user_id}; + my $project_id = $route->{project_id}; - if ($id) { - my $p_rs = $schema->resultset('Project'); - $project //= eval { $p_rs->find({name => $id}) }; - $project //= eval { $p_rs->find({project_id => $uuid}) }; - - if ($project) { - $params{search_base} = $params{search_base}->search_rs({'me.project_id' => $project->project_id}); - } - else { - my $u_rs = $schema->resultset('User'); - $user //= eval { $u_rs->find({username => $id}) }; - $user //= eval { $u_rs->find({user_id => $uuid}) }; - - if ($user) { - $uuid = uuid_inflate($user->user_id); - $params{search_base} = $params{search_base}->search_rs({'me.user_id' => $user->user_id}); - } - else { - $run_id //= $uuid; - } - } - } + + my ($project, $user, $run); if($run_id) { - $run_id = uuid_inflate($run_id) or die error(404 => "Invalid run id"); + $params{id_field} = 'run_uuid' if looks_like_uuid($run_id); return $self->stream_single(%params, id => $run_id); } + if ($project_id) { + my $p_rs = $schema->resultset('Project'); + $project = eval { $p_rs->find({name => $project_id}) } // eval { $p_rs->find({project_id => $project_id}) } // die error(404 => 'Invalid Project'); + $params{search_base} = $params{search_base}->search_rs({'me.project_id' => $project->project_id}); + } + elsif ($user_id) { + my $u_rs = $schema->resultset('User'); + $user = eval { $u_rs->find({username => $user_id}) } // eval { $u_rs->find({user_id => $user_id}) } // die error(404 => 'Invalid User'); + $params{search_base} = $params{search_base}->search_rs({'me.user_id' => $user->user_id}); + } + return $self->stream_set(%params); } @@ -153,16 +131,7 @@ sub stream_jobs { my $run = $self->{+RUN} // return; - my $opts = { - join => 'test_file', - remove_columns => [qw/stdout stderr parameters/], - '+select' => [ - 'test_file.filename AS file', - ], - '+as' => [ - 'file', - ], - }; + my $opts = {}; my %params = ( type => 'job', @@ -171,20 +140,21 @@ sub stream_jobs { req => $req, track_status => 1, - id_field => 'job_key', - ord_field => 'job_ord', + id_field => 'job_id', + ord_field => 'job_id', method => 'glance_data', search_base => scalar($run->jobs), custom_opts => $opts, - order_by => [{'-desc' => 'status'}, {'-desc' => [qw/job_try job_ord name/]}], + order_by => [{'-desc' => 'status'}, {'-desc' => [qw/job_try job_id name/]}], ); if (my $job_uuid = $route->{job}) { - $job_uuid = uuid_inflate($job_uuid) or die error(404 => "Invalid job id"); - - my $schema = $self->{+CONFIG}->schema; - return $self->stream_single(%params, item => find_job($schema, $job_uuid, $route->{try})); + my $schema = $self->schema; + my ($job, $try) = find_job_and_try($schema, $job_uuid, $route->{try}); + $self->{+JOB} = $job; + $self->{+TRY} = $try; + return $self->stream_single(%params, item => $job); } return $self->stream_set(%params); @@ -195,37 +165,25 @@ sub stream_events { my ($req, $route) = @_; my $job = $self->{+JOB} // return; + my $try = $self->{+TRY} // return; # we only stream nested events when the job is still running - my $query = $job->complete ? {nested => 0} : undef; - - my $opts = { - remove_columns => ['orphan'], - '+select' => [ - 'facets IS NOT NULL AS has_facets', - 'orphan IS NOT NULL AS has_orphan', - ], - '+as' => [ - 'has_facets', - 'has_orphan', - ], - }; + my $query = $try->complete ? {nested => 0} : undef; return $self->stream_set( type => 'event', - parent => $job, + parent => $try, req => $req, track_status => 0, id_field => 'event_id', - ord_field => 'insert_ord', - sort_field => 'event_ord', + ord_field => 'event_idx', + sort_field => 'event_idx', sort_dir => '-asc', method => 'line_data', custom_query => $query, - custom_opts => $opts, - search_base => scalar($job->events), + search_base => scalar($try->events), ); } @@ -246,6 +204,7 @@ sub stream_single { $it = $params{item} or die error(404 => "Invalid Item"); } else { + $id = format_uuid_for_db($id) if $id_field =~ m/_uuid$/; $it = $search_base->search({%$custom_query, "me.$id_field" => $id}, $custom_opts)->first or die error(404 => "Invalid $type"); } $self->{$type} = $it; @@ -266,7 +225,7 @@ sub stream_single { return if $unchanged; - my $data = $method ? $it->$method : $it->TO_JSON; + my ($data) = $method ? $it->$method : $it->TO_JSON; return encode_json({type => $type, update => $update, data => $data}) . "\n"; }, ]; @@ -293,10 +252,12 @@ sub stream_set { my $order_by = $params{order_by} // $sort_field ? {$sort_dir => $sort_field} : croak "Must specify either 'order_by' or 'sort_field'"; my $items = $search_base->search($custom_query, {%$custom_opts, order_by => $order_by, $limit ? (rows => $limit) : ()}); + my (@buffer, $buffer_item); my $start = time; my $ord; my $incomplete = {}; + my $update; return [ sub { @@ -311,10 +272,10 @@ sub stream_set { return 0; }, sub { - unless ($items) { + unless ($items || @buffer) { my $val; if (blessed($ord) && $ord->isa('DateTime')) { - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; my $dtf = $schema->storage->datetime_parser; $val = $dtf->format_datetime($ord); } @@ -324,10 +285,11 @@ sub stream_set { my $query = { ($custom_query ? %$custom_query : ()), - $ord_field => {'>' => $val}, + defined($val) ? ($ord_field => {'>' => $val}) : (), }; my @ids = $track ? keys %$incomplete : (); + @ids = map { format_uuid_for_db($_) } @ids if $id_field =~ m/_uuid$/; $query = [$query, {"me.$id_field" => { -in => \@ids }}] if @ids; $items = $search_base->search( @@ -336,28 +298,42 @@ sub stream_set { ); } - while (my $item = $items->next()) { - $ord = $item->$ord_field; + while (1) { + my ($item); - my $update = JSON::PP::false; - if ($track) { - my $id = $item->$id_field; - if (my $old = $incomplete->{$id}) { - $update = JSON::PP::true; - # Nothing has changed, no need to send it. - next if $old->sig eq $item->sig; + if (@buffer) { + $item = $buffer_item; + } + else { + $item = $items->next() or last; + + $ord = max($ord || 0, $item->$ord_field); + + $update = JSON::PP::false; + + if ($track) { + my $id = $item->$id_field; + if (my $old = $incomplete->{$id}) { + $update = JSON::PP::true; + # Nothing has changed, no need to send it. + next if $old->sig eq $item->sig; + } + + if ($item->complete) { + delete $incomplete->{$id}; + } + else { + $incomplete->{$id} = $item; + } } + } - if ($item->complete) { - delete $incomplete->{$id}; - } - else { - $incomplete->{$id} = $item; - } + unless (@buffer) { + @buffer = $method ? $item->$method : $item->TO_JSON; + $buffer_item = $item; } - my $data = $method ? $item->$method : $item->TO_JSON; - return encode_json({type => $type, update => $update, data => $data}) . "\n"; + return encode_json({type => $type, update => $update, data => shift(@buffer)}) . "\n"; } $items = undef; diff --git a/lib/App/Yath/Server/Controller/Sweeper.pm b/lib/App/Yath/Server/Controller/Sweeper.pm index 3a0bdc5b1..961546e6e 100644 --- a/lib/App/Yath/Server/Controller/Sweeper.pm +++ b/lib/App/Yath/Server/Controller/Sweeper.pm @@ -28,7 +28,7 @@ sub handle { my $sweeper = App::Yath::Schema::Sweeper->new( interval => $interval, - config => $self->{+CONFIG}, + config => $self->{+SCHEMA_CONFIG}, ); my $purged = $sweeper->sweep; diff --git a/lib/App/Yath/Server/Controller/Upload.pm b/lib/App/Yath/Server/Controller/Upload.pm index a6b86306d..fd8b02f08 100644 --- a/lib/App/Yath/Server/Controller/Upload.pm +++ b/lib/App/Yath/Server/Controller/Upload.pm @@ -6,13 +6,13 @@ our $VERSION = '2.000000'; use Text::Xslate(); -use App::Yath::Schema::UUID qw/uuid_inflate/; use Test2::Harness::Util::JSON qw/decode_json/; use Test2::Harness::Util qw/open_file/; + use App::Yath::Schema::Queries(); -use App::Yath::Server::Util qw/share_dir/; +use App::Yath::Util qw/share_dir/; use App::Yath::Server::Response qw/resp error/; use parent 'App::Yath::Server::Controller'; @@ -44,9 +44,9 @@ sub handle { 'upload.tx', { base_uri => $req->base->as_string, - single_user => $self->{+CONFIG}->single_user, + single_user => $self->single_user, user => $user, - projects => App::Yath::Schema::Queries->new(config => $self->{+CONFIG})->projects, + projects => App::Yath::Schema::Queries->new(config => $self->{+SCHEMA_CONFIG})->projects, } ); @@ -95,11 +95,12 @@ sub process_form { open(my $fh, '<:raw', $tmp) or die "Could not open uploaded file '$tmp': $!"; my $run = $self->schema->resultset('Run')->create({ - $run_id ? (run_id => $run_id) : (), + $run_id ? (run_uuid => $run_id) : (), user_id => ref($user) ? $user->user_id : 1, project_id => $project->project_id, mode => $mode, status => 'pending', + canon => 1, log_file => { name => $file, @@ -118,7 +119,7 @@ sub api_user { return unless $key_val; my $schema = $self->schema; - my $key = $schema->resultset('ApiKey')->find({value => uuid_inflate($key_val)}) + my $key = $schema->resultset('ApiKey')->find({value => $key_val}) or return undef; return undef unless $key->status eq 'active'; diff --git a/lib/App/Yath/Server/Controller/User.pm b/lib/App/Yath/Server/Controller/User.pm index cf5c6fcd8..d87aa2c60 100644 --- a/lib/App/Yath/Server/Controller/User.pm +++ b/lib/App/Yath/Server/Controller/User.pm @@ -5,9 +5,9 @@ use warnings; our $VERSION = '2.000000'; use Text::Xslate(); -use App::Yath::Server::Util qw/share_dir/; +use App::Yath::Util qw/share_dir/; use App::Yath::Server::Response qw/resp error/; -use App::Yath::Schema::UUID qw/uuid_inflate/; + use Email::Sender::Simple qw(sendmail); use Email::Simple; @@ -66,13 +66,14 @@ sub process_form { # This one we allow non-post, all others need post. if ('logout' eq $action) { $req->session_host->update({'user_id' => undef}); + $req->set_user(undef); return $res->add_msg("You have been logged out."); } elsif ($action eq 'verify') { - my $evcode_id = $p->{verification_code} + my $evcode = $p->{verification_code} or return $res->add_error("Invalid verification code"); - my $code = $schema->resultset('EmailVerificationCode')->find({evcode_id => uuid_inflate($evcode_id)}) + my $code = $schema->resultset('EmailVerificationCode')->find({evcode => $evcode}) or return $res->add_error("Invalid verification code"); my $email = $code->email; @@ -95,6 +96,7 @@ sub process_form { unless $user && $user->verify_password($password); $req->session_host->update({'user_id' => $user->user_id}); + $req->set_user($user); return $res->add_msg("You have been logged in."); } @@ -145,11 +147,11 @@ sub process_form { return $res->add_msg("Password Changed."); } - if ($p->{api_key_id} && $KEY_ACTION_MAP{$action}) { - my $key_id = uuid_inflate($p->{api_key_id}); + if ($p->{api_key} && $KEY_ACTION_MAP{$action}) { + my $api_key = $p->{api_key}; my $user = $req->user or return $res->add_error("You must be logged in"); - my $key = $schema->resultset('ApiKey')->find({api_key_id => $key_id, user_id => $user->user_id}); + my $key = $schema->resultset('ApiKey')->find({value => $api_key, user_id => $user->user_id}); return $res->add_error("Invalid key") unless $key; $key->update({status => $KEY_ACTION_MAP{$action}}); @@ -157,7 +159,6 @@ sub process_form { } if (my $email_id = $p->{email_id}) { - $email_id = uuid_inflate($email_id) or return $res->add_error("Invalid email id"); my $user = $req->user or return $res->add_error("You must be logged in"); my $email = $schema->resultset('Email')->find({email_id => $email_id, user_id => $user->user_id}); return $res->add_error("Invalid Email") unless $email; @@ -200,15 +201,15 @@ sub send_verification_code { my $schema = $self->schema; - my $code = $schema->resultset('EmailVerificationCode')->find_or_create({email_id => $email->email_id}); - my $text = $code->evcode_id; + my $our_email = $schema->config('email') or die "System email address is not set"; - my $config = $self->{+CONFIG}; + my $code = $schema->resultset('EmailVerificationCode')->find_or_create({email_id => $email->email_id}); + my $text = $code->evcode; my $msg = Email::Simple->create( header => [ To => $email->address, - From => $config->email, + From => $our_email, Subject => "Email verification code", ], body => "Verification code: $text\n", diff --git a/lib/App/Yath/Server/Controller/View.pm b/lib/App/Yath/Server/Controller/View.pm index 3e1c2eda7..ad60a63f7 100644 --- a/lib/App/Yath/Server/Controller/View.pm +++ b/lib/App/Yath/Server/Controller/View.pm @@ -4,11 +4,11 @@ use warnings; our $VERSION = '2.000000'; -use Data::GUID; use Text::Xslate(qw/mark_raw/); -use App::Yath::Server::Util qw/share_dir find_job/; +use App::Yath::Util qw/share_dir/; +use App::Yath::Schema::Util qw/find_job_and_try/; use App::Yath::Server::Response qw/resp error/; -use App::Yath::Schema::UUID qw/uuid_inflate/; + use parent 'App::Yath::Server::Controller'; use Test2::Harness::Util::HashBase qw/-title/; @@ -28,57 +28,49 @@ sub handle { $res->add_js('eventtable.js'); $res->add_js('view.js'); - my $schema = $self->{+CONFIG}->schema; + my $schema = $self->schema; - my $id = $route->{id}; - my $uuid = uuid_inflate($id); - my $run_id = $route->{run_id}; - my ($project, $user); + use Data::Dumper; + print Dumper($route); + my $run_id = $route->{run_id}; + my $user_id = $route->{user_id}; + my $project_id = $route->{project_id}; + my ($project, $user, $run); - if ($id) { + my @url; + if ($project_id) { my $p_rs = $schema->resultset('Project'); - $project //= eval { $p_rs->find({name => $id}) }; - $project //= eval { $p_rs->find({project_id => $uuid}) }; - - if ($project) { - $uuid = uuid_inflate($project->project_id); - $self->{+TITLE} .= ">" . $project->name; - } - else { - my $u_rs = $schema->resultset('User'); - $user //= eval { $u_rs->find({username => $id}) }; - $user //= eval { $u_rs->find({user_id => $uuid}) }; - - if ($user) { - $uuid = uuid_inflate($user->user_id); - $self->{+TITLE} .= ">" . $user->username; - } - else { - $run_id //= $uuid; - } - } + $project = eval { $p_rs->find({name => $project_id}) } // eval { $p_rs->find({project_id => $project_id}) } // die error(404 => 'Invalid Project'); + $self->{+TITLE} .= ">" . $project->name; + @url = ('project', $project_id); } + elsif ($user_id) { + my $u_rs = $schema->resultset('User'); + $user = eval { $u_rs->find({username => $user_id}) } // eval { $u_rs->find({user_id => $user_id}) } // die error(404 => 'Invalid User'); + $self->{+TITLE} .= ">" . $user->username; + @url = ('user', $user_id); + } + elsif($run_id) { + push @url => $run_id; - if($run_id) { - $run_id = uuid_inflate($run_id) or die error(404 => 'Invalid Run'); - - my $run = eval { $schema->resultset('Run')->find({run_id => $run_id}) } or die error(404 => 'Invalid Run'); + $run = eval { $schema->resultset('Run')->find_by_id_or_uuid($run_id) } or die error(404 => 'Invalid Run'); $self->{+TITLE} .= ">" . $run->project->name; - } - my $job_uuid = $route->{job}; - my $job_try = $route->{try}; + my $job_try = $route->{try}; + + if (my $job_uuid = $route->{job}) { + my ($job, $try) = find_job_and_try($schema, $job_uuid, $job_try) or die error(404 => 'Invalid Job'); + $self->{+TITLE} .= ">" . ($job->shortest_file // 'HARNESS'); + push @url => $job_uuid; + } - if ($job_uuid) { - $job_uuid = uuid_inflate($job_uuid) or die error(404 => 'Invalid Job'); - my $job = find_job($schema, $job_uuid, $job_try) or die error(404 => 'Invalid Job'); - $self->{+TITLE} .= ">" . ($job->shortest_file // 'HARNESS'); + push @url => $job_try if $job_try; } my $tx = Text::Xslate->new(path => [share_dir('templates')]); my $base_uri = $req->base->as_string; - my $stream_uri = join '/' => $base_uri . 'stream', grep {length $_} ($uuid // $run_id), $job_uuid, $job_try; + my $stream_uri = join '/' => $base_uri . 'stream', @url; my $content = $tx->render( 'view.tx', diff --git a/lib/App/Yath/Server/Plack.pm b/lib/App/Yath/Server/Plack.pm new file mode 100644 index 000000000..172da97c9 --- /dev/null +++ b/lib/App/Yath/Server/Plack.pm @@ -0,0 +1,347 @@ +package App::Yath::Server::Plack; +use strict; +use warnings; + +our $VERSION = '2.000000'; + +use Router::Simple; +use DateTime; + +use Text::Xslate(qw/mark_raw/); +use Scalar::Util qw/blessed/; +use Carp qw/croak/; + +use Plack::Builder; +use Plack::App::Directory; +use Plack::App::File; + +use App::Yath::Server::Request; +use App::Yath::Server::Controller::Upload; +use App::Yath::Server::Controller::Recent; +use App::Yath::Server::Controller::User; +use App::Yath::Server::Controller::Run; +use App::Yath::Server::Controller::RunField; +use App::Yath::Server::Controller::Job; +use App::Yath::Server::Controller::JobTryField; +use App::Yath::Server::Controller::Download; +use App::Yath::Server::Controller::Sweeper; +use App::Yath::Server::Controller::Project; +use App::Yath::Server::Controller::Resources; + +use App::Yath::Server::Controller::Stream; +use App::Yath::Server::Controller::View; +use App::Yath::Server::Controller::Lookup; + +use App::Yath::Server::Controller::Query; +use App::Yath::Server::Controller::Events; + +use App::Yath::Server::Controller::Durations; +use App::Yath::Server::Controller::Coverage; +use App::Yath::Server::Controller::Files; +use App::Yath::Server::Controller::ReRun; + +use App::Yath::Server::Controller::Interactions; +use App::Yath::Server::Controller::Binary; + +use App::Yath::Server::Response qw/resp error/; + +use App::Yath::Util qw/share_dir/; + +use Test2::Harness::Util::JSON qw/encode_json decode_json/; + +use Test2::Harness::Util::HashBase qw{ + {+SCHEMA_CONFIG}; + + my $schema = $self->schema; + $self->{+SINGLE_RUN} //= $schema->config('single_run'); + $self->{+SINGLE_USER} //= $schema->config('single_user'); +} + +sub schema { $_[0]->{+SCHEMA_CONFIG}->schema } + +sub router { + my $self = shift; + + return $self->{+ROUTER} if $self->{+ROUTER}; + + my $router = Router::Simple->new; + my $schema = $self->schema; + + $router->connect('/' => {controller => 'App::Yath::Server::Controller::View'}); + + $router->connect('/upload' => {controller => 'App::Yath::Server::Controller::Upload'}) + unless $self->single_run; + + $router->connect('/user' => {controller => 'App::Yath::Server::Controller::User'}) + unless $self->single_user; + + $router->connect('/resources/:run_id' => {controller => 'App::Yath::Server::Controller::Resources'}); + $router->connect('/resources/:run_id/:ord' => {controller => 'App::Yath::Server::Controller::Resources'}); + $router->connect('/resources/:run_id/data/stream' => {controller => 'App::Yath::Server::Controller::Resources', data => 1}); + $router->connect('/resources/:run_id/data/min' => {controller => 'App::Yath::Server::Controller::Resources', data => 1, min => 1}); + $router->connect('/resources/:run_id/data/max' => {controller => 'App::Yath::Server::Controller::Resources', data => 1, max => 1}); + $router->connect('/resources/:run_id/data/:ord' => {controller => 'App::Yath::Server::Controller::Resources', data => 1}); + + $router->connect('/interactions/:id' => {controller => 'App::Yath::Server::Controller::Interactions'}); + $router->connect('/interactions/:id/:context' => {controller => 'App::Yath::Server::Controller::Interactions'}); + $router->connect('/interactions/data/:id' => {controller => 'App::Yath::Server::Controller::Interactions', data => 1}); + $router->connect('/interactions/data/:id/:context' => {controller => 'App::Yath::Server::Controller::Interactions', data => 1}); + + $router->connect('/project/:id' => {controller => 'App::Yath::Server::Controller::Project'}); + $router->connect('/project/:id/stats' => {controller => 'App::Yath::Server::Controller::Project', stats => 1}); + $router->connect('/project/:id/:n' => {controller => 'App::Yath::Server::Controller::Project'}); + $router->connect('/project/:id/:n/:count' => {controller => 'App::Yath::Server::Controller::Project'}); + + $router->connect('/recent/:project/:user/:count' => {controller => 'App::Yath::Server::Controller::Recent'}); + $router->connect('/recent/:project/:user' => {controller => 'App::Yath::Server::Controller::Recent'}); + + $router->connect('/query/:name' => {controller => 'App::Yath::Server::Controller::Query'}); + $router->connect('/query/:name/:arg' => {controller => 'App::Yath::Server::Controller::Query'}); + + $router->connect('/run/:id' => {controller => 'App::Yath::Server::Controller::Run'}); + $router->connect('/run/:id/pin' => {controller => 'App::Yath::Server::Controller::Run', action => 'pin_toggle'}); + $router->connect('/run/:id/delete' => {controller => 'App::Yath::Server::Controller::Run', action => 'delete'}); + $router->connect('/run/:id/cancel' => {controller => 'App::Yath::Server::Controller::Run', action => 'cancel'}); + $router->connect('/run/:id/parameters' => {controller => 'App::Yath::Server::Controller::Run', action => 'parameters'}); + + $router->connect('/run/field/:id' => {controller => 'App::Yath::Server::Controller::RunField'}); + $router->connect('/run/field/:id/delete' => {controller => 'App::Yath::Server::Controller::RunField', action => 'delete'}); + + $router->connect('/job/field/:id' => {controller => 'App::Yath::Server::Controller::JobTryField'}); + $router->connect('/job/field/:id/delete' => {controller => 'App::Yath::Server::Controller::JobTryField', action => 'delete'}); + + $router->connect('/job/:job' => {controller => 'App::Yath::Server::Controller::Job'}); + $router->connect('/job/:job/:try' => {controller => 'App::Yath::Server::Controller::Job'}); + $router->connect('/event/:id' => {controller => 'App::Yath::Server::Controller::Events', from => 'single_event'}); + $router->connect('/event/:id/events' => {controller => 'App::Yath::Server::Controller::Events', from => 'event'}); + + $router->connect('/durations/:project' => {controller => 'App::Yath::Server::Controller::Durations'}); + $router->connect('/durations/:project/median' => {controller => 'App::Yath::Server::Controller::Durations', median => 1}); + $router->connect('/durations/:project/median/:user' => {controller => 'App::Yath::Server::Controller::Durations', median => 1}); + $router->connect('/durations/:project/:short/:medium' => {controller => 'App::Yath::Server::Controller::Durations'}); + + $router->connect('/coverage/:source' => {controller => 'App::Yath::Server::Controller::Coverage'}); + $router->connect('/coverage/:source/:user' => {controller => 'App::Yath::Server::Controller::Coverage'}); + $router->connect('/coverage/:source/delete' => {controller => 'App::Yath::Server::Controller::Coverage', delete => 1}); + + $router->connect('/failed/:source' => {controller => 'App::Yath::Server::Controller::Files', failed => 1}); + $router->connect('/failed/:source/json' => {controller => 'App::Yath::Server::Controller::Files', failed => 1, json => 1}); + $router->connect('/failed/:project/:id' => {controller => 'App::Yath::Server::Controller::Files', failed => 1, json => 1}); + $router->connect('/failed/:project/:username/:id' => {controller => 'App::Yath::Server::Controller::Files', failed => 1, json => 1}); + + $router->connect('/files/:source' => {controller => 'App::Yath::Server::Controller::Files', failed => 0}); + $router->connect('/files/:source/json' => {controller => 'App::Yath::Server::Controller::Files', failed => 0, json => 1}); + $router->connect('/files/:project/:id' => {controller => 'App::Yath::Server::Controller::Files', failed => 0, json => 1}); + $router->connect('/files/:project/:username/:id' => {controller => 'App::Yath::Server::Controller::Files', failed => 0, json => 1}); + + $router->connect('/rerun/:run_id' => {controller => 'App::Yath::Server::Controller::ReRun'}); + $router->connect('/rerun/:project/:username' => {controller => 'App::Yath::Server::Controller::ReRun'}); + + $router->connect('/binary/:binary_id' => {controller => 'App::Yath::Server::Controller::Binary'}); + + $router->connect('/download/:id' => {controller => 'App::Yath::Server::Controller::Download'}); + + $router->connect('/lookup' => {controller => 'App::Yath::Server::Controller::Lookup'}); + $router->connect('/lookup/:lookup' => {controller => 'App::Yath::Server::Controller::Lookup'}); + $router->connect('/lookup/data/:lookup' => {controller => 'App::Yath::Server::Controller::Lookup', data => 1}); + + $router->connect('/view' => {controller => 'App::Yath::Server::Controller::View'}); + $router->connect('/view/project/:project_id' => {controller => 'App::Yath::Server::Controller::View'}); + $router->connect('/view/user/:user_id' => {controller => 'App::Yath::Server::Controller::View'}); + $router->connect('/view/:run_id' => {controller => 'App::Yath::Server::Controller::View'}); + $router->connect('/view/:run_id/:job' => {controller => 'App::Yath::Server::Controller::View'}); + $router->connect('/view/:run_id/:job/:try' => {controller => 'App::Yath::Server::Controller::View'}); + + $router->connect('/stream' => {controller => 'App::Yath::Server::Controller::Stream'}); + $router->connect('/stream/run/:run_id' => {controller => 'App::Yath::Server::Controller::Stream', run_only => 1}); + $router->connect('/stream/user/:user_id' => {controller => 'App::Yath::Server::Controller::Stream'}); + $router->connect('/stream/project/:project_id' => {controller => 'App::Yath::Server::Controller::Stream'}); + $router->connect('/stream/:run_id' => {controller => 'App::Yath::Server::Controller::Stream'}); + $router->connect('/stream/:run_id/:job' => {controller => 'App::Yath::Server::Controller::Stream'}); + $router->connect('/stream/:run_id/:job/:try' => {controller => 'App::Yath::Server::Controller::Stream'}); + + $router->connect('/sweeper/:count/days' => {controller => 'App::Yath::Server::Controller::Sweeper', units => 'day'}); + $router->connect('/sweeper/:count/hours' => {controller => 'App::Yath::Server::Controller::Sweeper', units => 'hour'}); + $router->connect('/sweeper/:count/minutes' => {controller => 'App::Yath::Server::Controller::Sweeper', units => 'minute'}); + $router->connect('/sweeper/:count/seconds' => {controller => 'App::Yath::Server::Controller::Sweeper', units => 'second'}); + + return $self->{+ROUTER} = $router; +} + +sub to_app { + my $self = shift; + + return $self->{+APP} //= builder { + mount '/js' => Plack::App::Directory->new({root => share_dir('js')})->to_app; + mount '/css' => Plack::App::Directory->new({root => share_dir('css')})->to_app; + mount '/img' => Plack::App::Directory->new({root => share_dir('img')})->to_app; + mount '/favicon.ico' => Plack::App::File->new({file => share_dir('img') . '/favicon.ico'})->to_app; + mount '/' => sub { $self->handle_request(@_) }; + }; +} + +sub handle_request { + my $self = shift; + my ($env) = @_; + + my $schema = $self->schema; + my $router = $self->router; + my $route = $router->match($env) || {}; + my $controller_class = $route->{controller} or return error(404); + + my $req = App::Yath::Server::Request->new(env => $env, schema => $schema); + + my ($controller, $res, $session, $session_host, $user); + my $ok = eval { + $session = $req->session(); + $session_host = $req->session_host(); + + if ($self->{+SINGLE_USER}) { + $user = $self->schema->resultset('User')->find({username => 'root'}); + } + elsif ($session_host) { + $user = $session_host->user if $session_host->user_id; + } + + $req->set_user($user) if $user; + + $controller = $controller_class->new( + request => $req, + route => $route, + schema => $self->schema, + schema_config => $self->schema_config, + session => $session, + session_host => $session_host, + single_run => $self->single_run, + single_user => $self->single_user, + user => $user, + ); + + $res = $controller->auth_check() // $controller->handle($route); + + 1; + }; + my $err = $@ || 'Internal Error'; + + unless ($ok && $res) { + if (blessed($err) && $err->isa('App::Yath::Server::Response')) { + $res = $err; + } + else { + warn $err; + my $msg = ($ENV{T2_HARNESS_SERVER_DEV} || '') eq 'dev' ? "$err\n" : undef; + $res = error(500 => $msg); + } + } + + my $ct = $route->{json} ? 'application/json' : blessed($res) ? $res->content_type() : 'text/html'; + $ct ||= 'text/html'; + $ct = lc($ct); + $res->content_type($ct) if blessed($res); + + if (my $stream = $res->stream) { + return $stream; + } + + if ($ct eq 'text/html') { + my $dt = DateTime->now(time_zone => 'local'); + + my $tx = Text::Xslate->new(path => [share_dir('templates')]); + my $wrapped = $tx->render( + 'main.tx', + { + single_user => $self->single_user // 0, + single_run => $self->single_run // 0, + no_upload => $schema->config('no_upload') // 0, + show_user => $self->single_user ? 0 : 1, + + user => $req->user || undef, + errors => $res->errors || [], + messages => $res->messages || [], + add_css => $res->css || [], + add_js => $res->js || [], + title => $res->title || ($controller ? $controller->title : 'Yath-Server'), + + time_zone => $dt->strftime("%Z"), + + base_uri => $req->base->as_string || '', + content => mark_raw($res->raw_body) || '', + } + ); + + $res->body($wrapped); + } + elsif($ct eq 'application/json') { + if (my $data = $res->raw_body) { + $res->body(ref($data) ? encode_json($data) : $data); + } + elsif (my $errors = $res->errors) { + $res->body(encode_json({errors => $errors})); + } + } + + $res->cookies->{uuid} = {value => $session->session_uuid, httponly => 1, expires => '+1M'} + if $session; + + return $res->finalize; +} + + +__END__ + +=pod + +=encoding UTF-8 + +=head1 NAME + +App::Yath::Server::Plack - Plack app module for Yath Server. + +=head1 DESCRIPTION + + +=head1 SYNOPSIS + + +=head1 SOURCE + +The source code repository for Test2-Harness-UI can be found at +F. + +=head1 MAINTAINERS + +=over 4 + +=item Chad Granum Eexodist@cpan.orgE + +=back + +=head1 AUTHORS + +=over 4 + +=item Chad Granum Eexodist@cpan.orgE + +=back + +=head1 COPYRIGHT + +Copyright Chad Granum Eexodist7@gmail.comE. + +This program is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. + +See F + +=cut diff --git a/lib/App/Yath/Server/Request.pm b/lib/App/Yath/Server/Request.pm index 142fd1080..498b79fd8 100644 --- a/lib/App/Yath/Server/Request.pm +++ b/lib/App/Yath/Server/Request.pm @@ -4,57 +4,56 @@ use warnings; our $VERSION = '2.000000'; -use App::Yath::Schema::UUID qw/gen_uuid uuid_inflate/; -use Data::GUID; use Carp qw/croak/; +use Test2::Util::UUID qw/gen_uuid/; +use App::Yath::Schema::Util qw/format_uuid_for_db/; + use parent 'Plack::Request'; +use Test2::Harness::Util::HashBase qw{ + +session + +session_host + SUPER::new($env); - $self->{'config'} = delete $params{config} or croak "'config' is a required attribute"; + croak "'env' is a required attribute" unless $params{env}; + croak "'schema' is a required attribute" unless $params{+SCHEMA}; - return $self; + return bless(\%params, $class); } -sub schema { $_[0]->{config}->schema } - sub session { my $self = shift; - return $self->{session} if $self->{session}; + return $self->{+SESSION} if $self->{+SESSION}; my $schema = $self->schema; my $session; my $cookies = $self->cookies; - if (my $id = uuid_inflate($cookies->{id})) { - $session = $schema->resultset('Session')->find({session_id => $id}); + if (my $uuid = $cookies->{uuid}) { + $session = $schema->resultset('Session')->find({session_uuid => $uuid}); $session = undef unless $session && $session->active; } - $session ||= $self->schema->resultset('Session')->create( - {session_id => gen_uuid}, + my $uuid = gen_uuid(); + $session ||= $schema->resultset('Session')->create( + {session_uuid => format_uuid_for_db($uuid)}, ); - $self->{session} = $session; - - # Vivify this - $self->session_host; - - return $session; + return $self->{+SESSION} = $session; } sub session_host { my $self = shift; - return $self->{session_host} if $self->{session_host}; + return $self->{+SESSION_HOST} if $self->{+SESSION_HOST}; my $session = $self->session or return undef; @@ -71,7 +70,6 @@ sub session_host { ); $host //= $schema->resultset('SessionHost')->create({ - session_host_id => gen_uuid, session_id => $session->session_id, address => $self->address // 'SOCKET', agent => $self->user_agent, @@ -79,20 +77,9 @@ sub session_host { $schema->txn_commit; - return $self->{session_host} = $host; + return $self->{+SESSION_HOST} = $host; } -sub user { - my $self = shift; - - return $self->schema->resultset('User')->find({username => 'root'}) - if $self->{config}->single_user; - - my $host = $self->session_host or return undef; - - return undef unless $host->user_id; - return $host->user; -} 1; diff --git a/lib/App/Yath/Server/Response.pm b/lib/App/Yath/Server/Response.pm index 2ce612113..838f6b619 100644 --- a/lib/App/Yath/Server/Response.pm +++ b/lib/App/Yath/Server/Response.pm @@ -6,7 +6,7 @@ our $VERSION = '2.000000'; use Carp qw/croak/; use Time::HiRes qw/sleep time/; -use Test2::Harness::Util::JSON qw/encode_json/; +use Test2::Harness::Util::JSON qw/encode_json encode_ascii_json/; use parent 'Plack::Response'; @@ -76,7 +76,6 @@ sub stream { my ($done, $fetch); if(my $rs = $params{resultset}) { - my $json = Test2::Harness::Util::JSON::JSON()->new->utf8(0)->convert_blessed(1)->allow_nonref(1); my $go = 1; $done = sub { !$go }; $fetch = sub { @@ -85,7 +84,7 @@ sub stream { if(my $meth = $params{data_method}) { $data = $go->$meth(); } - my $out = $json->encode($data) . "\n"; + my $out = encode_ascii_json($data) . "\n"; return $out; }; } diff --git a/lib/App/Yath/Server/Tester.pm b/lib/App/Yath/Server/Tester.pm index 6211d9921..c6b2efe64 100644 --- a/lib/App/Yath/Server/Tester.pm +++ b/lib/App/Yath/Server/Tester.pm @@ -18,7 +18,7 @@ use Carp qw/croak/; use Time::HiRes qw/sleep/; use Test2::Util qw/pkg_to_file/; use App::Yath::Server::Util qw/dbd_driver qdb_driver share_dir share_file/; -use App::Yath::Schema::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use Scope::Guard qw/guard/; use File::Temp qw/tempfile/; @@ -87,8 +87,8 @@ sub init { local $ENV{YATH_UI_SCHEMA} = $schema; require(pkg_to_file("App::Yath::Server::Schema::$schema")); - my $user = $config->schema->resultset('User')->create({username => 'root', password => 'root', realname => 'root', user_id => gen_uuid()}); - my $project = $config->schema->resultset('Project')->create({name => 'test', project_id => gen_uuid()}); + my $user = $config->schema->resultset('User')->create({username => 'root', password => 'root', realname => 'root'}); + my $project = $config->schema->resultset('Project')->create({name => 'test'}); exec('starman', '-Ilib', '--listen' => ($self->{+PORT} ? ":$self->{+PORT}" : $self->{+SOCKET}), '--workers', 5, share_file('psgi/test.psgi')), } diff --git a/lib/App/Yath/Server/Util.pm b/lib/App/Yath/Server/Util.pm deleted file mode 100644 index 1d66bc792..000000000 --- a/lib/App/Yath/Server/Util.pm +++ /dev/null @@ -1,267 +0,0 @@ -package App::Yath::Server::Util; -use strict; -use warnings; - -our $VERSION = '2.000000'; - -use Carp qw/croak/; - -use File::ShareDir(); - -use Test2::Harness::Util qw/mod2file/; -use App::Yath::Schema::UUID qw/uuid_inflate/; - -use Importer Importer => 'import'; - -our @EXPORT = qw/share_dir share_file qdb_driver dbd_driver config_from_settings find_job format_duration parse_duration is_invalid_subtest_name/; - -my %SCHEMA_TO_QDB_DRIVER = ( - mariadb => 'MySQL', - mysql => 'MySQL', - postgresql => 'PostgreSQL', -); - -my %SCHEMA_TO_DBD_DRIVER = ( - mariadb => 'DBD::MariaDB', - mysql => 'DBD::mysql', - postgresql => 'DBD::Pg', -); - -my %BAD_ST_NAME = ( - '__ANON__' => 1, - 'unnamed' => 1, - 'unnamed subtest' => 1, - 'unnamed summary' => 1, - '' => 1, -); - -sub is_invalid_subtest_name { - my ($name) = @_; - return $BAD_ST_NAME{$name} // 0; -} - -sub find_job { - my ($schema, $uuid, $try) = @_; - - $uuid = uuid_inflate($uuid) or croak "Invalid job identifier"; - - my $jobs = $schema->resultset('Job'); - - if (length $try) { - return $jobs->search({job_id => $uuid}, {order_by => {'-desc' => 'job_try'}, limit => 1})->first - if $try == -1; - - return $jobs->find({job_id => $uuid, job_try => $try}); - } - - return $jobs->find({job_key => $uuid}) - || $jobs->search({job_id => $uuid}, {order_by => {'-desc' => 'job_try'}, limit => 1})->first; -} - -sub base_name { - my ($in) = @_; - - my $out = lc($in); - $out =~ s/\.sql$//; - $out =~ s/\d+$//g; - - return $out; -} - -sub qdb_driver { - my $base = base_name(@_); - return $SCHEMA_TO_QDB_DRIVER{$base}; -} - -sub dbd_driver { - my $base = base_name(@_); - return $SCHEMA_TO_DBD_DRIVER{$base}; -} - -sub share_file { - my ($file) = @_; - - return File::ShareDir::dist_file('Test2-Harness-UI' => $file) - unless 'dev' eq ($ENV{T2_HARNESS_UI_ENV} || ''); - - my $path = "share/$file"; - croak "Could not find '$file'" unless -e $path; - - return $path; -} - -sub share_dir { - my ($dir) = @_; - - my $path; - - if ('dev' eq ($ENV{T2_HARNESS_UI_ENV} || '')) { - $path = "share/$dir"; - } - else { - my $root = File::ShareDir::dist_dir('Test2-Harness-UI'); - $path = "$root/$dir"; - } - - croak "Could not find '$dir'" unless -d $path; - - return $path; -} - -sub config_from_settings { - my ($settings) = @_; - - my $db = $settings->prefix('yathui-db') or die "No DB settings"; - - if (my $cmod = $db->config) { - my $file = mod2file($cmod); - require $file; - - return $cmod->yath_ui_config(%$$db); - } - - my $dsn = $db->dsn; - - unless ($dsn) { - $dsn = ""; - - my $driver = $db->driver; - my $name = $db->name; - - $dsn .= "dbi:$driver" if $driver; - $dsn .= ":dbname=$name" if $name; - - if (my $socket = $db->socket) { - my $ld = lc($driver); - if ($ld eq 'pg') { - $dsn .= ";host=$socket"; - } - else { - $dsn .= ";${ld}_socket=$socket"; - } - } - else { - my $host = $db->host; - my $port = $db->port; - - $dsn .= ";host=$host" if $host; - $dsn .= ";port=$port" if $port; - } - } - - require App::Yath::Server::Config; - return App::Yath::Server::Config->new( - dbi_dsn => $dsn, - dbi_user => $db->user // '', - dbi_pass => $db->pass // '', - ); -} - -sub format_duration { - my $seconds = shift; - - my $minutes = int($seconds / 60); - my $hours = int($minutes / 60); - my $days = int($hours / 24); - - $minutes %= 60; - $hours %= 24; - - $seconds -= $minutes * 60; - $seconds -= $hours * 60 * 60; - $seconds -= $days * 60 * 60 * 24; - - my @dur; - push @dur => sprintf("%02dd", $days) if $days; - push @dur => sprintf("%02dh", $hours) if @dur || $hours; - push @dur => sprintf("%02dm", $minutes) if @dur || $minutes; - push @dur => sprintf("%07.4fs", $seconds); - - return join ':' => @dur; -} - -sub parse_duration { - my $duration = shift; - - return 0 unless $duration; - - return $duration unless $duration =~ m/:?.*[dhms]$/i; - - my $out = 0; - - my (@parts) = split ':' => $duration; - for my $part (@parts) { - my ($num, $type) = ($part =~ m/^([0-9\.]+)([dhms])$/); - - unless ($num && $type) { - warn "invalid duration section '$part'"; - next; - } - - if ($type eq 'd') { - $out += ($num * 60 * 60 * 24); - } - elsif ($type eq 'h') { - $out += ($num * 60 * 60); - } - elsif ($type eq 'm') { - $out += ($num * 60); - } - else { - $out += $num; - } - } - - return $out; -} - - -1; - -__END__ - -=pod - -=encoding UTF-8 - -=head1 NAME - -App::Yath::Server::Util - General Utilities - -=head1 DESCRIPTION - -=head1 SYNOPSIS - -TODO - -=head1 SOURCE - -The source code repository for Test2-Harness-UI can be found at -F. - -=head1 MAINTAINERS - -=over 4 - -=item Chad Granum Eexodist@cpan.orgE - -=back - -=head1 AUTHORS - -=over 4 - -=item Chad Granum Eexodist@cpan.orgE - -=back - -=head1 COPYRIGHT - -Copyright Chad Granum Eexodist7@gmail.comE. - -This program is free software; you can redistribute it and/or -modify it under the same terms as Perl itself. - -See F - -=cut diff --git a/lib/App/Yath/Util.pm b/lib/App/Yath/Util.pm index af574bd0e..e93d4a877 100644 --- a/lib/App/Yath/Util.pm +++ b/lib/App/Yath/Util.pm @@ -4,18 +4,47 @@ use warnings; our $VERSION = '2.000000'; -use File::Spec; +use File::Spec(); +use File::ShareDir(); use Test2::Harness::Util qw/clean_path/; use Importer Importer => 'import'; use Config qw/%Config/; +use Carp qw/croak/; our @EXPORT_OK = qw{ is_generated_test_pl find_yath + share_dir share_file }; +sub share_file { + my ($file) = @_; + + my $path = "share/$file"; + return $path if -f $path; + + return File::ShareDir::dist_file('Test2-Harness' => $file); + + croak "Could not find '$file'"; +} + +sub share_dir { + my ($dir) = @_; + + my $path = "share/$dir"; + return $path if -d $path; + + my $root = File::ShareDir::dist_dir('Test2-Harness'); + + $path .= "/$dir"; + + croak "Could not find '$dir'" unless -d $path; + + return $path; +} + sub find_yath { return $App::Yath::Script::SCRIPT if defined $App::Yath::Script::SCRIPT; diff --git a/lib/Test2/Formatter/Stream.pm b/lib/Test2/Formatter/Stream.pm index 4c4ae8a87..b4842a4fe 100644 --- a/lib/Test2/Formatter/Stream.pm +++ b/lib/Test2/Formatter/Stream.pm @@ -11,7 +11,7 @@ use Carp qw/croak confess/; use Time::HiRes qw/time/; use Test2::Util qw/get_tid/; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use Test2::Harness::Util qw/hub_truth apply_encoding/; use Test2::Harness::Collector::Child qw/send_event/; diff --git a/lib/Test2/Harness/Collector/Auditor/Job.pm b/lib/Test2/Harness/Collector/Auditor/Job.pm index ca817a568..fc409d38c 100644 --- a/lib/Test2/Harness/Collector/Auditor/Job.pm +++ b/lib/Test2/Harness/Collector/Auditor/Job.pm @@ -9,7 +9,7 @@ use Scalar::Util qw/blessed/; use List::Util qw/first max/; use Time::HiRes qw/time/; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use Test2::Harness::Util qw/hub_truth parse_exit/; diff --git a/lib/Test2/Harness/Collector/Child.pm b/lib/Test2/Harness/Collector/Child.pm index f9a53b61b..38691c52b 100644 --- a/lib/Test2/Harness/Collector/Child.pm +++ b/lib/Test2/Harness/Collector/Child.pm @@ -13,7 +13,7 @@ use Time::HiRes qw/time/; use Test2::Util qw/get_tid/; use Carp qw/confess croak/; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use Test2::Harness::Util::JSON qw/encode_ascii_json/; use vars qw/$STDERR_APIPE $STDOUT_APIPE/; diff --git a/lib/Test2/Harness/Collector/IOParser.pm b/lib/Test2/Harness/Collector/IOParser.pm index 9fca027f9..e2cd6ed50 100644 --- a/lib/Test2/Harness/Collector/IOParser.pm +++ b/lib/Test2/Harness/Collector/IOParser.pm @@ -4,7 +4,7 @@ use warnings; use Carp qw/confess/; use Time::HiRes qw/time/; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; our $VERSION = '2.000000'; diff --git a/lib/Test2/Harness/IPC/Protocol/AtomicPipe/Connection.pm b/lib/Test2/Harness/IPC/Protocol/AtomicPipe/Connection.pm index 6030d52cf..7829620cc 100644 --- a/lib/Test2/Harness/IPC/Protocol/AtomicPipe/Connection.pm +++ b/lib/Test2/Harness/IPC/Protocol/AtomicPipe/Connection.pm @@ -13,7 +13,7 @@ use Time::HiRes qw/sleep/; use Scalar::Util qw/weaken blessed/; use Test2::Harness::IPC::Util qw/check_pipe ipc_warn pid_is_running/; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use Test2::Harness::Util::JSON qw/encode_json/; use Test2::Harness::Instance::Message; diff --git a/lib/Test2/Harness/Resource.pm b/lib/Test2/Harness/Resource.pm index 8e316394b..ceadc848a 100644 --- a/lib/Test2/Harness/Resource.pm +++ b/lib/Test2/Harness/Resource.pm @@ -7,27 +7,24 @@ our $VERSION = '2.000000'; use Carp qw/croak/; use Term::Table; +use Time::HiRes qw/time/; +use Sys::Hostname qw/hostname/; use Test2::Harness::Util qw/parse_exit/; use Test2::Harness::IPC::Util qw/start_collected_process ipc_connect set_procname/; use Test2::Harness::Util::JSON qw/decode_json encode_json/; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use Test2::Harness::Util::HashBase qw{ {+RESOURCE_ID} //= gen_uuid(); -} - sub spawns_process { 0 } sub is_job_limiter { 0 } -sub setup { } sub teardown { } sub tick { } sub cleanup { } @@ -43,11 +40,19 @@ sub assign { croak "'$_[0]' does not implement 'assign'" } sub release { croak "'$_[0]' does not implement 'release'" } sub subprocess_run { croak "'$_[0]' does not implement 'subprocess_run'" } +sub init { $_[0]->host } +sub host { $_[0]->{+HOST} //= hostname() } + sub DESTROY { my $self = shift; $self->cleanup(); } +sub setup { + my $self = shift; + $self->send_data_event; +} + sub sort_weight { my $class = shift; return 100 if $class->is_job_limiter; @@ -131,6 +136,33 @@ sub _subprocess_run { exit 0; } +sub send_data_event { + my $self = shift; + + my ($data) = $self->status_data(); + + return unless $data; + + $self->send_event({ + facet_data => { + resource_state => { + module => ref($self) || $self, + data => $data, + host => $self->{+HOST}, + }, + }, + }); +} + +sub send_event { + my $self = shift; + my ($e) = @_; + + my $send = $self->{+_SEND_EVENT} //= Test2::Harness::Collector::Child->send_event; + + $send->($e); +} + sub status_data { () } 1; diff --git a/lib/Test2/Harness/Resource/JobCount.pm b/lib/Test2/Harness/Resource/JobCount.pm index 6e970fd81..3c424c62a 100644 --- a/lib/Test2/Harness/Resource/JobCount.pm +++ b/lib/Test2/Harness/Resource/JobCount.pm @@ -73,6 +73,8 @@ sub assign { $env->{T2_HARNESS_MY_JOB_CONCURRENCY} = $count; + $self->send_data_event; + return $env; } @@ -85,6 +87,8 @@ sub release { $self->{+USED} -= $count; + $self->send_data_event; + return $id; } diff --git a/lib/Test2/Harness/Run.pm b/lib/Test2/Harness/Run.pm index 108e4e2f3..d63847a78 100644 --- a/lib/Test2/Harness/Run.pm +++ b/lib/Test2/Harness/Run.pm @@ -10,7 +10,7 @@ use Test2::Harness::TestSettings; use Test2::Harness::IPC::Protocol; use Test2::Harness::Util qw/mod2file/; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; our $VERSION = '2.000000'; @@ -54,6 +54,7 @@ use Test2::Harness::Util::HashBase( { harness_job_queued => { - file => $job->test_file->file, - job_id => $job->job_id, - stamp => $stamp, + file => $job->test_file->file, + rel_file => $job->test_file->relative, + job_id => $job->job_id, + stamp => $stamp, } }, ); diff --git a/lib/Test2/Harness/Run/Job.pm b/lib/Test2/Harness/Run/Job.pm index 5ffae6cae..9f36e3bd5 100644 --- a/lib/Test2/Harness/Run/Job.pm +++ b/lib/Test2/Harness/Run/Job.pm @@ -11,7 +11,7 @@ use Scalar::Util qw/blessed/; use Test2::Harness::TestFile; use Test2::Harness::Util qw/clean_path/; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use Test2::Harness::Util::HashBase qw{ 'import'; - -our @EXPORT = qw/gen_uuid/; -our @EXPORT_OK = qw/UG gen_uuid/; - -my ($UG, $UG_PID); -sub UG { - return $UG if $UG && $UG_PID && $UG_PID == $$; - - $UG_PID = $$; - return $UG = Data::UUID->new; -} - -sub gen_uuid { UG()->create_str() } - -1; - -__END__ - -=pod - -=encoding UTF-8 - -=head1 NAME - -Test2::Harness::Util::UUID - Utils for generating UUIDs. - -=head1 DESCRIPTION - -This module provides a consistent UUID source for all of Test2::Harness. - -=head1 SYNOPSIS - - use Test2::Harness::Util::UUID qw/gen_uuid/; - - my $uuid = gen_uuid; - -=head1 EXPORTS - -=over 4 - -=item $uuid = gen_uuid() - -Generate a UUID. - -=back - -=head1 SOURCE - -The source code repository for Test2-Harness can be found at -L. - -=head1 MAINTAINERS - -=over 4 - -=item Chad Granum Eexodist@cpan.orgE - -=back - -=head1 AUTHORS - -=over 4 - -=item Chad Granum Eexodist@cpan.orgE - -=back - -=head1 COPYRIGHT - -Copyright Chad Granum Eexodist7@gmail.comE. - -This program is free software; you can redistribute it and/or -modify it under the same terms as Perl itself. - -See L - -=cut diff --git a/lib/Test2/Tools/HarnessTester.pm b/lib/Test2/Tools/HarnessTester.pm index 60d9b1f46..9914119cc 100644 --- a/lib/Test2/Tools/HarnessTester.pm +++ b/lib/Test2/Tools/HarnessTester.pm @@ -4,7 +4,7 @@ use warnings; our $VERSION = '2.000000'; -use Test2::Harness::Util::UUID qw/gen_uuid/; +use Test2::Util::UUID qw/gen_uuid/; use App::Yath::Tester qw/make_example_dir/; diff --git a/share/css/job.css b/share/css/job.css index d4a6783a1..ad880f4ec 100644 --- a/share/css/job.css +++ b/share/css/job.css @@ -37,6 +37,10 @@ div.live { scroll-snap-align: end; } +tr.stuck_orphan { + background: #FFAAAA !important; +} + tr.temp_orphan { background: #C0E0FF; } diff --git a/share/css/resources.css b/share/css/resources.css index a1a841e48..33ee560f7 100644 --- a/share/css/resources.css +++ b/share/css/resources.css @@ -90,7 +90,7 @@ div.resource table th { font-weight: bold; } -div#put_stamps_here { +div#resource_wrapper.rendered { position: fixed; bottom: 30px; overflow: hidden; @@ -98,7 +98,7 @@ div#put_stamps_here { margin-left: -8px; } -div.stamp_selector { +div.range_selector { background: white; border: 1px solid #000000; overflow: hidden; @@ -111,7 +111,7 @@ div.stamp_selector { margin-right: 10px; } -div.stamp_selector h1 { +div.range_selector h1 { margin: -1px; margin-bottom: 8px; padding-left: 6px; @@ -124,7 +124,7 @@ div.stamp_selector h1 { overflow: hidden; } -div.stamp_selector div.stamp_selector_inner { +div.range_selector div.range_selector_inner { overflow: none; padding: 4px; margin: 4px; @@ -132,7 +132,7 @@ div.stamp_selector div.stamp_selector_inner { display: block; } -div.stamp_selector input[type=button] { +div.range_selector input[type=button] { float: right; width: auto; margin-right: 8px; @@ -144,14 +144,14 @@ div.stamp_selector input[type=button] { border-radius: 8px; } -div.stamp_selector input { +div.range_selector input { overflow: hidden; padding: 0px; margin: 0; width: 100%; } -div.stamp_selector div.stamp_selector_left_bound { +div.range_selector div.range_selector_left_bound { display: block; float: left; margin-left: 4px; @@ -159,7 +159,7 @@ div.stamp_selector div.stamp_selector_left_bound { margin-bottom: 4px; } -div.stamp_selector div.stamp_selector_right_bound { +div.range_selector div.range_selector_right_bound { display: block; float: right; margin-left: 4px; @@ -167,23 +167,23 @@ div.stamp_selector div.stamp_selector_right_bound { margin-bottom: 4px; } -div.stamp_selector div.stamp_selector_select { +div.range_selector div.range_selector_select { width: fit-content; margin: 0 auto; font-weight: bold; } -div.stamp_selector div.stamp_selector_select a, -div.stamp_selector div.stamp_selector_select a:link, -div.stamp_selector div.stamp_selector_select a:visited, -div.stamp_selector div.stamp_selector_select a:active, -div.stamp_selector div.stamp_selector_select a:hover { +div.range_selector div.range_selector_select a, +div.range_selector div.range_selector_select a:link, +div.range_selector div.range_selector_select a:visited, +div.range_selector div.range_selector_select a:active, +div.range_selector div.range_selector_select a:hover { text-decoration: none; color: black; cursor: pointer; } -div.stamp_selector div.stamp_selector_select a:hover { +div.range_selector div.range_selector_select a:hover { color: blue; } diff --git a/share/css/view.css b/share/css/view.css index 677811700..0d4131e9c 100644 --- a/share/css/view.css +++ b/share/css/view.css @@ -45,6 +45,11 @@ div.live { scroll-snap-align: end; } +tr.stuck_orphan { + background: #FFF0E0 !important; + border: 2px dashed #FF90F0 !important; +} + tr.temp_orphan { background: #C0E0FF; } diff --git a/share/js/eventtable.js b/share/js/eventtable.js index 9fa84996d..debec1453 100644 --- a/share/js/eventtable.js +++ b/share/js/eventtable.js @@ -66,6 +66,7 @@ t2hui.eventtable.build_table = function(run, job, controls) { t2hui.eventtable.expand_lines = function(item) { var out = []; + var tools = true; var count = 0; item.lines.forEach(function(line) { @@ -78,7 +79,7 @@ t2hui.eventtable.expand_lines = function(item) { 'item': item, 'set_ord': count++, 'set_total': item.lines.length, - 'id': item.event_id, + 'id': item.event_uuid, }); tools = false; }); @@ -94,10 +95,10 @@ t2hui.eventtable.message_builder = function(item, dest, data, table) { if (item.item.is_parent == false) { return } - var events_uri = base_uri + 'event/' + item.item.event_id + '/events'; + var events_uri = base_uri + 'event/' + item.item.event_uuid + '/events'; var jumpto = window.location.hash.substr(1); - var highlight = item.item.event_id === jumpto ? true : false; + var highlight = item.item.event_uuid === jumpto ? true : false; var expand = $('
+
'); @@ -126,17 +127,17 @@ t2hui.eventtable.message_builder = function(item, dest, data, table) { if (highlight) { row.addClass('highlight'); - $('[data-parent-id="' + item.item.event_id + '"]').addClass('highlight'); + $('[data-parent-id="' + item.item.event_uuid + '"]').addClass('highlight'); } else { row.removeClass('highlight'); - $('[data-parent-id="' + item.item.event_id + '"]').removeClass('highlight'); + $('[data-parent-id="' + item.item.event_uuid + '"]').removeClass('highlight'); } }); } }, function(e) { - var params = {"data": {"parent-id": item.item.event_id}}; + var params = {"data": {"parent-id": item.item.event_uuid}}; if (highlight) { params.class = "highlight"; } @@ -157,8 +158,11 @@ t2hui.eventtable.message_builder = function(item, dest, data, table) { } t2hui.eventtable.place_row = function(row, item, table, state) { - if (!item.item['loading_subtest']) { - if (item.item.orphan) { + if (item.item.orphan) { + if (item.item['loading_subtest']) { + row.addClass('stuck_orphan'); + } + else { row.addClass('temp_orphan'); if (!state['orphan']) { state['orphan'] = row; @@ -178,7 +182,7 @@ t2hui.eventtable.place_row = function(row, item, table, state) { var pid = item.item['parent_id']; if (!state[pid]) { - var got = table.table.find('tr[data-event-id="' + item.item.parent_id + '"]'); + var got = $('tr[data-event-id="' + item.item.parent_uuid + '"]'); state[pid] = got.last(); } @@ -196,7 +200,7 @@ t2hui.eventtable.message_inner_builder = function(item, dest, data) { return; } - if (typeof(item.extra) === "string") { + if (typeof(item.extra) === "number" && item.tag === "IMAGE") { var out = $('
'); var link = $('' + item.message + ''); out.append(link); @@ -249,14 +253,19 @@ t2hui.eventtable.tool_builder = function(item, tools, data) { } if (item.item.facets) { - var efacet = $('
'); + var img = "/img/data.png"; + if (item.item.orphan) { + img = "/img/orphan.png"; + } + + var efacet = $('
'); tools.append(efacet); efacet.click(function() { $('#modal_body').empty(); $('#modal_body').text("loading..."); $('#free_modal').slideDown(); - var uri = base_uri + 'event/' + item.item.event_id; + var uri = base_uri + 'event/' + item.item.event_uuid; $.ajax(uri, { 'data': { 'content-type': 'application/json' }, @@ -268,27 +277,6 @@ t2hui.eventtable.tool_builder = function(item, tools, data) { }); }); } - - if (item.item.orphan) { - var eorphan = $('
'); - tools.append(eorphan); - eorphan.click(function() { - $('#modal_body').empty(); - $('#modal_body').text("loading..."); - $('#free_modal').slideDown(); - - var uri = base_uri + 'event/' + item.item.event_id; - - $.ajax(uri, { - 'data': { 'content-type': 'application/json' }, - 'success': function(event) { - $('#modal_body').empty(); - var formatter = new JSONFormatter(event.orphan, 2); - $('#modal_body').html(formatter.render()); - }, - }); - }); - } } t2hui.eventtable.clean_tag = function(tag) { @@ -304,7 +292,7 @@ t2hui.eventtable.modify_row = function(row, item, table, controls) { row.addClass('facet_' + item.facet); row.addClass('tag_' + ctag); - row.attr('data-event-id', item.item.event_id); + row.attr('data-event-id', item.item.event_uuid); if (!controls.filters.seen[tag]) { controls.filters.state[tag] = !controls.filters.hide[tag]; diff --git a/share/js/fieldtable.js b/share/js/fieldtable.js index b94480281..7adb7b7d3 100644 --- a/share/js/fieldtable.js +++ b/share/js/fieldtable.js @@ -221,7 +221,7 @@ function FieldTable(spec) { toolrow.prepend(td); } - if (field.data) { + if (field.has_data) { var viewer = $('
'); var td = $(''); td.append(viewer); @@ -319,6 +319,9 @@ function FieldTable(spec) { var col = $(''); col.append(inner); inner.click(function() { col.trigger('click'); }); + if (data.tooltip) { + inner.attr('title', data.tooltip); + } return col; } diff --git a/share/js/interactions.js b/share/js/interactions.js index 43317df4e..bcdf5ae52 100644 --- a/share/js/interactions.js +++ b/share/js/interactions.js @@ -4,7 +4,7 @@ function build_interactions(item, state) { if (item.type === 'run') { var run_table = t2hui.runtable.build_table(); content.append(run_table.render()); - run_table.render_item(item.data, item.data.run_id); + run_table.render_item(item.data, item.data.run_uuid); return; } @@ -24,7 +24,7 @@ function build_interactions(item, state) { content.empty(); state = {}; - var uri = base_uri + 'interactions/data/' + event_id + '/' + val; + var uri = base_uri + 'interactions/data/' + event_uuid + '/' + val; t2hui.fetch(uri, {}, function(item) { build_interactions(item, state) }); return true; }); @@ -36,16 +36,16 @@ function build_interactions(item, state) { else if (item.type === 'job') { state.event_table = null; - content.append('
'); + content.append('
'); var job_table = t2hui.jobtable.build_table(null); if (state.list) { - state.list.append('
  • ' + item.data.file + '
    • '); + state.list.append('
  • ' + item.data.file + '
    • '); } content.append(job_table.render()); - job_table.render_item(item.data, item.data.job_key); + job_table.render_item(item.data, item.data.job_uuid); return; } @@ -61,7 +61,7 @@ function build_interactions(item, state) { state.event_table = event_table; } - state.event_table.render_item(item.data, item.data.event_id); + state.event_table.render_item(item.data, item.data.event_uuid); } } diff --git a/share/js/jobtable.js b/share/js/jobtable.js index cb699a9c6..0e18d654e 100644 --- a/share/js/jobtable.js +++ b/share/js/jobtable.js @@ -4,10 +4,10 @@ t2hui.jobtable.build_table = function() { var columns = [ { 'name': 'tools', 'label': 'tools', 'class': 'tools', 'builder': t2hui.jobtable.tool_builder }, - { 'name': 'try', 'label': 'T', 'class': 'count', 'builder': t2hui.jobtable.build_try }, + { 'name': 'try', 'label': 'T', 'tooltip': '[T]ry', 'class': 'count', 'builder': t2hui.jobtable.build_try }, - { 'name': 'pass_count', 'label': 'P', 'class': 'count', 'builder': t2hui.jobtable.build_pass }, - { 'name': 'fail_count', 'label': 'F', 'class': 'count', 'builder': t2hui.jobtable.build_fail }, + { 'name': 'pass_count', 'tooltip': '[P]ass', 'label': 'P', 'class': 'count', 'builder': t2hui.jobtable.build_pass }, + { 'name': 'fail_count', 'tooltip': '[F]ail', 'label': 'F', 'class': 'count', 'builder': t2hui.jobtable.build_fail }, { 'name': 'exit', 'label': 'exit', 'class': 'exit', 'builder': t2hui.jobtable.build_exit }, @@ -85,25 +85,28 @@ t2hui.jobtable.tool_builder = function(item, tools, data) { $('#modal_body').text("loading..."); $('#free_modal').slideDown(); - var uri = base_uri + 'job/' + item.job_key; + var uri = base_uri + 'job/' + item.job_uuid + '/' + item.job_try_ord; $.ajax(uri, { 'data': { 'content-type': 'application/json' }, 'success': function(job) { - var formatter = new JSONFormatter(job.parameters, 2); + var formatter = new JSONFormatter(job.try.parameters, 2); $('#modal_body').html(formatter.render()); }, }); }); - var link = base_uri + 'view/' + item.run_id + '/' + item.job_key; + var link = base_uri + 'view/' + item.run_uuid + '/' + item.job_uuid + '/' + item.job_try_ord; var go = $(''); tools.append(go); }; t2hui.jobtable.modify_row = function(row, item) { if (item.short_file) { - if (item.retry == true) { + if (item.is_harness_out) { + row.addClass('harness_out'); + } + else if (item.retry == true) { row.addClass('iffy_set'); row.addClass('retry_txt'); } @@ -129,11 +132,11 @@ t2hui.jobtable.modify_row = function(row, item) { }; t2hui.jobtable.field_preprocess = function(field_data) { - field_data.delete = base_uri + 'job/field/' + field_data.job_field_id + '/delete'; + field_data.delete = base_uri + 'job/field/' + field_data.job_try_field_id + '/delete'; }; t2hui.jobtable.field_fetch = function(field_data, item) { - return base_uri + 'job/field/' + field_data.job_field_id; + return base_uri + 'job/field/' + field_data.job_try_field_id; }; @@ -157,7 +160,7 @@ t2hui.jobtable.init_table = function(table, state) { } t2hui.jobtable.place_row = function(row, item, table, state, existing) { - if (!item.short_file) { + if (item.is_harness_out) { state['header'].after(row); return true; } diff --git a/share/js/lookup.js b/share/js/lookup.js index 7c1308dab..750a9f1d3 100644 --- a/share/js/lookup.js +++ b/share/js/lookup.js @@ -32,7 +32,7 @@ $(function() { state.event_table = event_table; } - state.event_table.render_item(item.data, item.data.event_id); + state.event_table.render_item(item.data, item.data.event_uuid); } else if (item.type === 'job') { if (!state.job_table) { @@ -41,7 +41,7 @@ $(function() { jobs.append(job_table.render()); state.job_table = job_table; } - state.job_table.render_item(item.data, item.data.job_key); + state.job_table.render_item(item.data, item.data.job_uuid); } else if (item.type === 'run') { if (!state.run_table) { @@ -50,7 +50,7 @@ $(function() { runs.append(run_table.render()); state.run_table = run_table; } - state.run_table.render_item(item.data, item.data.run_id); + state.run_table.render_item(item.data, item.data.run_uuid); } } ); diff --git a/share/js/project.js b/share/js/project.js index 51843a681..b5d99a2c6 100644 --- a/share/js/project.js +++ b/share/js/project.js @@ -192,7 +192,7 @@ t2hui.project_stats.reload = function(all) { var run_table = t2hui.runtable.build_table(); div.html(run_table.render()); item.runs.forEach(function(run) { - run_table.render_item(run, run.run_id); + run_table.render_item(run, run.run_uuid); }) } diff --git a/share/js/resources.js b/share/js/resources.js index d90bab6df..35668efc3 100644 --- a/share/js/resources.js +++ b/share/js/resources.js @@ -1,10 +1,169 @@ -function build_resource(cont, item) { +$(function() { + var content = $('div#content'); + var runs = $('div#run_list'); + + var state = { + 'min': null, + 'max': null, + 'data': null, + 'rendered': null, + 'tailing': tailing, + 'selected': selected, + 'complete': false, + }; + + t2hui.fetch( + data_uri + "/stream", + { + "done": function() { + if (state.complete) { return } + content.prepend('
    Connection has timed out, reload page to get updates.
    '); + } + }, + function(item) { + if (!item) { return } + if (item.complete) { state.complete = true } + if (item.max) { state.max = item.max } + if (item.min) { state.min = item.min } + + if (item.run_uuid) { + var stream_url = base_uri + 'stream/run/' + item.run_uuid; + var run_table = t2hui.runtable.build_table(); + runs.append(run_table.render()); + + t2hui.fetch( + stream_url, + {}, + function(item) { + if (item.type === 'run') { + run_table.render_item(item.data, item.data.run_uuid); + } + } + ); + } + + redraw_resources(state, item.data); + } + ); +}); + +function redraw_resources(state, data) { + if (!state.min || !state.max) { + return; + } + + if (!state.rendered || !state.range) { + var range = {}; + range.dom = $(''); + range.tail = $(''); + range.stop = $(''); + range.prev = $(''); + range.next = $(''); + range.first = $(''); + range.last = $(''); + + var range_inner = $('
    '); + var range_wrap = $('

    Timerange Selector

    '); + + range.select = $('
    '); + + range_inner.append(range.dom); + range_wrap.append(range.last, range.next, range.tail, range.stop, range.prev, range.first, range_inner, range.select); + + state.pick_range = function(idx, idx_data) { + range.dom.val(idx); + state.selected = idx; + render_resource(idx, idx_data); + }; + + var selector_change = function() { + state.tailing = false; + var idx = range.dom.val(); + state.pick_range(idx); + }; + + range.dom.on('input', selector_change); + range.dom.change(selector_change); + + range.stop.click(function() { state.tailing = false }); + range.tail.click(function() { state.tailing = true; state.pick_range(state.max)}); + range.first.click(function() { state.tailing = false; state.pick_range(state.min)}); + range.last.click(function() { state.tailing = false; state.pick_range(state.max)}); + + range.next.click(function() { + var idx = range.dom.val(); + idx = Number(idx) + 1; + if (idx > state.max) { return } + state.tailing = false; + state.pick_range(idx); + }); + + range.prev.click(function() { + var idx = range.dom.val(); + idx = Number(idx) - 1; + if (idx < state.min) { return } + state.tailing = false; + state.pick_range(idx); + }); + + state.rendered = $('div#resource_wrapper'); + state.rendered.removeClass('loading'); + state.rendered.addClass('rendered'); + state.rendered.empty(); + state.rendered.prepend(range_wrap); + state.range = range; + } + + var range = state.range; + range.dom.attr('min', state.min); + range.dom.attr('max', state.max); + + if (state.tailing && range.selected != state.max) { + state.pick_range(state.max, data); + } +} + +function render_resource(idx, data) { + if (data && data.ord == idx) { + do_render_resource(idx, data.resources); + return; + } + + $.ajax(data_uri + '/' + idx, { + 'data': { 'content-type': 'application/json' }, + 'error': function() { + content.append('
    Could not load resources for index "' + idx + '"
    '); + }, + 'success': function(item) { + do_render_resource(idx, item.resources); + }, + }); +} + +function do_render_resource(idx, data) { + + var content = $('div#content'); + var resources = []; + + data.forEach(function(res) { + var res = build_resource(res); + resources.push(res); + }); + + content.children('div.resource').detach(); + content.append(resources); + + history.replaceState({"index": idx}, null, res_uri + '/' + idx); +} + + +function build_resource(item) { var res = $('
    '); - var name = $('

    ' + item.resource + '

    '); + var name = $('

    ' + item.name + '

    '); res.append(name); - if (item.groups) { - item.groups.forEach(function(group) { + if (item.data) { + item.data.forEach(function(group) { build_group(res, group); }); } @@ -55,175 +214,3 @@ function build_table(res, table) { res.append(t); } - -function select_stamp(stamps) { - if (!stamps) { return } - if (!stamps.select) { return } - - var selected = stamps.selected; - if (!selected) { return } - - var data = stamps.lookup[selected]; - if (data == null) { return } - - var name = data["val"]; - var idx = data["idx"]; - - stamps.selected = null; - stamps.select.html('' + name + ''); - stamps.dom.val(idx); -} - -function load_resource(stamp) { - var content = $('div#content'); - - $.ajax(stamp_uri + '/' + stamp, { - 'data': { 'content-type': 'application/json' }, - 'error': function() { - content.append('
    Could not load resources for timestamp "' + stamp + '"
    '); - }, - 'success': function(item) { - var resources = []; - - item.resources.forEach(function(res) { - var res = build_resource( content, res ); - resources.push(res); - }); - - content.children('div.resource').detach(); - content.append(resources); - - history.replaceState({"stamp": stamp}, null, res_uri + '/' + stamp); - }, - }); -} - -$(function() { - var content = $('div#content'); - var runs = $('div#run_list'); - - var stamps = { - "dom": null, - "list": [], - "lookup": {}, - "selected": null, - "select": null, - }; - - var complete = false; - t2hui.fetch( - stamp_uri, - { - "done": function() { - if (complete) { return } - content.prepend('
    Connection has timed out, reload page to get updates.
    '); - } - }, - function(item) { - if (!item) { return } - if (item.complete) { complete = true } - - if (item.run_id) { - var stream_url = base_uri + 'stream/run/' + item.run_id; - var run_table = t2hui.runtable.build_table(); - runs.append(run_table.render()); - - t2hui.fetch( - stream_url, - {}, - function(item) { - if (item.type === 'run') { - run_table.render_item(item.data, item.data.run_id); - } - } - ); - } - - if (item.stamps) { - if (!stamps.dom) { - stamps.dom = $(''); - stamps.tail = $(''); - stamps.stop = $(''); - stamps.prev = $(''); - stamps.next = $(''); - stamps.first = $(''); - stamps.last = $(''); - var stamp_inner = $('
    '); - var stamp_wrap = $('

    Timestamp Selector

    '); - - stamps.select = $('
    '); - - stamp_inner.append(stamps.dom); - stamp_wrap.append(stamps.last, stamps.next, stamps.tail, stamps.stop, stamps.prev, stamps.first, stamp_inner, stamps.select); - content.find('#put_stamps_here').append(stamp_wrap); - - var selector_change = function() { - tailing = false; - var idx = stamps.dom.val(); - load_resource(stamps.list[idx]); - stamps.selected = stamps.list[idx]; - select_stamp(stamps); - }; - - var pick_stamp = function(stamp) { - load_resource(stamp); - stamps.selected = stamp; - select_stamp(stamps); - } - - stamps.dom.on('input', selector_change); - stamps.dom.change(selector_change); - - stamps.stop.click(function() { tailing = false }); - stamps.tail.click(function() { tailing = true; pick_stamp(stamps.list[stamps.list.length - 1]) }); - stamps.first.click(function() { tailing = false; pick_stamp(stamps.list[0]); }); - stamps.last.click(function() { tailing = false; pick_stamp(stamps.list[stamps.list.length - 1]) }); - - stamps.next.click(function() { - var idx = stamps.dom.val(); - idx = idx - (0 - 1); - if (!stamps.list[idx]) { return } - tailing = false; - pick_stamp(stamps.list[idx]); - }); - - stamps.prev.click(function() { - var idx = stamps.dom.val(); - idx = idx - 1; - if (idx < 0) { return } - if (!stamps.list[idx]) { return } - tailing = false; - pick_stamp(stamps.list[idx]); - }); - - if (selected) { - pick_stamp(selected); - } - } - - item.stamps.forEach(function(stamp) { - var id = stamp[0]; - var val = stamp[1]; - - if (stamps.lookup[id]) { return } - - stamps.list.push(id); - var idx = stamps.list.length - 1 - stamps.lookup[id] = { - "idx": idx, - "val": val, - }; - - stamps.dom.attr("max", idx); - - if (tailing) { - stamps.selected = id; - load_resource(id); - } - }); - - if (stamps.selected && stamps.select) { select_stamp(stamps) } - } - }, - ); -}); diff --git a/share/js/runtable.js b/share/js/runtable.js index 0e29e712f..84ef6d908 100644 --- a/share/js/runtable.js +++ b/share/js/runtable.js @@ -4,10 +4,10 @@ t2hui.runtable.build_table = function() { var columns = [ { 'name': 'tools', 'label': 'tools', 'class': 'tools', 'builder': t2hui.runtable.tool_builder }, - { 'name': 'concurrency', 'label': 'C', 'class': 'count', 'builder': t2hui.runtable.build_concurrency }, - { 'name': 'passed', 'label': 'P', 'class': 'count', 'builder': t2hui.runtable.build_pass }, - { 'name': 'failed', 'label': 'F', 'class': 'count', 'builder': t2hui.runtable.build_fail }, - { 'name': 'retried', 'label': 'R', 'class': 'count', 'builder': t2hui.runtable.build_retry }, + { 'name': 'concurrency', 'label': 'C', 'tooltip': '[C]oncurrency', 'class': 'count', 'builder': t2hui.runtable.build_concurrency }, + { 'name': 'passed', 'label': 'P', 'tooltip': '[P]assed', 'class': 'count', 'builder': t2hui.runtable.build_pass }, + { 'name': 'failed', 'label': 'F', 'tooltip': '[F]ailed', 'class': 'count', 'builder': t2hui.runtable.build_fail }, + { 'name': 'retried', 'label': 'R', 'tooltip': '[R]etried', 'class': 'count', 'builder': t2hui.runtable.build_retry }, { 'name': 'project', 'label': 'project', 'class': 'project', 'builder': t2hui.runtable.build_project }, { 'name': 'status', 'label': 'status', 'class': 'status'}, @@ -46,12 +46,12 @@ t2hui.runtable.place_row = function(row, item, table, state, existing) { } if (!state['biggest']) { - state['biggest'] = item.run_ord; + state['biggest'] = item.run_id; return false; } - if (item.run_ord > state.biggest) { - state['biggest'] = item.run_ord; + if (item.run_id > state.biggest) { + state['biggest'] = item.run_id; table.body.prepend(row); return true; } @@ -64,8 +64,8 @@ t2hui.runtable.build_project = function(item, col) { if (val === null) { return }; if (val === undefined) { return }; - var vlink = base_uri + 'view/' + item.project_id; - var slink = base_uri + 'project/' + item.project_id; + var vlink = base_uri + 'view/project/' + val; + var slink = base_uri + 'project/' + val; var stats = $(''); var proj = $('' + val + ''); @@ -79,17 +79,25 @@ t2hui.runtable.build_user = function(item, col) { if (val === null) { return }; if (val === undefined) { return }; - var vlink = base_uri + 'view/' + item.user_id; + var vlink = base_uri + 'view/user/' + val; var proj = $('' + val + ''); col.append(proj); }; t2hui.runtable.build_concurrency = function(item, col) { - var val = item.concurrency; - if (val === null) { return }; - if (val === undefined) { return }; - col.text("-j" + val); + var valj = item.concurrency_j; + var valx = item.concurrency_x; + if (valj === null) { return }; + if (valj === undefined) { return }; + + var val = "-j" + valj; + + if (valx) { + val = val + ":" + valx; + } + + col.text(val); }; t2hui.runtable.build_pass = function(item, col) { @@ -104,7 +112,7 @@ t2hui.runtable.build_fail = function(item, col) { if (val === null) { return }; if (val === undefined) { return }; if (val == 0) { col.append($('
    ' + val + '
    ')) } - else { col.append($('' + val + '')) } + else { col.append($('' + val + '')) } }; t2hui.runtable.build_retry = function(item, col) { @@ -116,8 +124,8 @@ t2hui.runtable.build_retry = function(item, col) { }; t2hui.runtable.tool_builder = function(item, tools, data) { - var link = base_uri + 'view/' + item.run_id; - var downlink = base_uri + 'download/' + item.run_id; + var link = base_uri + 'view/' + item.run_uuid; + var downlink = base_uri + 'download/' + item.run_uuid; var params = $('
    '); tools.append(params); @@ -125,7 +133,7 @@ t2hui.runtable.tool_builder = function(item, tools, data) { $('#modal_body').html("Loading..."); $('#free_modal').slideDown(); - var url = base_uri + 'run/' + item.run_id + '/parameters'; + var url = base_uri + 'run/' + item.run_uuid + '/parameters'; $.ajax(url, { 'data': { 'content-type': 'application/json' }, 'error': function(a, b, c) { alert("Failed to load run paramaters") }, @@ -179,7 +187,7 @@ t2hui.runtable.tool_builder = function(item, tools, data) { var ok = confirm("Are you sure you wish to cancel this run? This action cannot be undone!\nNote: This only changes the runs status, it will not stop a running test. This is used to 'fix' an aborted run that is still set to 'running'"); if (!ok) { return; } - var url = base_uri + 'run/' + item.run_id + '/cancel'; + var url = base_uri + 'run/' + item.run_uuid + '/cancel'; $.ajax(url, { 'data': { 'content-type': 'application/json' }, 'error': function(a, b, c) { alert("Failed to cancel run") }, @@ -199,18 +207,18 @@ t2hui.runtable.tool_builder = function(item, tools, data) { var ok = confirm("Are you sure you wish to delete this run? This action cannot be undone!"); if (!ok) { return; } - var url = base_uri + 'run/' + item.run_id + '/delete'; + var url = base_uri + 'run/' + item.run_uuid + '/delete'; $.ajax(url, { 'data': { 'content-type': 'application/json' }, 'error': function(a, b, c) { alert("Could not delete run") }, 'success': function() { - $('tr#' + item.run_id).remove(); + $('tr#' + item.run_uuid).remove(); }, }); }); } - var resources = $('♻'); + var resources = $('♻'); tools.append(resources); var cimg = $(''); @@ -220,7 +228,7 @@ t2hui.runtable.tool_builder = function(item, tools, data) { dcover.append(dcimg); if (item.has_coverage && item.status === 'complete') { - var curl = base_uri + 'coverage/' + item.run_id; + var curl = base_uri + 'coverage/' + item.run_uuid; var clink = $(''); clink.append(cimg); cover.append(clink); @@ -268,7 +276,7 @@ t2hui.runtable.tool_builder = function(item, tools, data) { tools.prepend(pintool); pintool.click(function() { - var url = base_uri + 'run/' + item.run_id + '/pin'; + var url = base_uri + 'run/' + item.run_uuid + '/pin'; $.ajax(url, { 'data': { 'content-type': 'application/json' }, 'error': function(a, b, c) { alert("Failed to pin run") }, diff --git a/share/js/view.js b/share/js/view.js index 11dc49b48..e90fd8fa9 100644 --- a/share/js/view.js +++ b/share/js/view.js @@ -12,14 +12,14 @@ $(function() { state.run_table.make_sortable(); } - if (state.job_table) { + if (state.job_table && state.has_non_harness_job) { state.job_table.make_sortable(); } }}, function(item) { if (item.type === 'event') { - item.data.run_id = state.run.run_id; - item.data.job_key = state.job.job_key; + item.data.run_uuid = state.run.run_uuid; + item.data.job_uuid = state.job.job_uuid; state.event = item.data; if (!state.event_table) { var event_controls = t2hui.eventtable.build_controls(state.run, state.job); @@ -36,17 +36,22 @@ $(function() { } } - state.event_table.render_item(item.data, item.data.event_id); + state.event_table.render_item(item.data, item.data.event_uuid); } else if (item.type === 'job') { - item.data.run_id = state.run.run_id; + item.data.run_uuid = state.run.run_uuid; state.job = item.data; + + if (!state.job.is_harness_out) { + state.has_non_harness_job = 1; + } + if (!state.job_table) { var job_table = t2hui.jobtable.build_table(state.run); jobs.append(job_table.render()); state.job_table = job_table; } - state.job_table.render_item(item.data, item.data.job_key); + state.job_table.render_item(item.data, item.data.job_try_id); } else if (item.type === 'run') { state.run = item.data; @@ -55,7 +60,7 @@ $(function() { runs.append(run_table.render()); state.run_table = run_table; } - state.run_table.render_item(item.data, item.data.run_id); + state.run_table.render_item(item.data, item.data.run_uuid); } } ); diff --git a/share/psgi/demo.psgi b/share/psgi/demo.psgi deleted file mode 100644 index b6399e12b..000000000 --- a/share/psgi/demo.psgi +++ /dev/null @@ -1,33 +0,0 @@ -use strict; -use warnings; - -BEGIN {$ENV{T2_HARNESS_UI_ENV} = 'dev'} - -use Plack::Builder; -use Plack::App::Directory; -use Plack::App::File; - -use Test2::Harness::UI::Util qw/share_dir share_file/; - -builder { - enable "DBIx::DisconnectAll"; - mount '/js' => Plack::App::Directory->new({root => share_dir('js')})->to_app; - mount '/css' => Plack::App::Directory->new({root => share_dir('css')})->to_app; - mount '/img' => Plack::App::Directory->new({root => share_dir('img')})->to_app; - mount '/favicon.ico' => Plack::App::File->new({file => share_file('img/favicon.ico')})->to_app; - - mount '/' => sub { - require Test2::Harness::UI; - require Test2::Harness::UI::Config; - - my $config = Test2::Harness::UI::Config->new( - dbi_dsn => $ENV{HARNESS_UI_DSN}, - dbi_user => '', - dbi_pass => '', - single_user => 1, - show_user => 1, - ); - - Test2::Harness::UI->new(config => $config)->to_app->(@_); - } -} diff --git a/share/psgi/test.psgi b/share/psgi/test.psgi deleted file mode 100644 index b6399e12b..000000000 --- a/share/psgi/test.psgi +++ /dev/null @@ -1,33 +0,0 @@ -use strict; -use warnings; - -BEGIN {$ENV{T2_HARNESS_UI_ENV} = 'dev'} - -use Plack::Builder; -use Plack::App::Directory; -use Plack::App::File; - -use Test2::Harness::UI::Util qw/share_dir share_file/; - -builder { - enable "DBIx::DisconnectAll"; - mount '/js' => Plack::App::Directory->new({root => share_dir('js')})->to_app; - mount '/css' => Plack::App::Directory->new({root => share_dir('css')})->to_app; - mount '/img' => Plack::App::Directory->new({root => share_dir('img')})->to_app; - mount '/favicon.ico' => Plack::App::File->new({file => share_file('img/favicon.ico')})->to_app; - - mount '/' => sub { - require Test2::Harness::UI; - require Test2::Harness::UI::Config; - - my $config = Test2::Harness::UI::Config->new( - dbi_dsn => $ENV{HARNESS_UI_DSN}, - dbi_user => '', - dbi_pass => '', - single_user => 1, - show_user => 1, - ); - - Test2::Harness::UI->new(config => $config)->to_app->(@_); - } -} diff --git a/share/templates/main.tx b/share/templates/main.tx index 5ca434a67..f79aec19f 100644 --- a/share/templates/main.tx +++ b/share/templates/main.tx @@ -36,8 +36,8 @@ - - + +