diff --git a/Build.PL b/Build.PL index d51cb41..623b603 100644 --- a/Build.PL +++ b/Build.PL @@ -2,7 +2,30 @@ use strict; use warnings; use Module::Build; -my $builder = Module::Build->new( +my $class = Module::Build->subclass(code => <<'YAPP'); +use Parse::Yapp; +sub process_yapp_files { + my $self = shift; + + if ( $self->{invoked_action} && $self->{invoked_action} =~ m/^build$/i ) { + print "Build yapp files\n"; + + my $yapp_file = q{syncdiff_cfg_parser.y}; + my ($pack_file, $pack_name) = ("FileSync/SyncDiff/ParseCfg.pm", "FileSync::SyncDiff::ParseCfg"); + eval { + my ($yapp) = Parse::Yapp->new(inputfile => $yapp_file); + open(my $fh, '>', $pack_file); + print $fh $yapp->Output(classname => $pack_name); + close($fh); + }; + if ($@) { + print "Can not build yapp files $! \n"; + } + } +} +YAPP + +my $builder = $class->new( module_name => 'SyncDiff', dist_name => 'syncdiff', dist_version => '0.1', @@ -10,16 +33,21 @@ my $builder = Module::Build->new( dist_author => 'warthog9@eaglescrag.net', dist_abstract => 'SyncDiff(erent) is a statefull rsync-like file synchronizer. Think rsync + git + csync2 + unison', requires => { - 'Module::Build' => 0.4205, + 'Module::Build' => 0.4214, 'Moose' => 1.15, 'Parse::Lex' => 0, 'DBD::SQLite' => 1.27, 'Digest::SHA' => 5.47, + 'Parse::Yapp' => 0, 'Parse::Yapp::Driver' => 1.05, 'File::FnMatch' => 0.02, 'Try::Tiny' => 0.11, 'File::Rdiff' => 1.0, 'JSON::XS' => 2.27, + 'URI' => 1.67, + 'Plack' => 1.0037, + 'IO::Socket::Forwarder' => 0.02, + 'Log::Log4perl' => 1.46, }, meta_merge => { resources => { @@ -28,6 +56,6 @@ my $builder = Module::Build->new( }, test => {TESTS => 't/*.t'} ); - +$builder->add_build_element('yapp'); $builder->create_build_script(); print "Have a great day!\n"; diff --git a/FileSync/SyncDiff/Client.pm b/FileSync/SyncDiff/Client.pm index 9fbd02c..3818f79 100644 --- a/FileSync/SyncDiff/Client.pm +++ b/FileSync/SyncDiff/Client.pm @@ -42,6 +42,7 @@ extends qw(FileSync::SyncDiff::Forkable); use FileSync::SyncDiff::File; use FileSync::SyncDiff::Util; use FileSync::SyncDiff::Protocol::v1; +use FileSync::SyncDiff::Log; # # Other Includes @@ -51,6 +52,10 @@ use JSON::XS; use MIME::Base64; use IO::Socket; use IO::Handle; +use IO::Socket::INET; +use LWP::UserAgent; +use URI; +use Net::Domain qw(domainname); use Scalar::Util qw(looks_like_number); @@ -111,6 +116,16 @@ has 'protocol_object' => ( isa => 'Object', ); +# Logger system +has 'log' => ( + is => 'rw', + isa => 'FileSync::SyncDiff::Log', + default => sub { + my $self = shift; + return FileSync::SyncDiff::Log->new( config => $self->config ); + } +); + # End variables # @@ -162,25 +177,15 @@ sub fork_and_connect { my( $self ) = @_; my $dbref = $self->dbref(); - print "Client::fork_and_connect - ". $self->group ." - ". $self->groupbase ."\n"; - print Dumper $self->config_options; + $self->log->debug("Client::fork_and_connect - %s - %s",$self->group,$self->groupbase); + $self->log->debug("Config options: %s",Dumper($self->config_options)); - print "Client::fork_and_connect - path\n"; - print Dumper $self->groupbase_path; + $self->log->debug("Client::fork_and_connect - path %s", Dumper($self->groupbase_path)); if( ! -e $self->groupbase_path ){ - die( "Path: ". $self->groupbase_path ." does *NOT* exist in group ". $self->group ." - sadly dying now.\n" ); + $self->log->fatal( "Path: %s does *NOT* exist in group %s - sadly dying now", $self->groupbase_path, $self->group ); } - # - # Going chroot - # everything else we need should - # be in the chroot, or accessible - # via pipes - # - chroot( $self->groupbase_path ); - chdir("/"); - # # Ok now we need to connect to the # various hosts associated with @@ -206,16 +211,61 @@ sub fork_and_connect { # foreach my $host ( @{ $self->config_options->{groups}->{ $self->group }->{host} } ){ - print "Host: ". $host ."\n"; - my $ip = $self->dbref->gethostbyname( $host ); - print "Ip: ". $ip ."\n"; - my $sock = new IO::Socket::INET ( - PeerAddr => $ip, - PeerPort => '7070', - Proto => 'tcp', - ); + $self->log->debug("Host: %s", $host->{host}); + my $ip = $self->dbref->gethostbyname( $host->{host} ); + $self->log->debug("Ip: %s", $ip); + my $port = $host->{port} || '7070'; + + if ( $host->{proto} && $host->{proto} =~ /^http[s]?$/ ) { + my %agent_opt = ( timeout => 10 ); + my $ua = LWP::UserAgent->new(%agent_opt); + my $uri = URI->new(); + + my $params = { + key => $self->config_options->{groups}->{ $self->group }->{key}, + include => $self->groupbase_path, + host => inet_ntoa(inet_aton(domainname())), + }; + $uri->scheme($host->{proto}); + $uri->host($host->{host}); + $uri->port($host->{port}); + + my $response = $ua->post($uri,$params); + if ( $response->is_success ) { + $self->log->info("Success response from %s", $host->{host}); + } + else { + my $msg = $response->message; + $self->log->error("Failed response from %s : %s", $host->{host}, $msg); + next; + } + + my $json = decode_json($response->decoded_content); + if ( $json ) { + $ip = $json->{host} ? $json->{host} : $ip; + $port = $json->{port} ? $json->{port} : $port; + } + + } + + # + # Going chroot + # everything else we need should + # be in the chroot, or accessible + # via pipes + # + chroot( $self->groupbase_path ); + chdir("/"); + + my $sock = eval { + IO::Socket::INET->new( + PeerAddr => $ip, + PeerPort => $port, + Proto => 'tcp', + ); + }; if( ! $sock ){ - print "Could not create socket: $!\n"; + $self->log->warn("Could not create socket: %s", $!); next; } # end skipping if the socket is broken @@ -223,8 +273,6 @@ sub fork_and_connect { $self->socket( $sock ); -# print $sock "Hello World!\n"; -# # # We need to authenticate against the server # before we try to negotiate a protocol @@ -232,10 +280,10 @@ sub fork_and_connect { my $auth_status = $self->authenticate_to( $dbref->getlocalhostname, $self->group, $self->config_options->{groups}->{ $self->group }->{key} ); - print "Auth Status: $auth_status\n"; + $self->log->debug("Auth Status: %s",$auth_status); if( $auth_status == 0 ){ - print "Authentication failed for $host\n"; + $self->log->info("Authentication failed for %s", $host->{host}); $sock->shutdown(2); next; } @@ -243,7 +291,7 @@ sub fork_and_connect { # # Ok, here we get the proper protocol all worked out # - $self->request_protocol_versions( $host ); + $self->request_protocol_versions( $host->{host} ); # # Next we should let the protocol object take over @@ -253,7 +301,7 @@ sub fork_and_connect { # complex or a major issue. Pass it on and let go # - print "Protocol should be setup\n"; + $self->log->debug("Protocol should be setup"); my $protocol_obj = $self->protocol_object(); $protocol_obj->client_run(); @@ -274,8 +322,7 @@ sub authenticate_to { my $auth_status = $self->basic_send_request( %request ); - print "authenticate_to status:\n"; - print Dumper $auth_status; + $self->log->debug("authenticate_to status: %s",Dumper ($auth_status)); if( $auth_status == 0 ){ return 1; @@ -291,21 +338,20 @@ sub request_protocol_versions { 'operation' => 'request_protocol_versions', ); - print "Going to request Protocol Versions:\n"; + $self->log->debug("Going to request Protocol Versions:"); my $versions = $self->basic_send_request( %request ); - print "Got Back Version:\n"; - print Dumper $versions; + $self->log->debug("Got Back Version: %s",Dumper ($versions)); my $highest_proto_supported = "1.99"; my $proto_to_use = 0; foreach my $ver ( @{$versions} ){ - print "Version: ". $ver ."\n"; + $self->log->debug("Version: %s", $ver); if( ! looks_like_number($ver) ){ - print "*** $ver is not a version number\n"; + $self->log->warn("*** %s is not a version number",$ver); next; } @@ -315,11 +361,11 @@ sub request_protocol_versions { $ver >= $proto_to_use ){ $proto_to_use = $ver; - print "Currently selected Protocol Version: ". $ver ."\n"; + $self->log->debug("Currently selected Protocol Version: %s", $ver); } } # end foreach $ver - $self->protocol_version( $proto_to_use ); + $self->protocol_version( $proto_to_use ); my $protocol_obj; @@ -328,7 +374,7 @@ sub request_protocol_versions { && $proto_to_use < 2.0 ){ - $protocol_obj = FileSync::SyncDiff::Protocol::v1->new( socket => $self->socket, version => $proto_to_use, dbref => $self->dbref, group => $self->group, hostname => $host, groupbase => $self->groupbase ); + $protocol_obj = FileSync::SyncDiff::Protocol::v1->new( socket => $self->socket, version => $proto_to_use, dbref => $self->dbref, group => $self->group, hostname => $host, groupbase => $self->groupbase, log => $self->log ); } $protocol_obj->setup(); @@ -380,8 +426,8 @@ sub basic_send_request { chomp( $line ); - print "Basic send receive line back:\n"; - print Dumper $line; + $self->log->debug("Basic send receive line back:"); + $self->log->debug(Dumper $line); if( $line eq "0" ){ return 0; @@ -389,8 +435,8 @@ sub basic_send_request { my $response = decode_json( $line ); - print Dumper $response; - print "Ref: ". ref( $response ). "\n"; + $self->log->debug('Response: %s', Dumper ($response)); + $self->log->debug("Ref: %s", ref( $response )); if( ref( $response ) eq "ARRAY" ){ return $response; @@ -411,8 +457,6 @@ sub basic_send_request { return $response; } # end send_request() -#no moose; __PACKAGE__->meta->make_immutable; -#__PACKAGE__->meta->make_immutable(inline_constructor => 0,); 1; diff --git a/FileSync/SyncDiff/Config.pm b/FileSync/SyncDiff/Config.pm index 2e44701..2f7be9e 100644 --- a/FileSync/SyncDiff/Config.pm +++ b/FileSync/SyncDiff/Config.pm @@ -52,15 +52,18 @@ has 'lexer' => ( ); # end config ## writer => '_write_config', +has 'log' => ( + is => 'rw', + isa => 'FileSync::SyncDiff::Log', + default => sub { + my $self = shift; + return FileSync::SyncDiff::Log->new(); + } +); + sub read_config { my ($self, $config_file) = @_; -# print "---------------------------\n"; -# print "config file:\n"; -# print "---------------------------\n"; -# print Dumper $config_file; -# print "^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - open( my $fh, "<", $config_file) or die "can't open $config_file - you've done something horribly wrong! $!\n"; my @tokens = ( @@ -94,49 +97,21 @@ sub read_config { NL [\n\r] ), qw(COMMENT), "#.*\n*" , - qw(TK_STRING), [qw(" (?:[^"]+|"")* ")], + qw(TK_STRING), [qw(["'] (?:[^"]+|"")* ["'])], qw(TK_STRING), q([^\s;:{}\(\)\@\n\r\#]+), qw(config), sub { - print "got an additional config file $_[1]\n"; + $self->log->debug("got an additional config file %s", $_[1]); }, qw(ERROR .*), sub { - die qq!can\'t analyze: "$_[1]"!; + $self->log->fatal("can not analyze: %s", $_[1]); }, ); -# qw(COMMENT), qw( \#.*\n* ), -# qw(TK_STRING), qw(\S+), -# COMMENT #.*\n* - - -# print "FileSync::SyncDiff::Config->read_config() - Turning trace on\n"; -# Parse::Lex->trace; # Class method -# print "FileSync::SyncDiff::Config->read_config() - Trace is enabled\n"; - - #print Dumper $self; - -## print Dumper \@tokens; -## print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; my $lexer = Parse::Lex->new(@tokens); - #$lexer->from($fh); - -## print "FileSync::SyncDiff::Config->read_config() - Made it past adding the tokens\n"; - - #bless $lexer, "Parse::Lex"; $self->{lexer} = \$lexer; - #my $new_lexer = $self->{lexer}; - - #bless $new_lexer, "Parse::Lex"; - -## print "-------------------------------------\n"; -## print Dumper $self; -## print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - - print "Tokenization of DATA:\n"; - -## print "Reference: ". ref( ${ $self->{lexer} } ) ."\n"; + $self->log->debug("Tokenization of DATA:"); my $parser = FileSync::SyncDiff::ParseCfg->new(); @@ -151,62 +126,24 @@ sub read_config { $parser->YYData->{DATA} = $file_data; $lexer->from( $parser->YYData->{DATA} ); -## print "FileSync::SyncDiff::Config->read_config() - About to run the parser...\n"; - $parser->YYParse( YYlex => $callback_ref_lex, YYerror => \&perl_error, ); - # YYdebug => 0x1F, - -## print "FileSync::SyncDiff::Config->read_config() - Made it past the parser\n"; - ##YYdebug => 0x1F, - -# print "#########################################################\n"; -# print Dumper \$parser; -# print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - -# my %groups = $parser->get_groups(); -# print "#########################################################\n"; -# print Dumper \%groups; -# print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - -# print "#########################################################\n"; -# print Dumper \$self; -# print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; my %config; - #print Dumper $self->config; if( defined undef ){ %config = $self->config; } -# print "#########################################################\n"; -# print Dumper $self->config; -# print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - my %group_temp = $parser->get_groups(); $config{'groups'} = \%group_temp; my %prefixes_temp = $parser->get_prefixes(); $config{'prefixes'} = \%prefixes_temp; ( $config{'ignore_uid'}, $config{'ignore_gid'}, $config{'ignore_mod'} ) = $parser->get_ignores(); -# print "#########################################################\n"; -# print Dumper \%config; -# print "---------------------------------------------------------\n"; -# print Dumper %group_temp; -# print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - -# print "#########################################################\n"; -# print Dumper \%config; -# print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - $self->_write_config( \%config ); - -# print "#########################################################\n"; -# print Dumper $self->config; -# print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; } # end read_config() sub lex { @@ -214,20 +151,16 @@ sub lex { my $token = ${ $self->{lexer} }->next; - #return("\n", "\n") if( $token->name eq "NL" ); + if ( $token->name =~ m/^TK_STRING$/ ) { + # Cut ' and " in beginning + (my $text = $token->text) =~ m/(['"])?(?.*?)\g{-2}/; + $token->text($+{str}); + } + while( $token->name eq "NL" || $token->name eq "COMMENT"){ -# print "Lexer: Line $.\t"; -# print "Lexer: Type: ", $token->name, "\t"; -# print "Lexer: Getstring: ". $token->getText ."\t"; -# print "Lexer: Content:->", $token->text, "<-\n"; $token = ${ $self->{lexer} }->next; } -# print "Lexer: Line $.\t"; -# print "Lexer: Type: ", $token->name, "\t"; -# print "Lexer: Getstring: ". $token->getText ."\t"; -# print "Lexer: Content:->", $token->text, "<-\n"; - return ('', undef) if ${ $self->{lexer} }->eoi; return ($token->name, $token->name) if( $token->getText eq "" ); return ($token->name, $token->getText); @@ -251,10 +184,6 @@ sub get_truepath { my $prefix = $path; $prefix =~ s/^\%([^\%]+)\%.*$/$1/; -## print "FileSync::SyncDiff::Config->get_truepath() - prefix: ". $prefix ."\n"; - - #print Dumper $self->{config}->{prefixes}; - my $prefix_path = $self->{config}->{prefixes}->{$prefix}; my $truepath = $path; @@ -279,7 +208,6 @@ sub get_group_config { return \%group_config; } # end get_group_config() -#no moose; __PACKAGE__->meta->make_immutable; 1; diff --git a/FileSync/SyncDiff/DB.pm b/FileSync/SyncDiff/DB.pm index 30c71aa..36030c7 100644 --- a/FileSync/SyncDiff/DB.pm +++ b/FileSync/SyncDiff/DB.pm @@ -41,6 +41,7 @@ extends qw(FileSync::SyncDiff::Forkable); use FileSync::SyncDiff::File; use FileSync::SyncDiff::Util; use FileSync::SyncDiff::Config; +use FileSync::SyncDiff::Log; # # Needed for dealing with DB stuff @@ -82,6 +83,16 @@ has 'config' => ( required => 1, ); +# Logger system +has 'log' => ( + is => 'rw', + isa => 'FileSync::SyncDiff::Log', + default => sub { + my $self = shift; + return FileSync::SyncDiff::Log->new( config => $self->config ); + } +); + # End variables sub connect_and_fork { @@ -98,8 +109,6 @@ sub connect { $self->file( $file_to_open ); } -## print "DB:connect(): File as it currently exists: |". $self->file ."|\n"; - my $file = $self->file; if( @@ -107,7 +116,7 @@ sub connect { || $file eq "" ){ - die("Database file not defined\n"); + $self->log->fatal("Database file not defined"); } my $dbh = DBI->connect( @@ -132,10 +141,9 @@ sub connect { my $sth = $dbh->table_info("", "%", 'transactions', "TABLE"); if ( ! $sth->fetch) { # doesn't exist - print "*** New Database, initializing and doing a file scan\n"; + $self->log->info("*** New Database, initializing and doing a file scan"); $self->create_database(); -# FileSync::SyncDiff::Scanner->full_scan( $self->config, $self ); } return; @@ -144,20 +152,11 @@ sub connect { # Beyond this is all random testing code # -## print "----------------------------------\n"; -## print Dumper $dbh; -## print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - my $select_all = $dbh->prepare("SELECT * FROM files"); $select_all->execute(); my $row_ref = $select_all->fetchall_hashref('id'); - -## print "----------------------------------\n"; -## print Dumper \$row_ref; -## print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - } # end connect() # @@ -179,11 +178,6 @@ sub recv_loop { chomp($line); my $response = $self->process_request( $line ); -## print "DB:recv_loop() - going to push response back at parent\n"; - -## print Dumper $response; -## print "DB:recv_loop() - pushing...\n"; - if( $response eq "0" ){ @@ -193,9 +187,6 @@ sub recv_loop { $response = \%temp_resp; } -## print "Reference check: ". ref( $response ) ."\n"; -## print Dumper $response; - my $ref_resp = ref( $response ); if( @@ -211,9 +202,6 @@ sub recv_loop { $response = \%temp_resp; } -## print "Why is this a dud:\n"; -## print Dumper $response; - my $json_response = encode_json( $response ); print $PARENT_IPC $json_response ."\n"; } @@ -222,21 +210,14 @@ sub recv_loop { sub process_request { my( $self, $line ) = @_; -## print "-----------------------\n"; -## print "DB:process_request - line:\n"; -## print Dumper $line; -## print "^^^^^^^^^^^^^^^^^^^^^^^\n"; - my $request = decode_json( $line ); if( ! defined $request->{operation} ){ - print "FileSync::SyncDiff::DB->process_request() - No Operation specified!\n"; - print Dumper $request; + $self->log->debug("FileSync::SyncDiff::DB->process_request() - No Operation specified!"); + $self->log->debug(Dumper $request); return; } -## print "FileSync::SyncDiff::DB->process_request() - Operation: |". $request->{operation} ."|\n"; - if( $request->{operation} eq "new_transaction_id" ){ return $self->_new_transaction_id( $request->{group}, $request->{transaction_id} ); } @@ -329,11 +310,13 @@ sub create_database { $dbh->do("CREATE TABLE servers_seen (id INTEGER PRIMARY KEY AUTOINCREMENT, hostname TEXT unique, transactionid TEXT, 'group' TEXT, timeadded INTEGER)"); + $dbh->do("CREATE TABLE if not exists connections (id INTEGER PRIMARY KEY AUTOINCREMENT, host TEXT, port TEXT, auth_key TEXT, syncbase TEXT, timeadded INTEGER)"); + my $transaction_id = sha256_hex( hostname() ."-". $$ ."-". time() ); - print Dumper $self->config->config; + $self->log->debug('Config: %s', Dumper ($self->config->config)); foreach my $group ( sort keys %{ $self->config->config->{groups} } ){ - print "Group: $group\n"; + $self->log->debug("Group: %s", $group); $self->_new_transaction_id( $group, $transaction_id ); } } # end create_database() @@ -341,17 +324,12 @@ sub create_database { sub send_request { my( $self, %request ) = @_; -## print "FileSync::SyncDiff::DB->send_request() - Starting\n"; my $json = encode_json( \%request ); my $db_pipe = $self->CHILD_IPC; -## print Dumper $db_pipe; - print $db_pipe $json ."\n"; -## print "We sent the thing off, waiting for return\n"; - my $line = undef; while( $line = <$db_pipe> ){ @@ -361,15 +339,8 @@ sub send_request { } } -## print Dumper $line; - chomp( $line ); -## print "Got response\n"; - -## print "*** DB->send_request() - return line:\n"; -## print Dumper $line; - if( $line eq "0" ){ return 0; } @@ -391,11 +362,73 @@ sub send_request { return $response; } +sub new_connection { + my ($self,$info) = @_; + my $dbh = $self->dbh; + + my $res = eval { + my $add_connection = $dbh->prepare("INSERT INTO connections (host, port, auth_key, syncbase, timeadded) VALUES( ?, ?, ?, ?, strftime('%s','now') )"); + $add_connection->execute( $info->{host}, $info->{port}, $info->{auth_key}, $info->{syncbase} ) || die $DBI::errstr; + }; + if ($@) { + $self->log->error("Error on add new connection: %s,", $@); + } + + return $res; +} + +sub clean_connections { + my ($self,$info) = @_; + my $dbh = $self->dbh; + + my $sql = qq{ DELETE FROM connections }; + my @params = (); + if ( defined $info ) { + push @params, { $info->{host} => 'host = ?' } if ( exists $info->{host} ); + push @params, { $info->{port} => 'port = ?' } if ( exists $info->{port} ); + push @params, { $info->{auth_key} => 'auth_key = ?' } if ( exists $info->{auth_key} ); + push @params, { $info->{syncbase} => 'syncbase = ?' } if ( exists $info->{syncbase} ); + if ( scalar @params > 0 ){ + $sql .= qq{ WHERE }; + $sql .= join( ' AND ', map { values %{$_} }@params ); + } + } + + my $res = eval { + my $del_connection = $dbh->prepare($sql); + $del_connection->execute( map { keys %{$_} }@params ) || die $DBI::errstr; + }; + if ($@) { + $self->log->error("Error on clean up connections: %s", $@); + } + + return $res; +} + +sub is_exists_connection { + my ($self,$info) = @_; + my $dbh = $self->dbh; + my $sth = undef; + + eval { + $sth = $dbh->prepare("SELECT * FROM connections WHERE host = ? AND auth_key = ? AND syncbase = ?"); + $sth->execute( $info->{host}, $info->{auth_key}, $info->{syncbase} ) || die $DBI::errstr; + }; + if ($@) { + $self->log->error("Error with working on connections: %s", $@); + } + + my $row_ref = $sth->fetchall_hashref('id'); + if( scalar ( keys %$row_ref ) != 0 ){ + return 1; + } + + return 0; +} + sub new_transaction_id { my( $self, $group, $transaction_id ) = @_; -## print "FileSync::SyncDiff::DB->new_transaction_id() - Starting\n"; - my %request = ( operation => 'new_transaction_id', group => $group, @@ -409,12 +442,6 @@ sub _new_transaction_id { my( $self, $group, $transaction_id ) = @_; my $dbh = $self->dbh; -## print "~~~ Adding a transaction\n"; - -## print Dumper $dbh; -## -## print "^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - my $add_transaction = $dbh->prepare("INSERT INTO transactions (`group`, transactionid, timeadded) VALUES( ?, ?, strftime('%s','now') )"); $add_transaction->execute( $group, $transaction_id ); @@ -425,12 +452,11 @@ sub _get_transaction_id { my( $self, $group, $transactionid ) = @_; my $dbh = $self->dbh; - my $sth = $dbh->prepare("SELECT * FROM transactions WHERE transactionid=? and `group`=?"); $sth->execute( $transactionid, $group ); if ( $sth->err ){ - die "ERROR! return code: ". $sth->err . " error msg: " . $sth->errstr . "\n"; + $self->log->fatal("ERROR! return code: %s error msg: %s", $sth->err, $sth->errstr); } my $row_ref = $sth->fetchall_hashref('id'); @@ -453,8 +479,6 @@ sub lookup_file { my $response = $self->send_request( %request ); -## print Dumper $response; - return $response; } #end lookup_file() @@ -494,24 +518,16 @@ sub lookup_filelist { my $response = $self->send_request( %request ); -## print Dumper $response; - my @filelist; foreach my $id ( sort keys %$response ){ -## print "Hash ID: ". $id ."\n"; my $fileobj = FileSync::SyncDiff::File->new( dbref => $self ); $fileobj->from_hash( $response->{$id} ); -# my %filehash = $fileobj->to_hash(); -# -# push( @filelist, \%filehash ); push( @filelist, $fileobj ); } -## print Dumper \@filelist; - return \@filelist; } # end lookup_filelist() @@ -528,9 +544,6 @@ sub _lookup_filelist { my $row_ref = $lookup_file->fetchall_hashref('id'); -## print "**** FILELIST\n"; -## print Dumper $row_ref; - if( ( scalar ( keys %$row_ref ) ) == 0 ){ return 0; } @@ -538,16 +551,6 @@ sub _lookup_filelist { my %filelist_arr; my %return_hash; -# foreach my $id ( sort keys %$row_ref ){ -# print "Hash ID: ". $id ."\n"; -# my $fileobj = FileSync::SyncDiff::File->new( dbref => $self ); -# -# $fileobj->parse_dbrow( $row_ref->{$id} ); -# -# %return_hash = $fileobj->to_hash(); -# } - - #return \%return_hash; return $row_ref; } # end _lookup_filelist() @@ -649,9 +652,6 @@ sub add_file { my %file_hash = $file->to_hash(); -## print "File hash:\n"; -## print Dumper \%file_hash; - my %request = ( operation => 'add_file', file => \%file_hash, @@ -664,17 +664,9 @@ sub _add_file { my( $self, $file ) = @_; my $dbh = $self->dbh; -## print "------------------------\n"; -## print "DB->_add_file()\n"; -## print "------------------------\n"; -## print Dumper $file; -## print "------------------------\n"; - my $file_obj = FileSync::SyncDiff::File->new(dbref => $self ); $file_obj->from_hash( $file ); -## print Dumper \$file_obj; -## print "^^^^^^^^^^^^^^^^^^^^^^^^\n"; my $new_file_sth = $dbh->prepare("INSERT INTO files (filepath, syncgroup, syncbase, filetype, inode_num, perms, uid, username, gid, groupname, size_bytes, mtime, extattr, checksum, last_transaction, deleted) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0 )"); @@ -719,16 +711,8 @@ sub _mark_deleted { my $file_obj = FileSync::SyncDiff::File->new(dbref => $self ); $file_obj->from_hash( $file ); -## print "Marking deleted:\n"; -## print "\tlast transaction: ". $file_obj->last_transaction ."\n"; -## print "\tFilepath: ". $file_obj->filepath ."\n"; -## print "\tsyncgroup: ". $file_obj->syncgroup ."\n"; -## print "\tsyncbase: ". $file_obj->syncbase ."\n"; - my $sql = "UPDATE files set last_transaction=?, deleted=1 WHERE filepath=? and syncgroup=? and syncbase=?"; -## print "\tSQL: ". $sql ."\n"; - my $mark_deleted_sth = $dbh->prepare($sql); $mark_deleted_sth->execute( @@ -740,7 +724,7 @@ sub _mark_deleted { ); if ( $mark_deleted_sth->err ){ - die "ERROR! return code: ". $mark_deleted_sth->err . " error msg: " . $mark_deleted_sth->errstr . "\n"; + $self->log->fatal("ERROR! return code: %s error msg: %s", $mark_deleted_sth->err, $mark_deleted_sth->errstr); } return 0; @@ -836,12 +820,12 @@ sub current_log_position { operation => 'current_log_position', ); - print "Pushing request for current log position\n"; + $self->log->debug("Pushing request for current log position"); my $response = $self->send_request( %request ); - print "Got response and it is...\n"; - print Dumper $response; + $self->log->debug("Got response and it is..."); + $self->log->debug(Dumper $response); return $response; } # end current_log_position() @@ -849,20 +833,19 @@ sub _current_log_position { my( $self ) = @_; my $dbh = $self->dbh; - print "Got request for current log position\n"; + $self->log->debug("Got request for current log position"); my $get_current_transaction_id = $dbh->prepare("select id, transactionid from transactions order by id desc limit 1;"); $get_current_transaction_id->execute(); if ( $get_current_transaction_id->err ){ - die "ERROR! return code: ". $get_current_transaction_id->err . " error msg: " . $get_current_transaction_id->errstr . "\n"; + $self->log->fatal("ERROR! return code: %s error msg: %s", $get_current_transaction_id->err, $get_current_transaction_id->errstr); } my $row_ref = $get_current_transaction_id->fetchall_hashref('id'); - print "Current Log position stuff:\n"; - print Dumper $row_ref; + $self->log->debug("Current Log position stuff: %s", Dumper $row_ref); - print "How many keys *ARE* there... ". ( scalar ( keys %{$row_ref} ) ) ."\n"; + $self->log->debug( "How many keys *ARE* there... %s", scalar ( keys %{$row_ref} )); if( ( scalar ( keys %{$row_ref} ) ) == 0 || @@ -874,8 +857,8 @@ sub _current_log_position { my $id; foreach $id ( sort keys %{$row_ref} ){ - print "Id is: $id\n"; - print "Hash is: ". $row_ref->{ $id }->{'transactionid'} ."\n"; + $self->log->debug("Id is: %s", $id); + $self->log->debug("Hash is: %s", $row_ref->{ $id }->{'transactionid'}); return $row_ref->{ $id }->{'transactionid'}; } } # end _current_log_position() @@ -903,10 +886,9 @@ sub _set_remote_log_position { $sth->execute( $hostname, $transactionid, $group ); if ( $sth->err ){ - die "ERROR! return code: ". $sth->err . " error msg: " . $sth->errstr . "\n"; + $self->log->fatal("ERROR! return code: %s error msg: %s", $sth->err, $sth->errstr); } - return 0; } # end _set_remote_log_position() @@ -928,22 +910,21 @@ sub _get_remote_log_position { my( $self, $hostname, $group ) = @_; my $dbh = $self->dbh; - print "Hostname: |$hostname| | Group: |$group|\n"; + $self->log->debug("Hostname: |%s| | Group: |%s|", $hostname, $group); my $sql = "SELECT id, transactionid FROM servers_seen WHERE hostname=? AND `group`=? order by id desc limit 1;"; - print "Sql: $sql\n"; + $self->log->debug("Sql: %s", $sql); my $sth = $dbh->prepare($sql); $sth->execute($hostname, $group); if ( $sth->err ){ - die "ERROR! return code: ". $sth->err . " error msg: " . $sth->errstr . "\n"; + $self->log->fatal("ERROR! return code: %s error msg: %s", $sth->err, $sth->errstr); } my $row_ref = $sth->fetchall_hashref('id'); - print "Current Log position stuff:\n"; - print Dumper $row_ref; + $self->log->debug("Current Log position stuff: %s", Dumper $row_ref); - print "How many keys *ARE* there... ". ( scalar ( keys %{$row_ref} ) ) ."\n"; + $self->log->debug("How many keys *ARE* there... %s", scalar ( keys %{$row_ref} )); if( ( scalar ( keys %{$row_ref} ) ) == 0 || @@ -955,8 +936,8 @@ sub _get_remote_log_position { my $id; foreach $id ( sort keys %{$row_ref} ){ - print "Id is: $id\n"; - print "Hash is: ". $row_ref->{ $id }->{'transactionid'} ."\n"; + $self->log->debug("Id is: %s", $id); + $self->log->debug("Hash is: %s", $row_ref->{ $id }->{'transactionid'}); return $row_ref->{ $id }->{'transactionid'}; } @@ -973,12 +954,6 @@ sub get_files_changed_since { my $response = $self->send_request( %request ); -# print "---------------------\n"; -# print "Response from _get_files_changed_since:\n"; -# print "---------------------\n"; -# print Dumper $response; -# print "^^^^^^^^^^^^^^^^^^^^^\n"; - return $response; } #end get_files_changed_since() @@ -991,7 +966,7 @@ sub _get_files_changed_since { my $sth = undef; my $sql = undef; - print "Transaction status: ". $transaction_status ."\n"; + $self->log->info("Transaction status: %s", $transaction_status); if( $transaction_status eq "0" @@ -1001,7 +976,7 @@ sub _get_files_changed_since { # Transaction wasn't found for this # group, lets assume we need to give it everything - print "*** Transaction not found, should just send everything\n"; + $self->log->debug("*** Transaction not found, should just send everything"); $sql = "SELECT * FROM files WHERE syncgroup=?;"; $sth = $dbh->prepare($sql); @@ -1021,7 +996,7 @@ sub _get_files_changed_since { ." ) " ." ); "; - print "Sql: |$sql|\n"; + $self->log->debug("Sql: |%s|",$sql); $sth = $dbh->prepare($sql); @@ -1029,45 +1004,35 @@ sub _get_files_changed_since { } if ( $sth->err ){ - die "ERROR! return code: ". $sth->err . " error msg: " . $sth->errstr . "\n"; + $self->log->fatal("ERROR! return code: %s error msg: %s", $sth->err, $sth->errstr); } my $row_ref = $sth->fetchall_hashref('id'); - print "Files Found:\n"; - print Dumper $row_ref; + $self->log->info("Files Found:"); my @return_array = (); my %return_hash = (); my $x = 0; foreach my $id ( sort keys %{$row_ref} ){ - print "Id is: $id\n"; - print "Hash is: ". $row_ref->{ $id }->{'last_transaction'} ."\n"; + $self->log->debug("Id is: %s",$id); + $self->log->debug("Hash is: %s", $row_ref->{ $id }->{'last_transaction'}); my $temp_file_obj = FileSync::SyncDiff::File->new(dbref => $self); $temp_file_obj->from_hash( $row_ref->{ $id } ); + $self->log->info("File path: %s",$temp_file_obj->filename); + my %temp_file_hash = $temp_file_obj->to_hash(); - #push( @return_array, %temp_file_hash ); $return_hash{$x} = \%temp_file_hash; $x = $x + 1; } -# print "---------------------\n"; -# print "Files that have changed:\n"; -# print "---------------------\n"; -# #print Dumper \@return_array; -# print Dumper \%return_hash; -# print "^^^^^^^^^^^^^^^^^^^^^\n"; - - #return \@return_array; return \%return_hash; } # end _get_files_changed_since() -#no moose; __PACKAGE__->meta->make_immutable; -#__PACKAGE__->meta->make_immutable(inline_constructor => 0,); 1; diff --git a/FileSync/SyncDiff/File.pm b/FileSync/SyncDiff/File.pm index c8dc843..b78176a 100644 --- a/FileSync/SyncDiff/File.pm +++ b/FileSync/SyncDiff/File.pm @@ -39,6 +39,7 @@ use Moose; use FileSync::SyncDiff::Util; use FileSync::SyncDiff::DB; +use FileSync::SyncDiff::Log; # # Needed for the file scanning @@ -153,6 +154,15 @@ has 'deleted' => ( default => '0', ); +# Logger system +has 'log' => ( + is => 'rw', + isa => 'FileSync::SyncDiff::Log', + default => sub { + return FileSync::SyncDiff::Log->new(); + } +); + # # End Local Variables # @@ -171,19 +181,15 @@ use overload 'eq' => \&_overload_comparison, fallback => 1; sub _overload_comparison { my( $left, $right, $options ) = @_; -# print "*** Left:\n"; -# print Dumper $left; -# -# print "*** Right:\n"; -# print Dumper $right; - for my $attr ( $left->meta->get_all_attributes ) { - print "COMPARISON: ". $attr->name ."\n"; + # TODO: Required attributes should be labeled + next if ( $attr->type_constraint->name eq 'FileSync::SyncDiff::Log' ); + $left->log->debug("COMPARISON: %s", $attr->name); if( $attr->name eq "checksum" ){ - print "State of options: ". ( defined $options ) ."\n"; + $left->log->debug("State of options: %s", ( defined $options ? 1 : 0 )); if( ref $options eq ref {} ){ - print "State of options->{checksum}: ". ( ! defined $options->{checksum} ) ."\n"; + $left->log->debug("State of options->{checksum}: %s", ( ! defined $options->{checksum} ) ); } } @@ -212,10 +218,8 @@ sub _overload_comparison { next; } -# print Dumper $attr->name; - - print "Left value: ". $left->_get_file_attr_value( $attr->name ) ."\n"; - print "Right value: ". $right->_get_file_attr_value( $attr->name ) ."\n"; + $left->log->debug("Left value: %s", $left->_get_file_attr_value( $attr->name )); + $left->log->debug("Right value: %s", $right->_get_file_attr_value( $attr->name )); if( "". $left->_get_file_attr_value( $attr->name ) ."" @@ -284,9 +288,6 @@ sub get_file { $filetype = 'dir'; } -# print "Username: \n"; -# print Dumper $username; - my ( $filename, $path, $suffix ) = fileparse($file); $self->filename( $filename ); $self->path( $path ); @@ -309,13 +310,11 @@ sub get_file { sub checksum_file { my( $self ) = @_; - #return "dummy_value"; - if( $self->filetype ne "file" ){ return; } - print "Checksumming: |". $self->filepath."|\n"; + $self->log->debug("Checksumming: | %s |", $self->filepath); open(FILE, $self->filepath ) or die "ERROR. $_ could not be opened: $!"; # This is critical to prevent ulcers and create correct SHA256 checksums binmode(FILE); @@ -327,7 +326,7 @@ sub checksum_file { my $return = "sha256:". $checksum; - print "Checksum: returning |$return|\n"; + $self->log->debug("Checksum: returning | %s |", $return); $self->checksum( $return ); } # end checksum_file() @@ -344,15 +343,7 @@ sub filepath { sub parse_dbrow { my( $self, $db_row ) = @_; -# print "~~~ File->parse_dbrow()\n"; -# print Dumper \$db_row; -# print "^^^^^^^^^^^^^^^^^^^^^^^\n"; - foreach my $row ( sort keys %{ $db_row } ){ -# print $db_row->{$row}->{last_transaction} ."\n"; -# for my $attr ( $self->meta->get_all_attributes ) { -# print "**** Column ". $attr->name ." - ". $db_row->{$row}->{$attr->name} ."\n"; -# } $self->from_hash( $db_row->{$row} ); } } # end parse_dbrow() @@ -360,10 +351,6 @@ sub parse_dbrow { sub to_hash { my( $self ) = @_; -# print "File::to_hash()\n"; -# print Dumper $self; -# print "^^^^^^^^^^^^^^^\n"; - my %file_hash = ( path => $self->path, filename => $self->filename, @@ -421,7 +408,6 @@ sub from_hash { $self->deleted( $file_hash->{deleted} ) if( defined $file_hash->{deleted} ); } # end from_hash() -#no moose; __PACKAGE__->meta->make_immutable; 1; diff --git a/FileSync/SyncDiff/Forwarder.pm b/FileSync/SyncDiff/Forwarder.pm new file mode 100644 index 0000000..18991b4 --- /dev/null +++ b/FileSync/SyncDiff/Forwarder.pm @@ -0,0 +1,266 @@ +#!/usr/bin/perl + +########################################################################### +# Copyright (C) 2014 John 'Warthog9' Hawley # +# jhawley@redhat.com # +# warthog9@eaglescrag.net # +# # +# This file is originally part of SyncDiff(erent). # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +# You should have received a copy of the GNU Lesser General Public # +# License along with this library; if not, write to the: # +# Free Software Foundation, Inc. # +# 51 Franklin Street # +# Fifth Floor # +# Boston, MA 02110-1301 # +# USA # +# # +# Or, see . # +########################################################################### + +package FileSync::SyncDiff::Forwarder; +$FileSync::SyncDiff::Forwarder::VERSION = '0.01'; + +use Moose; + +extends 'FileSync::SyncDiff::Forkable'; + +use FileSync::SyncDiff::Log; + +# +# Other Includes +# +use IO::Socket; +use IO::Socket::INET; +use IO::Socket::Forwarder qw(forward_sockets); +use Net::Domain qw(domainname); +use JSON::XS qw(encode_json); + +# +# Debugging +# +use Data::Dumper; + +# +# moose variables +# +has 'dbref' => ( + is => 'rw', + isa => 'Object', + required => 1, +); + +has 'client' => ( + is => 'rw', + isa => 'HashRef', + required => 1, +); + +has 'server' => ( + is => 'rw', + isa => 'HashRef', + required => 1, + default => sub { + { + port => '7070', + host => inet_ntoa(inet_aton(domainname())), + proto => 'tcp', + } + }, +); + +has 'middleware' => ( + is => 'rw', + isa => 'HashRef', + init_arg => undef, + lazy => 1, + default => sub { + { + port => '7069', + host => inet_ntoa(inet_aton(domainname())), + proto => 'tcp', + } + }, +); + +has 'response' => ( + is => 'ro', + isa => 'HashRef', + init_arg => undef, + lazy => 1, + writer => '_response', + default => sub { {} }, +); + +has 'log' => ( + is => 'rw', + isa => 'FileSync::SyncDiff::Log', + default => sub { + return FileSync::SyncDiff::Log->new(); + } +); + +# +# constants +# +use constant { + FIRST_PUBLIC_PORT => 1025, + MAX_PORT_NUMBER => 65535, + TRY_PORT_LIMIT => 10000, +}; + +#---------------------------------------------------------------------- +#** @method public run ($self) +# @brief Run forwarding beetween client and server +# @return HashRef, Forwarding response +# code - response code +# content_type - type of content +# content - content in appropriate format +#* +sub run { + my( $self ) = @_; + my $response = {}; + if ( !$self->dbref->is_exists_connection($self->client) ) { + if ( my $port = $self->_get_random_port() ) { + $self->middleware->{port} = $port; + my $content = encode_json({ + host => $self->middleware->{host}, + port => $port, + }); + $response = { + code => 200, + content_type => 'application/json', + content => $content, + }; + + $self->fork(); + } + } + else { + $response = { + code => 500, + content_type => 'application/json', + }; + } + $self->_response($response); + + return $self->response; +} # end run() + +# +# Need to override this from Forkable +# + +override 'run_child' => sub { + my( $self ) = @_; + + # Single client process - single forward connection + $self->_forward(); +}; # end run_child() + +#---------------------------------------------------------------------- +#** @method private _get_random_port ($self) +# @brief Return a random port from FIRST_PUBLIC_PORT to MAX_PORT_NUMBER +# @return scalar port +#* +sub _get_random_port { + my ( $self ) = @_; + + my $port = undef; + my $socket = undef; + my $limit = 0; + do { + $port = int( rand( MAX_PORT_NUMBER - FIRST_PUBLIC_PORT ) + FIRST_PUBLIC_PORT ); + $socket = eval { + IO::Socket::INET->new( + LocalPort => $port, + Proto => 'tcp', + Timeout => 1 + ); + }; + ++$limit; + } until( $socket || $limit > TRY_PORT_LIMIT ); + + close($socket) if ( defined $socket ); + + return $port; +} + +#---------------------------------------------------------------------- +#** @method private _forward($self) +# @brief Create a forward connection beetween server and client +# @return scalar, True value in success +#* +sub _forward { + my ( $self ) = @_; + + my $listener = eval { + IO::Socket::INET->new( + LocalPort => $self->middleware->{port}, + Proto => $self->middleware->{proto}, + Listen => 2, + ReuseAddr => 1 + ); + }; + if ( !$listener ) { + $self->log->fatal("Could not create socket %s", $!); + } + + $self->middleware->{socket} = $listener; + $self->client->{port} = $self->middleware->{port}; + $self->dbref->new_connection($self->client); + + my $client = $listener->accept(); + my $server = eval { + IO::Socket::INET->new( + PeerAddr => $self->server->{host}, + PeerPort => $self->server->{port}, + Proto => $self->server->{proto}, + ); + }; + if( !$server ){ + $self->log->fatal("Could not connect to syncdiff server: %s", $!); + } + + $self->server->{socket} = $server; + $server->autoflush(1); + forward_sockets($client, $server); + + $self->_clean_up(); + + return 1; +} # end _forward + +#---------------------------------------------------------------------- +#** @method private _clean_up ($self) +# @brief Clean forward connection from DB +# @return scalar True value in success +#* +sub _clean_up { + my ( $self ) = @_; + + if ( defined $self->middleware->{socket} ){ + close($self->middleware->{socket}); + } + if ( defined $self->server->{socket} ){ + close($self->server->{socket}); + } + + $self->dbref->clean_connections($self->client); + + return 1; +} # end _clean_up + +__PACKAGE__->meta->make_immutable; + +1; \ No newline at end of file diff --git a/FileSync/SyncDiff/Log.pm b/FileSync/SyncDiff/Log.pm new file mode 100644 index 0000000..614f98a --- /dev/null +++ b/FileSync/SyncDiff/Log.pm @@ -0,0 +1,233 @@ +#!/usr/bin/perl + +########################################################################### +# Copyright (C) 2014 John 'Warthog9' Hawley # +# jhawley@redhat.com # +# warthog9@eaglescrag.net # +# # +# This file is originally part of SyncDiff(erent). # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +# You should have received a copy of the GNU Lesser General Public # +# License along with this library; if not, write to the: # +# Free Software Foundation, Inc. # +# 51 Franklin Street # +# Fifth Floor # +# Boston, MA 02110-1301 # +# USA # +# # +# Or, see . # +########################################################################### + +package FileSync::SyncDiff::Log; +$FileSync::SyncDiff::Log::VERSION = '0.01'; + +use Moose; +use Log::Log4perl qw(get_logger :levels); +use Log::Log4perl::Appender; +use Log::Log4perl::Layout::PatternLayout; + +use Data::Dumper; + +use constant { + LOG_LEVEL_MAP => { + $INFO => { + appender => Log::Log4perl::Appender->new( + "Log::Log4perl::Appender::Screen", + stderr => 1, + utf8 => 1, + ), + format => qq{%d [%p] %m{chomp} %n} + }, + $DEBUG => { + appender => Log::Log4perl::Appender->new( + "Log::Log4perl::Appender::File", + filename => "/var/log/syncdiff.log", + mode => "append", + utf8 => 1, + ), + format => qq{%d [%p] %m{chomp} %n} + }, + $WARN => { + appender => Log::Log4perl::Appender->new( + "Log::Log4perl::Appender::File", + filename => "/var/log/syncdiff.log", + mode => "append", + utf8 => 1, + ), + format => qq{%d [%p] %m{chomp} %T %n} + }, + $ERROR => { + appender => Log::Log4perl::Appender->new( + "Log::Log4perl::Appender::File", + filename => "/var/log/syncdiff.log", + mode => "append", + utf8 => 1, + ), + format => qq{%d [%p] %m{chomp} %T %n} + }, + $FATAL => { + appender => Log::Log4perl::Appender->new( + "Log::Log4perl::Appender::File", + filename => "/var/log/syncdiff.log", + mode => "append", + utf8 => 1, + ), + format => qq{%d [%p] %m{chomp} %T %n} + }, + }, +}; + +has 'config' => ( + is => 'rw', + isa => 'FileSync::SyncDiff::Config', + required => 0, +); + +has '_logger' => ( + is => 'ro', + init_arg => undef, + isa => 'HashRef', + builder => '_build_logger' +); + +sub _build_logger { + my ($self) = @_; + + my $log_info = {}; + for my $level ( ($INFO, $DEBUG, $WARN, $ERROR, $FATAL) ) { + my $format = &LOG_LEVEL_MAP->{$level}{format}; + my $appender = &LOG_LEVEL_MAP->{$level}{appender}; + my $layout = Log::Log4perl::Layout::PatternLayout->new( $format ); + + $appender->layout($layout); + $log_info->{$level}{logger} = get_logger( __PACKAGE__ . $level ); + + ($log_info->{$level}{logger})->additivity(0); + ($log_info->{$level}{logger})->add_appender($appender); + ($log_info->{$level}{logger})->level($level); + } + + return $log_info; +} + +sub _log { + my ($self, $level) = @_; + + return $self->_logger->{$level}{logger}; +} + +#---------------------------------------------------------------------- +#** @method private _compose_msg ($self, $format, @values) +# @brief Compose message from format and values. +# Messages should be in sprintf format or +# as a simple text +# @param $format - sprintf format message +# @param @values - values for formatting message +# @return scalar Message +#* +sub _compose_msg { + my ( $self, $format, @values ) = @_; + my $msg = ''; + if ( scalar @values > 0 ) { + $msg = sprintf( $format, map { defined $_ ? $_ : '' }@values ); + } + else { + $msg = $format; + } + + return $msg; +} + +#---------------------------------------------------------------------- +#** @method public debug ($self, $format, @values) +# @brief Print message with DEBUG level. +# @param $format - sprintf format message +# @param @values - values for formatting message +# @return scalar Message +#* +sub debug { + my ( $self, $format, @values ) = @_; + my $msg = $self->_compose_msg($format, @values); + + $self->_log($DEBUG)->debug($msg); + + return $msg; +} + +#---------------------------------------------------------------------- +#** @method public info ($self, $format, @values) +# @brief Print message with INFO level. +# @param $format - sprintf format message +# @param @values - values for formatting message +# @return scalar Message +#* +sub info { + my ( $self, $format, @values ) = @_; + my $msg = $self->_compose_msg($format, @values); + + $self->_log($INFO)->info($msg); + + return $msg; +} + +#---------------------------------------------------------------------- +#** @method public warn ($self, $format, @values) +# @brief Print message with WARN level. +# @param $format - sprintf format message +# @param @values - values for formatting message +# @return scalar Message +#* +sub warn { + my ( $self, $format, @values ) = @_; + my $msg = $self->_compose_msg($format, @values); + + $self->_log($WARN)->logcarp($msg); + + return $msg; +} + +#---------------------------------------------------------------------- +#** @method public error ($self, $format, @values) +# @brief Print message with ERROR level. +# @param $format - sprintf format message +# @param @values - values for formatting message +# @return scalar Message +#* +sub error { + my ( $self, $format, @values ) = @_; + my $msg = $self->_compose_msg($format, @values); + + $self->_log($ERROR)->error_warn($msg); + + return $msg; +} + +#---------------------------------------------------------------------- +#** @method public fatal ($self, $format, @values) +# @brief Print message with FATAL level. +# @param $format - sprintf format message +# @param @values - values for formatting message +# @return scalar Message +#* +sub fatal { + my ( $self, $format, @values ) = @_; + my $msg = $self->_compose_msg($format, @values); + + $self->_log($FATAL)->logcroak($msg); + + return $msg; +} + +__PACKAGE__->meta->make_immutable; + +1; diff --git a/FileSync/SyncDiff/ParseCfg.pm b/FileSync/SyncDiff/ParseCfg.pm index 7b7574a..77c48da 100644 --- a/FileSync/SyncDiff/ParseCfg.pm +++ b/FileSync/SyncDiff/ParseCfg.pm @@ -7,6 +7,15 @@ # ANY CHANGE MADE HERE WILL BE LOST ! # #################################################################### +package FileSync::SyncDiff::ParseCfg; +use vars qw ( @ISA ); +use strict; + +@ISA= qw ( Parse::Yapp::Driver ); +use Parse::Yapp::Driver; + +#line 1 "syncdiff_cfg_parser.y" + # start of the code section ########################################################################### # Copyright (C) 2014 John 'Warthog9' Hawley # @@ -36,19 +45,12 @@ # Or, see . # ########################################################################### -package FileSync::SyncDiff::ParseCfg; -use vars qw ( @ISA ); -use strict; - -@ISA= qw ( Parse::Yapp::Driver ); -use Parse::Yapp::Driver; - -#line 1 "psync_cfg_parser.y" - # start of the code section - use Data::Dumper; use Sys::Hostname; use File::FnMatch qw(:fnmatch); # import everything +use URI; + +use FileSync::SyncDiff::Log; my $autonum = 0; @@ -64,6 +66,8 @@ my $ignore_uid = 0; my $ignore_gid = 0; my $ignore_mod = 0; +my $log = FileSync::SyncDiff::Log->new(); + sub get_groups { return %groups; } @@ -78,9 +82,6 @@ sub get_ignores { sub new_group { my ($name) = @_; -# print "function: new_group()\n"; -# -# print "\tnew_group - name: ". $name ."\n"; if( $name eq "" ){ $name = "group_". $autonum; @@ -96,43 +97,52 @@ sub add_host { my ($hostname) = @_; my $local_hostname = hostname; -# print "function: add_host()\n"; -# -# print "\tadd_host: ". $hostname ."\n"; - $hostname = lc($hostname); $local_hostname = lc($local_hostname); if( $hostname eq $local_hostname){ - #print "\t~~ Found ourselves\n"; return; } - if( - $groups{$curgroup}->{'host'} == undef - || - $groups{$curgroup}->{'host'} eq "" - ){ - my @temparray = (); - $groups{$curgroup}->{'host'} = \@temparray; + if( $groups{$curgroup}->{'host'} ){ + $groups{$curgroup}->{'host'} = \(); + return; } -# print Dumper \@_; + my $uri = URI->new($hostname); + my ($host, $port) = (undef,undef); + my $proto = eval { $uri->scheme() }; + if ($@) { + $log->warn("Invalid protocol name"); + } - my @temparray = ( $hostname, ); + if ( $proto ) { + eval { + $host = $uri->host(); + $port = $uri->port(); + }; + if ($@) { + $log->warn("Invalid host format %s",$@); + return; + } + } + else { + $host = $hostname; + $port = 7070; + } -# print Dumper \@temparray; -# print Dumper $groups{$curgroup}->{'host'}; + my @uri_info = ( { + host => $host, + port => $port, + proto => $proto, + } ); - push( @{ $groups{$curgroup}->{'host'} }, $hostname); + push( @{ $groups{$curgroup}->{'host'} }, @uri_info); } # end add_host sub add_patt { my($flat, $pattern) = @_; -# print "function: add_patt()\n"; -# print "\tadd_patt - pattern: ". $pattern ."\n"; - if( $groups{$curgroup}->{'patterns'} == undef || @@ -147,10 +157,9 @@ sub add_patt { sub set_key { my( $key ) = @_; -# print "function: set_key()\n"; if( $groups{$curgroup}->{'key'} ne "" ){ - print "*** Multiple keys found for group '". $curgroup ."' - last one wins! You are warned. ***\n"; + $log->info("*** Multiple keys found for group %s - last one wins! You are warned. ***", $curgroup); } if( @@ -164,19 +173,16 @@ sub set_key { close (FILE); } - print "Key is: $key\n"; + $log->debug("Key is: %s",$key); if( length($key) < 32 ){ - print "*** WARNING ***\n"; - print "\tKey for group:". $curgroup ." is less than 32 charaters. Security is at risk\n"; - print "***************\n"; + $log->warn("Key for group: %s is less than 32 charaters. Security is at risk", $curgroup); } $groups{$curgroup}->{'key'} = $key; } # end set_key() sub set_auto { my ( $auto_resolve ) = @_; -# print "function: set_auto()\n"; $auto_resolve = lc( $auto_resolve ); @@ -215,28 +221,22 @@ sub set_auto { return; } - print "*** WARNING ***\n"; - print "\tUnknown auto resolution mechanism: ". $auto_resolve ."\n"; - print "\tIgnoring option\n"; - print "***************\n"; + $log->warn("Unknown auto resolution mechanism: %s", $auto_resolve); + $log->warn("Ignoring option"); } # end set_auto() sub set_bak_dir { my ($back_dir) = @_; -# print "function: set_bak_dir()\n"; $groups{$curgroup}->{'back_dir'} = $back_dir; } # end set_bak_dir(); sub set_bak_gen { my ($backup_generations) = @_; -# print "function: set_bak_gen()\n"; if( $backup_generations =~ /[^0-9]+/ ){ - print "*** WARNING ***\n"; - print "\tUnknown number of Backup Generations: ". $backup_generations ."\n"; - print "\tIgnoring option\n"; - print "***************\n"; + $log->warn("Unknown number of Backup Generations: %s", $backup_generations); + $log->warn("Ignoring option"); return; } @@ -244,38 +244,34 @@ sub set_bak_gen { } # end set_back_gen() sub check_group { -# print "function: check_group()\n"; - if( length( $groups{$curgroup}->{'key'} ) <= 0 ){ - die("Config error: groups must have a key.\n"); + $log->fatal("Config error: groups must have a key."); } } # end check_group() sub new_action { - print "function: new_action()\n"; + $log->debug("function: new_action()"); } # end new_action() sub add_action_pattern { - print "function: add_action_pattern()\n"; + $log->debug("function: add_action_pattern()"); } # end add_action_pattern sub add_action_exec { - print "function: add_action_exec()\n"; + $log->debug("function: add_action_exec()"); } # end add_action_exec() sub set_action_logfile { - print "function: set_action_logfile()\n"; + $log->debug("function: set_action_logfile()"); } # end set_action_logfile() sub set_action_dolocal { - print "function: set_action_dolocal()\n"; + $log->debug("function: set_action_dolocal()"); } # end set_action_dolocal sub new_prefix { my( $pname ) = @_; -# print "function: new_prefix: $pname\n"; - $curprefix = $pname; $prefixes{$pname} = ""; @@ -283,10 +279,9 @@ sub new_prefix { sub new_prefix_entry { my( $pattern, $path ) = @_; -# print "function: new_prefix_entry()\n"; if( $path !~ /^\// ){ - print "\t Prefix Path: '". $path ."' is not an absolute path.\n"; + $log->warn("Prefix Path: '%s' is not an absolute path", $path); } my $hostname = hostname; @@ -306,7 +301,6 @@ sub new_prefix_entry { sub new_ignore { my( $propname ) = @_; -# print "function: new_ignore()\n"; if( $propname eq "uid" ){ $ignore_uid = 1; @@ -315,23 +309,20 @@ sub new_ignore { } elsif( $propname eq "mod" ){ $ignore_mod = 1; } else { - print "\tInvalid ignore option: '". $propname ."' - IGNORING\n"; + $log->warn("Invalid ignore option: '%s' - IGNORING", $propname); } } # end new_ignore() sub on_cygwin_lowercase { my( $string ) = @_; -# print "function: on_cygwin_lowercase()\n"; lc( $string ); -# print "on_cygwin_loewrcase: ". $string ."\n"; - return $string; } # end on_cygwin_lowercase() sub disable_cygwin_lowercase_hack { - print "function: disable_cygwin_lowercase_hack()\n"; + $log->debug("function: disable_cygwin_lowercase_hack()"); } @@ -834,13 +825,13 @@ sub new { [#Rule 4 '@1-2', 0, sub -#line 316 "psync_cfg_parser.y" +#line 336 "syncdiff_cfg_parser.y" { new_prefix($_[2]); } ], [#Rule 5 'block', 6, sub -#line 318 "psync_cfg_parser.y" +#line 338 "syncdiff_cfg_parser.y" { } ], [#Rule 6 @@ -849,7 +840,7 @@ sub [#Rule 7 'block', 2, sub -#line 321 "psync_cfg_parser.y" +#line 341 "syncdiff_cfg_parser.y" { disable_cygwin_lowercase_hack(); } ], [#Rule 8 @@ -858,7 +849,7 @@ sub [#Rule 9 'ignore_list', 2, sub -#line 327 "psync_cfg_parser.y" +#line 347 "syncdiff_cfg_parser.y" { new_ignore($_[1]); } ], [#Rule 10 @@ -867,25 +858,25 @@ sub [#Rule 11 'prefix_list', 6, sub -#line 333 "psync_cfg_parser.y" +#line 353 "syncdiff_cfg_parser.y" { new_prefix_entry($_[3], on_cygwin_lowercase($_[5])); } ], [#Rule 12 'block_header', 1, sub -#line 338 "psync_cfg_parser.y" +#line 358 "syncdiff_cfg_parser.y" { new_group(0); } ], [#Rule 13 'block_header', 2, sub -#line 340 "psync_cfg_parser.y" +#line 360 "syncdiff_cfg_parser.y" { new_group($_[2]); } ], [#Rule 14 'block_body', 3, sub -#line 345 "psync_cfg_parser.y" +#line 365 "syncdiff_cfg_parser.y" { check_group(); } ], [#Rule 15 @@ -912,25 +903,25 @@ sub [#Rule 22 'stmt', 2, sub -#line 360 "psync_cfg_parser.y" +#line 380 "syncdiff_cfg_parser.y" { set_key($_[2]); } ], [#Rule 23 'stmt', 2, sub -#line 362 "psync_cfg_parser.y" +#line 382 "syncdiff_cfg_parser.y" { set_auto($_[2]); } ], [#Rule 24 'stmt', 2, sub -#line 364 "psync_cfg_parser.y" +#line 384 "syncdiff_cfg_parser.y" { set_bak_dir($_[2]); } ], [#Rule 25 'stmt', 2, sub -#line 366 "psync_cfg_parser.y" +#line 386 "syncdiff_cfg_parser.y" { set_bak_gen($_[2]); } ], [#Rule 26 @@ -939,13 +930,13 @@ sub [#Rule 27 'host_list', 2, sub -#line 372 "psync_cfg_parser.y" +#line 392 "syncdiff_cfg_parser.y" { add_host($_[2], $_[2], 0); } ], [#Rule 28 'host_list', 4, sub -#line 374 "psync_cfg_parser.y" +#line 394 "syncdiff_cfg_parser.y" { add_host($_[2], $_[4], 0); } ], [#Rule 29 @@ -957,13 +948,13 @@ sub [#Rule 31 'host_list_slaves', 2, sub -#line 381 "psync_cfg_parser.y" +#line 401 "syncdiff_cfg_parser.y" { add_host($_[2], $_[2], 1); } ], [#Rule 32 'host_list_slaves', 4, sub -#line 383 "psync_cfg_parser.y" +#line 403 "syncdiff_cfg_parser.y" { add_host($_[2], $_[4], 1); } ], [#Rule 33 @@ -972,7 +963,7 @@ sub [#Rule 34 'excl_list', 2, sub -#line 389 "psync_cfg_parser.y" +#line 409 "syncdiff_cfg_parser.y" { add_patt(0, on_cygwin_lowercase($_[2])); } ], [#Rule 35 @@ -981,7 +972,7 @@ sub [#Rule 36 'incl_list', 2, sub -#line 395 "psync_cfg_parser.y" +#line 415 "syncdiff_cfg_parser.y" { add_patt(1, on_cygwin_lowercase($_[2])); } ], [#Rule 37 @@ -990,13 +981,13 @@ sub [#Rule 38 'comp_list', 2, sub -#line 401 "psync_cfg_parser.y" +#line 421 "syncdiff_cfg_parser.y" { add_patt(2, on_cygwin_lowercase($_[2])); } ], [#Rule 39 '@2-1', 0, sub -#line 406 "psync_cfg_parser.y" +#line 426 "syncdiff_cfg_parser.y" { new_action(); } ], [#Rule 40 @@ -1017,13 +1008,13 @@ sub [#Rule 45 'action_stmt', 2, sub -#line 420 "psync_cfg_parser.y" +#line 440 "syncdiff_cfg_parser.y" { set_action_logfile($_[2]); } ], [#Rule 46 'action_stmt', 1, sub -#line 422 "psync_cfg_parser.y" +#line 442 "syncdiff_cfg_parser.y" { set_action_dolocal(); } ], [#Rule 47 @@ -1032,7 +1023,7 @@ sub [#Rule 48 'action_pattern_list', 2, sub -#line 428 "psync_cfg_parser.y" +#line 448 "syncdiff_cfg_parser.y" { add_action_pattern(on_cygwin_lowercase($_[2])); } ], [#Rule 49 @@ -1041,7 +1032,7 @@ sub [#Rule 50 'action_exec_list', 2, sub -#line 434 "psync_cfg_parser.y" +#line 454 "syncdiff_cfg_parser.y" { add_action_exec($_[2]); } ] ], @@ -1049,7 +1040,7 @@ sub bless($self,$class); } -#line 437 "psync_cfg_parser.y" +#line 457 "syncdiff_cfg_parser.y" diff --git a/FileSync/SyncDiff/Protocol/v1.pm b/FileSync/SyncDiff/Protocol/v1.pm index cd655dc..3b9874f 100644 --- a/FileSync/SyncDiff/Protocol/v1.pm +++ b/FileSync/SyncDiff/Protocol/v1.pm @@ -37,6 +37,7 @@ extends qw(FileSync::SyncDiff::SenderReceiver); # SyncDiff parts I need use FileSync::SyncDiff::File; +use FileSync::SyncDiff::Log; # # Other Includes @@ -100,7 +101,15 @@ has 'dbref' => ( has '+short_rev' => ( default => 'no', ); - + +# Logger system +has 'log' => ( + is => 'rw', + isa => 'FileSync::SyncDiff::Log', + default => sub { + return FileSync::SyncDiff::Log->new(); + } +); sub setup { my( $self ) = @_; @@ -111,12 +120,8 @@ sub setup { my $response = $self->send_request( %request ); -# print "Protocol V1 - Setup - Version request response:\n"; -# print Dumper $response; - if( $response ne "1.0" ){ # This is the only protocol version we support right now - print "We don't currently support protocol version: ". $response ." - bailing out\n"; - exit(1); + $self->log->fatal("We don't currently support protocol version: %s - bailing out", $response); } } # end setup() @@ -137,27 +142,26 @@ sub client_run { my( $self ) = @_; my $dbref = $self->dbref; - print "Client is now running with Protocol major version 1\n"; + $self->log->debug("Client is now running with Protocol major version 1"); my $remote_current_log_position = $self->getCurrentLogPosition(); my $remote_previous_log_position = $dbref->get_remote_log_position( $self->hostname, $self->group ); - print "Current log position |". $remote_current_log_position ."|\n"; - print "Previous log position |". $remote_previous_log_position ."|\n"; + $self->log->debug("Current log position | %s |", $remote_current_log_position); + $self->log->debug("Previous log position | %s |", $remote_previous_log_position); if( $remote_current_log_position ne $remote_previous_log_position ){ - print "Updates were found!\n"; + $self->log->info("Updates were found!"); my $file_updates = $self->get_updates_from_remote( $remote_previous_log_position); - print "Files changed array:\n"; - print Dumper $file_updates; - print "^^^^^^^^^^^^^^^^^^^^\n"; + $self->log->debug("Files changed array: %s", Dumper $file_updates); + $self->log->debug("^^^^^^^^^^^^^^^^^^^^"); - print "Going to save this out as: ". $self->hostname ." | ". $self->group ." | ". $remote_current_log_position ."\n"; + $self->log->debug("Going to save this out as: %s | %s | %s", $self->hostname, $self->group, $remote_current_log_position); $dbref->set_remote_log_position( $self->hostname, $self->group, $remote_current_log_position ); } else { - print "No updates found\n"; + $self->log->info("No updates found"); } } # end client_run() @@ -171,48 +175,48 @@ sub get_updates_from_remote { my $response = $self->send_request( %request ); - print "Files Changed Since $remote_previous_log_position:\n"; - print Dumper $response; + $self->log->debug("Files Changed Since %s", $remote_previous_log_position); + $self->log->debug(Dumper $response); my $x = 0; my $num_keys = keys %{$response}; - print "--------------------------------------------------------------\n"; - print "*** STARTING FILE ***\n"; - printf("*** %3d/%3d ***\n", $x, $num_keys); - print "--------------------------------------------------------------\n"; + $self->log->debug("--------------------------------------------------------------"); + $self->log->debug("*** STARTING FILE ***"); + $self->log->debug("*** %3d/%3d ***", $x, $num_keys); + $self->log->debug("--------------------------------------------------------------"); $x = $x + 1; foreach my $id ( sort { $a <=> $b } keys %{$response} ){ - print "Id is: $id\n"; + $self->log->debug("Id is: %s",$id); my $temp_file = FileSync::SyncDiff::File->new(dbref => $self->dbref ); $temp_file->from_hash( $response->{$id} ); - print "Before fork:\n"; - print Dumper $temp_file; + $self->log->debug("Before fork:"); + $self->log->debug(Dumper $temp_file); my $pid = 0; if( $temp_file->filetype ne "file" ){ - print "--------------------------------------------------------------\n"; - print "*** NEXT FILE ***\n"; - printf("*** %3d/%3d ***\n", $x, $num_keys); - print "--------------------------------------------------------------\n"; + $self->log->debug("--------------------------------------------------------------"); + $self->log->debug("*** NEXT FILE ***"); + $self->log->debug("*** %3d/%3d ***", $x, $num_keys); + $self->log->debug("--------------------------------------------------------------"); $x = $x + 1; next; } if( ( $pid = fork() ) == 0 ){ # child process - print "V1: About to chroot to - |". $self->groupbase ."|\n"; + $self->log->debug("V1: About to chroot to - | %s |", $self->groupbase); - print "Group base: ". $self->groupbase ."\n"; + $self->log->debug("Group base: %s", $self->groupbase); chroot( $self->groupbase ); chdir( "/" ); - print "chrooted\n"; + $self->log->debug("chrooted"); # Ok for simplicities sake, and to # get past the file transfer section @@ -234,9 +238,6 @@ sub get_updates_from_remote { # good marketing material on second # thought.... - #if( -e $temp_file->filepath ){ - #file exists, we should compare things before we get too far here.... *BUT* I don't want to deal with that code quite yet - #} if( $temp_file->filetype eq "file" ){ $self->sync_file( $temp_file->path, $temp_file->filename, $temp_file->filepath, $temp_file->checksum ); } @@ -261,15 +262,14 @@ sub get_updates_from_remote { # from the code perspective my $kid = undef; - print "Going to wait for child:\n"; + $self->log->debug("Going to wait for child:"); do { $kid = waitpid($pid,0); - print "."; } while $kid > 0; my $child_ret_code = $?; - print "\n"; - print "Pid we were expecting: $pid | Pid that Died: $kid\n"; - print "Child Return Code: |$child_ret_code|\n"; + + $self->log->debug("Pid we were expecting: %s | Pid that Died: %s", $pid, $kid); + $self->log->debug("Child Return Code: |%s|",$child_ret_code); if( $temp_file->filetype eq "file" ){ @@ -277,21 +277,20 @@ sub get_updates_from_remote { $new_file_obj->get_file( $temp_file->filepath, $self->group, $self->groupbase ); $new_file_obj->checksum_file(); - print Dumper $new_file_obj; + $self->log->debug($new_file_obj); - print "External checking of checksum: \n"; - print "Passed checksum: ". $temp_file->checksum() ."\n"; - print "Saved checksum: ". $new_file_obj->checksum() ."\n"; + $self->log->debug("External checking of checksum:"); + $self->log->debug("Passed checksum: %s", $temp_file->checksum()); + $self->log->debug("Saved checksum: %s", $new_file_obj->checksum()); if( $new_file_obj->checksum() ne $temp_file->checksum ){ - print "Checksums don't match - ERGH!\n"; - exit(); + $self->log->fatal("Checksums don't match - ERGH!"); } #exit(); } - print "--------------------------------------------------------------\n"; - print "*** NEXT FILE ***\n"; - printf("*** %3d/%3d ***\n", $x, $num_keys); - print "--------------------------------------------------------------\n"; + $self->log->debug("--------------------------------------------------------------"); + $self->log->debug("*** NEXT FILE ***"); + $self->log->debug("*** %3d/%3d ***", $x, $num_keys); + $self->log->debug("--------------------------------------------------------------"); $x = $x + 1; } #end foreach response I.E. files @@ -304,7 +303,7 @@ sub sync_file { my $basis = undef; my $sig = undef; - print "Going to sync the file $filepath\n"; + $self->log->debug("Going to sync the file %s", $filepath); if( ! -d $path ){ # path hasn't been created yet @@ -313,20 +312,11 @@ sub sync_file { # do a mkdir based on the # path for the file - print "Making directory: ". $path ."\n"; + $self->log->debug("Making directory: %s", $path); make_path($path, { verbose => 1, } ); } my $dir = "/"; - #my $dir = "/group2/Kickstarter Deluxe Digital Album/"; - -# opendir(DIR, $dir) or die $!; -# -# while (my $file = readdir(DIR)) { -# print "$file\n"; -# } -# -# closedir(DIR); if( ! -e $filepath ){ open HANDLE, ">>$filepath" or die "touch $filepath: $!\n"; @@ -361,12 +351,6 @@ sub sync_file { 'path' => $path, ); - #print "----------------------\n"; - #print "Request:\n"; - #print "----------------------\n"; - #print Dumper \%request; - #print "^^^^^^^^^^^^^^^^^^^^^^\n"; - my $response_hash = $self->send_request( %request ); while( @@ -378,17 +362,16 @@ sub sync_file { || $response_hash->{path} ne $path ){ - print "*******************************************\n"; - print "This isn't the response we are expecting...\n"; - print "*******************************************\n"; - print Dumper $response_hash; - print "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; + $self->log->error("*******************************************"); + $self->log->error("This isn't the response we are expecting..."); + $self->log->error("*******************************************"); + $self->log->error(Dumper $response_hash); + $self->log->error( "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"); $response_hash = $self->plain_receiver( \%request ); } # response is acquired, lets actually lay some bits down - if( $response_hash->{filename} ne $filename ){ my $line = undef; my $socket = $self->socket; @@ -400,16 +383,16 @@ sub sync_file { } } # end while loop waiting on socket to return - print "EXTRA: $line\n"; + $self->log->debug("EXTRA: %s", $line); } - print Dumper $response_hash; + $self->log->debug(Dumper $response_hash); my $checksum_resp = sha256_base64($response_hash->{delta}); - print "v1 - response_hash calculated: ". $checksum_resp ."\n"; - print "v1 - reponse_hash what sent: ". $response_hash->{checksum} ."\n"; + $self->log->debug("v1 - response_hash calculated: %s", $checksum_resp); + $self->log->debug("v1 - reponse_hash what sent: %s", $response_hash->{checksum}); if( $checksum_resp ne $response_hash->{checksum}){ - print "Ok on the recieve buffer the checksums don't match WTF!\n"; + $self->log->error("Ok on the recieve buffer the checksums don't match WTF!"); sleep 30; } @@ -417,14 +400,13 @@ sub sync_file { my $response = decode_base64( $response64 ); - print "----------------------\n"; - print "Response:\n"; - print "----------------------\n"; - #print Dumper $response; - #print Dumper $response64; - print "Length: ". length( $response64 ) ."\n"; - print "Response: |". $response64 ."|\n"; - print "^^^^^^^^^^^^^^^^^^^^^^\n"; + $self->log->debug("----------------------"); + $self->log->debug("Response:"); + $self->log->debug("----------------------"); + + $self->log->debug("Length: %s", length( $response64 )); + $self->log->debug("Response: | %s |", $response64); + $self->log->debug("^^^^^^^^^^^^^^^^^^^^^^"); my $base = undef; @@ -442,37 +424,29 @@ sub sync_file { print $delta $response; seek $delta, 0, 0; - print "Delta filename: $delta_filename\n"; - -# my $new_path = $filepath; -# print "New Path: ". $new_path ."$$\n"; - #open( $new, ">", $new_path .".new" ) or die "STUFF!: $!"; - #open( $new, ">", $new_path ) or die "STUFF!: $!"; + $self->log->debug("Delta filename: %s", $delta_filename); ($new, $new_path) = tempfile( UNLINK => 1, ); binmode( $new, ':raw'); - print "Chocking around here I bet\n"; + $self->log->info("Chocking around here I bet"); File::Rdiff::patch_file $base, $delta, $new; - print "Yup\n"; + $self->log->info("Yup"); close $new; close $base; close $delta; - #unlink $delta_filename; my $new_file_obj = FileSync::SyncDiff::File->new(dbref => $self->dbref); $new_file_obj->get_file( $new_path, $self->group, $self->groupbase ); $new_file_obj->checksum_file(); - + $self->log->debug("Transfered file checksum: %s", $checksum); - print "Transfered file checksum: ". $checksum ."\n"; - - print "New File Checksum: ". $new_file_obj->checksum() ."\n"; + $self->log->debug("New File Checksum: %s", $new_file_obj->checksum()); if( $checksum ne $new_file_obj->checksum() ){ - print "*************** Checksums don't match\n"; + $self->log->error("*************** Checksums don't match"); return; } @@ -486,8 +460,8 @@ sub _get_files_changed_since { my $file_list = $dbref->get_files_changed_since( $self->group, $transactionid ); - print "V1: Files found changed since $transactionid\n"; - print Dumper $file_list; + $self->log->debug("V1: Files found changed since %s", $transactionid); + $self->log->debug(Dumper $file_list); return $file_list } # end get_files_changed_since() @@ -501,9 +475,6 @@ sub getCurrentLogPosition { my $response = $self->send_request( %request ); -# print "Found Log Position response\n"; -# print Dumper $response; - return $response; } # end getCurrentLogPosition() @@ -512,9 +483,6 @@ sub shareCurrentLogPosition { my $logPosition = $self->dbref->current_log_position(); -# print "Log position is:\n"; -# print Dumper $logPosition - } # end shareCurrentLogPosition() sub getVersion { @@ -531,49 +499,32 @@ sub getVersion { sub server_process_request { my( $self, $response ) = @_; -# print "Server version 1 got response: \n"; -# print Dumper $response; - if( ! exists( $response->{v1_operation} ) ){ return; } if( $response->{v1_operation} eq "getLogPosition" ){ -# print "DBref:\n"; -# print Dumper $self->dbref; my $logPosition = $self->dbref->current_log_position(); -# print "Found log position on Server:\n"; -# print Dumper $logPosition; return $logPosition; } if( $response->{v1_operation} eq "get_files_changed_since" ){ my $files_changed_response = $self->_get_files_changed_since( $response->{transactionid} ); -# print "---------------------\n"; -# print "Response from get_files_changed_since:\n"; -# print "---------------------\n"; -# print Dumper $files_changed_response; -# print "^^^^^^^^^^^^^^^^^^^^^\n"; - return $files_changed_response; } if( $response->{v1_operation} eq "syncfile" ){ - print "--------------------------------------------------------------\n"; - print "*** START SYNCFILE ***\n"; - print "--------------------------------------------------------------\n"; + $self->log->debug("--------------------------------------------------------------"); + $self->log->debug("*** START SYNCFILE ***"); + $self->log->debug("--------------------------------------------------------------"); my $pid = 0; if( ( $pid = fork() ) == 0 ){ # child process - chroot( $self->groupbase ); chdir( "/" ); - print "\n\n"; - print "chrooted\n"; - print "\n\n"; + $self->log->debug("chrooted"); - my $sync_ret = $self->_syncfile( $response->{path}, $response->{filename}, @@ -581,8 +532,8 @@ sub server_process_request { $response->{signature}, ); - print "~~ after syncfile response length: ". length( $sync_ret ) ."\n"; - print "Length of encoded delta buffer: ". length( $sync_ret->{delta} ) ."\n"; + $self->log->debug("~~ after syncfile response length: %s", length( $sync_ret )); + $self->log->debug("Length of encoded delta buffer: %s", length( $sync_ret->{delta} )); $self->plain_send( $sync_ret ); exit(0); @@ -592,9 +543,9 @@ sub server_process_request { do { $child = waitpid( $pid, 0); } while( $child > 0); - print "--------------------------------------------------------------\n"; - print "*** END SYNCFILE ***\n"; - print "--------------------------------------------------------------\n"; + $self->log->debug("--------------------------------------------------------------"); + $self->log->debug("*** END SYNCFILE ***"); + $self->log->debug("--------------------------------------------------------------"); } return undef; @@ -612,99 +563,56 @@ sub _syncfile { open( $new, "<", $filepath ) or die "new_buffer: $filepath - $!\n"; binmode( $new, ':raw'); -# print "--------------------------\n"; -# print "Sig64 encoded:\n"; -# print "--------------------------\n"; -# #my $str = $signature64; -# #substr($str, 20) = ""; -# #print "$str\n"; # prints "abc" -# print Dumper $signature64; -# print "^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - - my ($sig, $sig_filename) = tempfile(); binmode( $sig, ":raw" ); print $sig decode_base64( $signature64 );; seek $sig, 0, 0; - print Dumper $sig; + $self->log->debug(Dumper $sig); - print "Loading sig file\n"; + $self->log->info("Loading sig from %s file", $filename); $sig = loadsig_file $sig; ref $sig or exit 1; - print "Building hash table\n"; + $self->log->info("Building hash table"); $sig->build_hash_table; - print "Deltafying things\n"; + $self->log->info("Deltafying things"); File::Rdiff::delta_file $sig, $new, $delta; -# print "--------------------------\n"; -# print "Sig:\n"; -# print "--------------------------\n"; -# print Dumper $sig; -# print "--------------------------\n"; -# print "New:\n"; -# print "--------------------------\n"; -# print Dumper $new; -# print "--------------------------\n"; -# print "Delta:\n"; -# print "--------------------------\n"; -# print Dumper $delta; -# print "^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - my $delta_buffer = ""; seek $delta, 0, 0; my $bytes_read = 0; my $data; while ((my $n = read $delta, $data, 4096) != 0) { - #print "~~ $n bytes read\n"; $bytes_read = $bytes_read + $n; $delta_buffer .= $data; } - print "~ $bytes_read from Delta file\n"; - - print "Length of Delta Buffer: ". length( $delta_buffer ) ."\n"; + $self->log->debug("~ %s from Delta file",$bytes_read); + $self->log->debug("Length of Delta Buffer: %s", length( $delta_buffer )); - #close $sig; unlink $sig_filename; close $delta; unlink $delta_filename; close $new; my $delta_buffer_encoded = encode_base64( $delta_buffer ); - print "Length of Delta Buffer: ". length( $delta_buffer ) ."\n"; - print "Length of encoded delta buffer: ". length( $delta_buffer_encoded ) ."\n"; - - - - print "--------------------------\n"; - print "Delta Buffer encoded:\n"; - print "--------------------------\n"; -# my $str = $delta_buffer_encoded; -# substr($str, 20) = ""; -# print "$str\n"; # prints "abc" - #print Dumper $delta_buffer_encoded; - print "Total length: ". length( $delta_buffer_encoded ). "\n"; - print "^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; + $self->log->debug("Length of Delta Buffer: %s", length( $delta_buffer )); + $self->log->debug("Length of encoded delta buffer: %s", length( $delta_buffer_encoded )); -# print "--------------------------\n"; -# print "Delta encoded:\n"; -# print "--------------------------\n"; -# my $str = $delta_buffer_encoded; -# substr($str, 250) = ""; -# print "$str\n"; # prints "abc" -# print substr( $str, 0, 250 ) ."\n"; -# print Dumper $delta_buffer_encoded; -# print "^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; + $self->log->debug("--------------------------"); + $self->log->debug("Delta Buffer encoded:"); + $self->log->debug("--------------------------"); + $self->log->debug("Total length: %s", length( $delta_buffer_encoded )); + $self->log->debug("^^^^^^^^^^^^^^^^^^^^^^^^^^"); my %response = ( delta => $delta_buffer_encoded, @@ -715,14 +623,11 @@ sub _syncfile { filepath => $filepath, ); - print "Length of encoded delta buffer: ". length( $response{delta} ) ."\n"; + $self->log->debug("Length of encoded delta buffer: %s", length( $response{delta} )); - #return $delta_buffer_encoded; return \%response; } -#no moose; __PACKAGE__->meta->make_immutable; -#__PACKAGE__->meta->make_immutable(inline_constructor => 0,); 1; diff --git a/FileSync/SyncDiff/Scanner.pm b/FileSync/SyncDiff/Scanner.pm index 673e1e6..1bcf574 100644 --- a/FileSync/SyncDiff/Scanner.pm +++ b/FileSync/SyncDiff/Scanner.pm @@ -40,6 +40,7 @@ use FileSync::SyncDiff::Config; use FileSync::SyncDiff::File; use FileSync::SyncDiff::DB; use FileSync::SyncDiff::Util; +use FileSync::SyncDiff::Log; # # Needed for the file scanning @@ -94,6 +95,15 @@ has 'scan_count' => ( isa => 'Int', ); +# Logger system +has 'log' => ( + is => 'rw', + isa => 'FileSync::SyncDiff::Log', + default => sub { + return FileSync::SyncDiff::Log->new(); + } +); + # # End # @@ -110,52 +120,44 @@ sub fork_and_scan { override 'run_child' => sub { my( $self ) = @_; - print "About to chroot\n"; + $self->log->debug("About to chroot"); chroot( $self->groupbase ); chdir( "/" ); - print "chrooted\n"; + $self->log->debug("chrooted"); $self->scan(); }; # end run_child(); sub full_scan { - my ( $self, $config, $dbconnection ) = @_; + my ( $self, $config, $dbconnection, $log ) = @_; + + $log ||= $self->log; my %running_scanners = (); foreach my $group_name ( keys %{ $config->config->{groups} } ){ - print "run only scan group: ". $group_name ."\n"; + $log->debug("run only scan group: %s", $group_name); my $scanner = undef; foreach my $base_path ( @{ $config->config->{groups}->{$group_name}->{patterns} } ){ - #print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"; - #print "\tBase Path: ". $base_path ."\n"; - - #print "\ttruepath: ". $config->get_truepath( $base_path ) ."\n"; $scanner = FileSync::SyncDiff::Scanner->new( dbref => $dbconnection, group => $group_name, groupbase => $base_path ); $scanner->fork_and_scan(); - $running_scanners{$group_name}{$base_path} = $scanner; - - #print Dumper $scanner; - - #$scanner->create_transaction_id(); - #print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"; } # end foreach $base_path } # end foreach $group_name my $kid; foreach my $group_name (keys %running_scanners) { - print "Group Wait: $group_name\n"; + $log->debug("Group Wait: %s",$group_name); foreach my $base_path ( keys %{ $running_scanners{$group_name} } ){ - print "\tBase Path: $base_path\n"; + $log->debug("Base Path: %s", $base_path); my $pid = $running_scanners{$group_name}{$base_path}->pid(); - print "\t\tPid: $pid\n"; + $log->debug("Pid: %s",$pid); $kid = waitpid( $pid, 0); } } @@ -163,7 +165,6 @@ sub full_scan { $dbconnection->clean_stop(); do { - $kid = waitpid(-1,0); } while $kid > 0; @@ -176,11 +177,10 @@ sub scan { # # Totally an Invader Zim reference # - print "I'm SCANNING I'm SCANNING! - $$\n"; + $self->log->debug("I'm SCANNING I'm SCANNING! - %s", $$); $self->create_transaction_id(); - # # This checks the files that we *DO* have on the # system, we also need to check the files in the @@ -199,12 +199,9 @@ sub scan { my $filelist = $self->dbref->lookup_filelist( $self->group, $self->groupbase ); -# print "Object...\n"; -# print Dumper $filelist; - foreach my $fileobj ( @{ $filelist } ){ if( ! -e $fileobj->filepath ){ - print "File ". $fileobj->filepath ." was deleted\n"; + $self->log->debug("File %s was deleted", $fileobj->filepath ); $fileobj->last_transaction( $self->current_transaction_id ); $self->dbref->mark_deleted( $fileobj); } @@ -218,23 +215,17 @@ sub find_wanted { my $lookup_file = $self->dbref->lookup_file( $found_file, $self->group, $self->groupbase ); -# print "*** lookup file\n"; -# print Dumper $lookup_file; -# print "^^^^^^^^^^^^^^^\n"; - if( ! defined $self->scan_count ){ $self->scan_count(1); } - print "File Found: ". $self->scan_count ." - ". $found_file ."\n"; + $self->log->info("File Found: %s - %s", $self->scan_count, $found_file); $self->scan_count( $self->scan_count + 1 ); my $found_file_obj = FileSync::SyncDiff::File->new(dbref => $self->dbref); $found_file_obj->get_file( $found_file, $self->group, $self->groupbase ); - #print Dumper $found_file_obj; - # # Ok at this point we've found a file # and if $lookup_file is 0 then we've @@ -245,8 +236,7 @@ sub find_wanted { $found_file_obj->checksum_file(); $found_file_obj->last_transaction( $self->current_transaction_id ); - print "Found File Object:\n"; - #print Dumper $found_file_obj; + $self->log->debug("Found File Object:"); $self->dbref->add_file( $found_file_obj ); return; @@ -256,9 +246,9 @@ sub find_wanted { $lookup_file_obj->from_hash( $lookup_file ); - print "Comparison status: ". ( $lookup_file_obj == $found_file_obj ) ."\n"; + $self->log->debug("Comparison status: %s", ( $lookup_file_obj == $found_file_obj ? 1 : 0 )); if( $lookup_file_obj == $found_file_obj ){ - print "*** Objects are identical\n"; + $self->log->debug("*** Objects are identical"); return; } @@ -267,7 +257,7 @@ sub find_wanted { # # prep the object since it's obviously changed - $found_file_obj->checksum(); + $found_file_obj->checksum_file(); $found_file_obj->last_transaction( $self->current_transaction_id() ); # update the file in the database @@ -279,27 +269,14 @@ sub create_transaction_id { my $transaction_id = sha256_hex( $self->group ."-". $self->groupbase ."-". hostname() ."-". $$ ."-". time() ); - -## print "--------------------------\n"; -## print "FileSync::SyncDiff::Scanner->create_transaction_id() - transaction_id:\n"; -## print "--------------------------\n"; -## print Dumper $transaction_id; -## print "^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - -## print "\t*** About to create new transaction id in database!!!!\n"; - my $retval = $self->dbref->new_transaction_id( $self->group, $transaction_id ); - print "create_transaction_id retval: ". $retval ."\n"; - -## print "\t*** Done creating new transaction id in database\n"; + $self->log->debug("create_transaction_id retval: %s", $retval); $self->current_transaction_id( $transaction_id ); return; } # end create_transaction_id() -#no moose; -#__PACKAGE__->meta->make_immutable; __PACKAGE__->meta->make_immutable(inline_constructor => 0,); 1; diff --git a/FileSync/SyncDiff/SenderReceiver.pm b/FileSync/SyncDiff/SenderReceiver.pm index dc2052d..d9161b6 100644 --- a/FileSync/SyncDiff/SenderReceiver.pm +++ b/FileSync/SyncDiff/SenderReceiver.pm @@ -89,20 +89,23 @@ my $TIMEOUT = 300; sub recv_loop { my ( $self ) = @_; - my $sock = new IO::Socket::INET ( - LocalPort => '7070', - Proto => 'tcp', - Listen => 1, - Reuse => 1, - ); - die "Could not create socket: $!\n" unless $sock; + my $sock = eval { + IO::Socket::INET->new ( + LocalPort => '7070', + Proto => 'tcp', + Listen => 1, + Reuse => 1, + ); + }; + if ( !$sock ) { + $self->log->fatal("Could not create socket: %s", $!); + } while( my $new_sock = $sock->accept() ){ my $child; if( ( $child = fork() ) == 0 ){ # child process - #print Dumper $new_sock; $self->socket( $new_sock ); $self->json( new JSON::XS ); $self->process_request(); @@ -122,12 +125,12 @@ sub process_request { my $socket = $self->socket; while(1){ - print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"; - print "SR - process_request - Top of While loop\n"; - print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"; + $self->log->debug("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"); + $self->log->debug("SR - process_request - Top of While loop"); + $self->log->debug("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"); $line = $self->plain_receiver( {} ); - print "Debugging Recieved line\n"; + $self->log->debug("Debugging Recieved line"); $self->print_debug( $line ); if( ! defined $line ){ @@ -162,7 +165,7 @@ sub send_request { sub plain_send { my( $self, $request ) = @_; - print "plain_send - length: ". length( $request ) ."\n"; + $self->log->debug("plain_send - length: %s", length( $request )); if( $request eq "0" @@ -184,8 +187,8 @@ sub plain_send { ){ my $checksum = sha256_base64($request); - print "SR - Scalar recieving - checksum: $checksum\n"; - + $self->log->debug("SR - Scalar recieving - checksum: %s",$checksum); + my %temp_request = ( SCALAR => $request, checksum => $checksum, @@ -194,21 +197,14 @@ sub plain_send { } my ($package, $filename, $line) = caller; - print "Called from:\n\tPackage: $package\n\tFilename: $filename\n\tLine: $line\n"; $self->print_debug( $request ); my $json = encode_json( $request ); -# print "Plain Send Encoded JSON\n"; -# print Dumper $json; - - print "Length of json: ". length( $json ) ."\n"; + $self->log->debug("Length of json: %s", length( $json )); my $socket = $self->socket; -# print $socket "WTF BBQ\n"; -# $socket->flush(); - print $socket $json ."\n"; $socket->flush(); print $socket "--END--\n"; @@ -235,10 +231,9 @@ sub plain_receiver { eval { alarm($TIMEOUT); while( $read_line = <$socket> ){ - print "reading: $read_line\n"; + $self->log->debug("reading: %s",$read_line); if( defined $read_line ){ chomp( $read_line ); - #last if( $line ne "" ); if( $read_line eq "--END--\n" || @@ -268,54 +263,25 @@ sub plain_receiver { return 0; }; # end eval / timeout -# } # end if/else - -# print "Got Line back:\n"; -# print Dumper $line; - if( ! defined $line ){ return undef; } - print "SR - Checking line that we've gotten:\n"; + $self->log->debug("SR - Checking line that we've gotten:"); $self->print_debug( $line ); -# if( $was_json eq "1" ){ -# return $json_req; -# } - chomp( $line ); -# if( -# defined $request->{v1_operation} -# && -# $request->{v1_operation} eq 'syncfile' -# ){ -# print "Raw line from syncfile:\n"; -# print Dumper $line; -# } - if( $line eq "0" ){ return 0; } - print "Length of recieved line: ". length( $line ) ."\n"; + $self->log->debug("Length of recieved line: ", length( $line )); return $line if( $self->short_rev eq "yes" ); my $response = decode_json( $line ); - if( - defined $request->{v1_operation} - && - $request->{v1_operation} eq 'syncfile' - ){ -# print "Response from send:\n"; -# print Dumper $response; -# print "Ref: ". ref( $response ). "\n"; -# print "^^^^^^^^^^^^^^^^^^^^^\n"; - } - if( ref( $response ) eq "ARRAY" ){ return $response; } @@ -326,16 +292,12 @@ sub plain_receiver { if( defined $response->{SCALAR} ){ -# print Dumper $response; - my $checksum = sha256_base64($response->{SCALAR}); - -# print "SR - SCALAR recieved - Calculated: $checksum\n"; -# print "SR - SCALAR recieved - Transfered: ". $response->{checksum} ."\n"; + if( $checksum ne $response->{checksum} ){ - print "*** Checksum's don't match\n"; - print "*** Calculated: $checksum\n"; - print "*** Transfered: ". $response->{checksum} ."\n"; + $self->log->debug("*** Checksum's don't match"); + $self->log->debug("*** Calculated: %s",$checksum); + $self->log->debug("*** Transfered: %s", $response->{checksum}); } return $response->{SCALAR}; @@ -351,14 +313,12 @@ sub plain_receiver { sub print_debug { my( $self, $request ) = @_; -# print Dumper $request; - my $temp_req = $request; my $temp_delta = undef; my $temp_sig = undef; - print "SenderReceiver Debug:\n"; + $self->log->debug("SenderReceiver Debug:"); if( ref($request) eq 'HASH' ){ if( @@ -385,11 +345,11 @@ sub print_debug { && length( $request ) > 400 ){ - print substr( $request, 0, 400 ) ."\n"; + $self->log->debug(substr( $request, 0, 400 )); return; } - print Dumper $temp_req; + $self->log->debug(Dumper $temp_req); if( defined $temp_delta ){ $request->{delta} = $temp_delta; @@ -401,9 +361,6 @@ sub print_debug { return; } # end print_debug() - -#no moose; __PACKAGE__->meta->make_immutable; -#__PACKAGE__->meta->make_immutable(inline_constructor => 0,); 1; diff --git a/FileSync/SyncDiff/Server.pm b/FileSync/SyncDiff/Server.pm index c66e7db..8fe3f88 100644 --- a/FileSync/SyncDiff/Server.pm +++ b/FileSync/SyncDiff/Server.pm @@ -42,6 +42,7 @@ extends 'FileSync::SyncDiff::Forkable', 'FileSync::SyncDiff::SenderReceiver'; use FileSync::SyncDiff::File; use FileSync::SyncDiff::Util; use FileSync::SyncDiff::Protocol::v1; +use FileSync::SyncDiff::Log; # # Other Includes @@ -107,6 +108,15 @@ has 'remote_hostname' => ( required => 0, ); +# Logger system +has 'log' => ( + is => 'rw', + isa => 'FileSync::SyncDiff::Log', + default => sub { + my $self = shift; + return FileSync::SyncDiff::Log->new( config => $self->config ); + } +); # End variables @@ -130,8 +140,6 @@ override 'run_child' => sub { sub _process_request { my( $self, $recv_line ) = @_; -# print "Listener got: $recv_line\n"; - if( ! defined $recv_line ){ return undef; } @@ -142,13 +150,10 @@ sub _process_request { }; if( ! $json_success ){ - print "JSON was malformed, ignoring - ". $recv_line ."\n"; + $self->log->error("JSON was malformed, ignoring - %s", $recv_line); return undef; } -# print "Server - _process_request:\n"; -# print Dumper $response; - if( exists( $response->{operation} ) && @@ -162,14 +167,14 @@ sub _process_request { ){ my $auth_status = $self->_check_authentication( $response->{hostname}, $response->{group}, $response->{key} ); - print "_process_request Auth Status: $auth_status\n"; + $self->log->debug("_process_request Auth Status: %s", $auth_status); if( $auth_status == 0 ){ - print "Socket Die, Auth really failed\n"; + $self->log->debug("Socket Die, Auth really failed"); return "SOCKDIE"; } - print "Going to return successful Authentication\n"; + $self->log->info("Going to return successful Authentication"); return 0; } if( @@ -187,8 +192,8 @@ sub _process_request { && $response->{request_version} < 2 ){ - print "Primary protocol version 1 found\n"; - print Dumper $self->groupbase; + $self->log->debug("Primary protocol version 1 found"); + $self->log->debug("GroupBase: %s",$self->groupbase); $self->proto( FileSync::SyncDiff::Protocol::v1->new( socket => $self->socket, @@ -197,6 +202,7 @@ sub _process_request { group => $self->group, hostname => $self->remote_hostname, groupbase => $self->groupbase, + log => $self->log, ) ); return $self->proto->getVersion(); @@ -211,20 +217,17 @@ sub _check_authentication { my ( $self, $remote_hostname, $group, $key ) = @_; my $config = $self->config; - my $group_key = $config->{groups}->{ $group }->{key}; -# print "--------------------------\n"; -# print Dumper $config; -# print "^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; + my $group_key = $config->{groups}->{ $group }->{key}; - print "Got: Group |$group| Key |$key|\n"; - print "We have |$group_key|\n"; + $self->log->debug("Got: Group |%s| Key |%s|",$group,$key); + $self->log->debug("We have |%s|",$group_key); if( $key eq $group_key ){ - print "Auth succeeded\n"; + $self->log->info("Auth succeeded"); $self->group( $group ); $self->remote_hostname( $remote_hostname ); - print Dumper $config->{groups}->{ $group }; + $self->log->debug(Dumper $config->{groups}->{ $group }); $self->groupbase( $config->{groups}->{ $group }->{patterns}[0] ); return 1; } @@ -232,8 +235,6 @@ sub _check_authentication { return 0; } # end _check_authentication() -#no moose; __PACKAGE__->meta->make_immutable; -#__PACKAGE__->meta->make_immutable(inline_constructor => 0,); 1; diff --git a/psgi/forwarder.psgi b/psgi/forwarder.psgi new file mode 100755 index 0000000..fed657d --- /dev/null +++ b/psgi/forwarder.psgi @@ -0,0 +1,66 @@ +#!/usr/bin/perl + +########################################################################### +# Copyright (C) 2014 John 'Warthog9' Hawley # +# jhawley@redhat.com # +# warthog9@eaglescrag.net # +# # +# This file is originally part of SyncDiff(erent). # +# # +# This library is free software; you can redistribute it and/or # +# modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation; either # +# version 2.1 of the License, or (at your option) any later version. # +# # +# This library is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +# You should have received a copy of the GNU Lesser General Public # +# License along with this library; if not, write to the: # +# Free Software Foundation, Inc. # +# 51 Franklin Street # +# Fifth Floor # +# Boston, MA 02110-1301 # +# USA # +# # +# Or, see . # +########################################################################### + +use Plack::Request; +use Data::Dumper; +use FindBin qw($Bin); + +use FileSync::SyncDiff::Forwarder; +use FileSync::SyncDiff::Config; +use FileSync::SyncDiff::DB; + +my $app = sub { + my $env = shift; + + my $req = Plack::Request->new($env); + my ( $key, $dir, $host) = ( undef, undef, undef ); + + if ( $req->method eq 'POST' ) { + my $config = FileSync::SyncDiff::Config->new(); + $config->read_config( "$Bin/../syncdiff.cfg" ); + + my $db = FileSync::SyncDiff::DB->new( config => $config ); + $db->file( "$Bin/../psync.sqlite" ); + $db->connect_and_fork(); + + #CGI compatible + ( $key, $dir, $host ) = ( $req->param('key'), $req->param('include'), $req->param('host') ); + + my $forwarder = FileSync::SyncDiff::Forwarder->new( + dbref => $db, + client => { host => $host, auth_key => $key, syncbase => $dir }, + ); + my $response = $forwarder->run(); + + return [ $response->{code}, ['Content-Type', $response->{content_type}], [ $response->{content} ] ]; + } + + return [ 200, ['Content-Type', 'text/html'], [] ]; +}; \ No newline at end of file diff --git a/syncdiff b/syncdiff index 2787703..96d93a2 100755 --- a/syncdiff +++ b/syncdiff @@ -47,6 +47,7 @@ use FileSync::SyncDiff::DB; use FileSync::SyncDiff::File; use FileSync::SyncDiff::Server; use FileSync::SyncDiff::Client; +use FileSync::SyncDiff::Log; # Debugging includes use Data::Dumper; @@ -57,8 +58,8 @@ use Data::Dumper; $main::VERSION = '0.01'; sub display_help { - print "Right now, there is no helping anyone.\n"; - print "If you are running this you had better be John 'Warthog9' Hawley\n"; + print STDOUT "Right now, there is no helping anyone.\n"; + print STDOUT "If you are running this you had better be John 'Warthog9' Hawley\n"; } # @@ -80,12 +81,8 @@ GetOptions ( "h|?|help" => \$get_help, ); -print "Config File: ". $config_file ."\n"; -print "Run as Server? ". $run_as_server ."\n"; -print "Get Version? ". $get_version ."\n"; - if( $get_version ){ - print "Version: ". $main::VERSION ."\n"; + print STDOUT "Version: ". $main::VERSION ."\n"; exit(0); } @@ -102,50 +99,43 @@ if( ! defined $config_file ){ $config_file = "syncdiff.cfg"; } -#print "---------------------------\n"; -#print "config file:\n"; -#print "---------------------------\n"; -#print Dumper $config_file; -#print "^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"; - my $config = FileSync::SyncDiff::Config->new(); - $config->read_config( $config_file ); +my $log = FileSync::SyncDiff::Log->new( config => $config ); + +$log->info('Config File: %s', $config_file); +$log->info('Run as Server? %s',$run_as_server); +$log->info('Get Version? %s', $get_version); + # End opening and parsing the config file # $config now exists and can be passed around # as needed -my $dbconnection = FileSync::SyncDiff::DB->new( config => $config ); +my $dbconnection = FileSync::SyncDiff::DB->new( config => $config, log => $log ); $dbconnection->file( "psync.sqlite" ); $dbconnection->connect_and_fork(); if( $run_only_scan ){ - FileSync::SyncDiff::Scanner->full_scan( $config, $dbconnection ); + FileSync::SyncDiff::Scanner->full_scan( $config, $dbconnection, $log ); exit(0); } # end if( $run_only_scan ) if( $run_as_server ){ - my $server = FileSync::SyncDiff::Server->new( config => $config->config, dbref => $dbconnection ); + my $server = FileSync::SyncDiff::Server->new( config => $config->config, dbref => $dbconnection, log => $log ); $server->run(); exit(0); } -#print Dumper $config->config; - foreach $group_name ( keys %{ $config->config->{groups} } ){ - print "Main client loop - group: ". $group_name ."\n"; + $log->info("Main client loop - group: %s", $group_name); foreach $base ( @{ $config->config->{groups}->{$group_name}->{patterns} } ){ - my $client = FileSync::SyncDiff::Client->new( config_options => $config->get_group_config( $group_name ), group => $group_name, groupbase => $base, groupbase_path => $config->get_truepath( $base ), dbref => $dbconnection ); + my $client = FileSync::SyncDiff::Client->new( config_options => $config->get_group_config( $group_name ), group => $group_name, groupbase => $base, groupbase_path => $config->get_truepath( $base ), dbref => $dbconnection, log => $log ); $client->fork_and_connect(); } # end base foreach -} # end group_name foreach - -#my $config = FileSync::SyncDiff::Config->new(); -# -#$config->read_config('csync2.cfg'); +} # end group_name foreach \ No newline at end of file diff --git a/syncdiff_cfg_parser.y b/syncdiff_cfg_parser.y index 51edeb5..8c7265e 100644 --- a/syncdiff_cfg_parser.y +++ b/syncdiff_cfg_parser.y @@ -31,6 +31,9 @@ use Data::Dumper; use Sys::Hostname; use File::FnMatch qw(:fnmatch); # import everything +use URI; + +use FileSync::SyncDiff::Log; my $autonum = 0; @@ -46,6 +49,8 @@ my $ignore_uid = 0; my $ignore_gid = 0; my $ignore_mod = 0; +my $log = FileSync::SyncDiff::Log->new(); + sub get_groups { return %groups; } @@ -60,9 +65,6 @@ sub get_ignores { sub new_group { my ($name) = @_; -# print "function: new_group()\n"; -# -# print "\tnew_group - name: ". $name ."\n"; if( $name eq "" ){ $name = "group_". $autonum; @@ -78,43 +80,52 @@ sub add_host { my ($hostname) = @_; my $local_hostname = hostname; -# print "function: add_host()\n"; -# -# print "\tadd_host: ". $hostname ."\n"; - $hostname = lc($hostname); $local_hostname = lc($local_hostname); if( $hostname eq $local_hostname){ - #print "\t~~ Found ourselves\n"; return; } - if( - $groups{$curgroup}->{'host'} == undef - || - $groups{$curgroup}->{'host'} eq "" - ){ - my @temparray = (); - $groups{$curgroup}->{'host'} = \@temparray; + if( $groups{$curgroup}->{'host'} ){ + $groups{$curgroup}->{'host'} = \(); + return; } -# print Dumper \@_; + my $uri = URI->new($hostname); + my ($host, $port) = (undef,undef); + my $proto = eval { $uri->scheme() }; + if ($@) { + $log->warn("Invalid protocol name"); + } - my @temparray = ( $hostname, ); + if ( $proto ) { + eval { + $host = $uri->host(); + $port = $uri->port(); + }; + if ($@) { + $log->warn("Invalid host format %s",$@); + return; + } + } + else { + $host = $hostname; + $port = 7070; + } -# print Dumper \@temparray; -# print Dumper $groups{$curgroup}->{'host'}; + my @uri_info = ( { + host => $host, + port => $port, + proto => $proto, + } ); - push( @{ $groups{$curgroup}->{'host'} }, $hostname); + push( @{ $groups{$curgroup}->{'host'} }, @uri_info); } # end add_host sub add_patt { my($flat, $pattern) = @_; -# print "function: add_patt()\n"; -# print "\tadd_patt - pattern: ". $pattern ."\n"; - if( $groups{$curgroup}->{'patterns'} == undef || @@ -129,10 +140,9 @@ sub add_patt { sub set_key { my( $key ) = @_; -# print "function: set_key()\n"; if( $groups{$curgroup}->{'key'} ne "" ){ - print "*** Multiple keys found for group '". $curgroup ."' - last one wins! You are warned. ***\n"; + $log->info("*** Multiple keys found for group %s - last one wins! You are warned. ***", $curgroup); } if( @@ -146,19 +156,16 @@ sub set_key { close (FILE); } - print "Key is: $key\n"; + $log->debug("Key is: %s",$key); if( length($key) < 32 ){ - print "*** WARNING ***\n"; - print "\tKey for group:". $curgroup ." is less than 32 charaters. Security is at risk\n"; - print "***************\n"; + $log->warn("Key for group: %s is less than 32 charaters. Security is at risk", $curgroup); } $groups{$curgroup}->{'key'} = $key; } # end set_key() sub set_auto { my ( $auto_resolve ) = @_; -# print "function: set_auto()\n"; $auto_resolve = lc( $auto_resolve ); @@ -197,28 +204,22 @@ sub set_auto { return; } - print "*** WARNING ***\n"; - print "\tUnknown auto resolution mechanism: ". $auto_resolve ."\n"; - print "\tIgnoring option\n"; - print "***************\n"; + $log->warn("Unknown auto resolution mechanism: %s", $auto_resolve); + $log->warn("Ignoring option"); } # end set_auto() sub set_bak_dir { my ($back_dir) = @_; -# print "function: set_bak_dir()\n"; $groups{$curgroup}->{'back_dir'} = $back_dir; } # end set_bak_dir(); sub set_bak_gen { my ($backup_generations) = @_; -# print "function: set_bak_gen()\n"; if( $backup_generations =~ /[^0-9]+/ ){ - print "*** WARNING ***\n"; - print "\tUnknown number of Backup Generations: ". $backup_generations ."\n"; - print "\tIgnoring option\n"; - print "***************\n"; + $log->warn("Unknown number of Backup Generations: %s", $backup_generations); + $log->warn("Ignoring option"); return; } @@ -226,38 +227,34 @@ sub set_bak_gen { } # end set_back_gen() sub check_group { -# print "function: check_group()\n"; - if( length( $groups{$curgroup}->{'key'} ) <= 0 ){ - die("Config error: groups must have a key.\n"); + $log->fatal("Config error: groups must have a key."); } } # end check_group() sub new_action { - print "function: new_action()\n"; + $log->debug("function: new_action()"); } # end new_action() sub add_action_pattern { - print "function: add_action_pattern()\n"; + $log->debug("function: add_action_pattern()"); } # end add_action_pattern sub add_action_exec { - print "function: add_action_exec()\n"; + $log->debug("function: add_action_exec()"); } # end add_action_exec() sub set_action_logfile { - print "function: set_action_logfile()\n"; + $log->debug("function: set_action_logfile()"); } # end set_action_logfile() sub set_action_dolocal { - print "function: set_action_dolocal()\n"; + $log->debug("function: set_action_dolocal()"); } # end set_action_dolocal sub new_prefix { my( $pname ) = @_; -# print "function: new_prefix: $pname\n"; - $curprefix = $pname; $prefixes{$pname} = ""; @@ -265,10 +262,9 @@ sub new_prefix { sub new_prefix_entry { my( $pattern, $path ) = @_; -# print "function: new_prefix_entry()\n"; if( $path !~ /^\// ){ - print "\t Prefix Path: '". $path ."' is not an absolute path.\n"; + $log->warn("Prefix Path: '%s' is not an absolute path", $path); } my $hostname = hostname; @@ -288,7 +284,6 @@ sub new_prefix_entry { sub new_ignore { my( $propname ) = @_; -# print "function: new_ignore()\n"; if( $propname eq "uid" ){ $ignore_uid = 1; @@ -297,23 +292,20 @@ sub new_ignore { } elsif( $propname eq "mod" ){ $ignore_mod = 1; } else { - print "\tInvalid ignore option: '". $propname ."' - IGNORING\n"; + $log->warn("Invalid ignore option: '%s' - IGNORING", $propname); } } # end new_ignore() sub on_cygwin_lowercase { my( $string ) = @_; -# print "function: on_cygwin_lowercase()\n"; lc( $string ); -# print "on_cygwin_loewrcase: ". $string ."\n"; - return $string; } # end on_cygwin_lowercase() sub disable_cygwin_lowercase_hack { - print "function: disable_cygwin_lowercase_hack()\n"; + $log->debug("function: disable_cygwin_lowercase_hack()"); } %} # end of the code section