Merge pull request #198 from xtuml/188_enhance_logging

188 enhance logging

bin/.env

@@ -1 +1 @@
-MASL_VERSION=4.4.0
+MASL_VERSION=4.4.3-logging

bin/munin-ec2.sh

@@ -25,7 +25,7 @@
 NAME=myname                         # name to prepend for tagging the instance
 
 AMI_REGION=us-east-1                # Virginia
-AMI_ID=ami-01efc7833f2cc9a3e        # us-east AMI name PV benchmarking 3
+AMI_ID=ami-05cf4407f1a37ffb2        # us-east AMI name PV benchmarking 4
 SG_ID=sg-017c46e14320a0f5d          # us-east security group name "Remote Access"
 KEY_PAIR=pvlinuxtest                # keypair for logging in with ssh to us-east
 

deploy/.env

@@ -1 +1 @@
-MASL_VERSION=4.4.0
+MASL_VERSION=4.4.3-logging

deploy/config/benchmarking-config.json

@@ -13,15 +13,15 @@
   "JobDefinitionDirectory": "config/job_definitions",
   "DefaultJobExpiryDuration": "P99W",
   "DefaultStaleAuditEventDuration": "PT10M",
-  "DefaultBlockedAuditEventDuration": "PT55S",
+  "DefaultBlockedAuditEventDuration": "PT5S",
   "JobStoreAgeLimit": "PT1H",
   "InvariantStoreLoadRate": "PT2M",
-  "MaxIntraSequenceEventTimeoutPeriod": "PT5S",
+  "MaxIntraSequenceEventTimeoutPeriod": "PT1S",
   "WaitPeriodForAllJobsCompletedCheck": "P1D",
   "WaitPeriodForJobDeletion": "PT0S",
   "WaitPeriodForInvariantDeletion": "P1D",
   "TimeoutPeriodForRetreivingStoredInvariants": "PT10S",
-  "TimeoutPeriodForHangingJob": "PT30S",
+  "TimeoutPeriodForHangingJob": "PT2S",
   "MaxJobsPerWorker": "500",
   "MaxUnassignedJobs": "100000",
   "RetiredWorkerDeletionTime": "PT10M",

deploy/config/configure-kafka.sh

@@ -1,34 +1,5 @@
 #!/bin/bash
 
 # configure topics
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AsyncLogger_service0 --add-config retention.ms=5000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AsyncLogger_service1 --add-config retention.ms=5000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AsyncLogger_service2 --add-config retention.ms=5000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AsyncLogger_service3 --add-config retention.ms=5000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AsyncLogger_service4 --add-config retention.ms=5000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AsyncLogger_service5 --add-config retention.ms=5000
 /opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic JobManagement_service0 --add-config retention.ms=60000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic JobManagement_service1 --add-config retention.ms=60000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic JobManagement_service2 --add-config retention.ms=60000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic JobManagement_service3 --add-config retention.ms=60000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic JobManagement_service4 --add-config retention.ms=60000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic JobManagement_service5 --add-config retention.ms=60000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic JobManagement_service6 --add-config retention.ms=60000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AEOrdering_service0 --add-config retention.ms=300000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AEOrdering_service1 --add-config retention.ms=300000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AEOrdering_service2 --add-config retention.ms=300000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AEOrdering_service3 --add-config retention.ms=300000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AEOrdering_service4 --add-config retention.ms=300000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AEOrdering_service5 --add-config retention.ms=300000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AEOrdering_service6 --add-config retention.ms=300000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AEOrdering_service7 --add-config retention.ms=300000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AEOrdering_service8 --add-config retention.ms=300000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AEOrdering_service9 --add-config retention.ms=300000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AESequenceDC_service12 --add-config retention.ms=300000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic AESequenceDC_service31 --add-config retention.ms=300000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic IStore_service0 --add-config retention.ms=30000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic IStore_service1 --add-config retention.ms=30000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic IStore_service2 --add-config retention.ms=30000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic IStore_service3 --add-config retention.ms=30000
-/opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic IStore_service4 --add-config retention.ms=30000
 /opt/kafka/bin/kafka-configs.sh --alter --bootstrap-server kafka:9093 --topic BenchmarkingProbe_service0 --add-config retention.ms=6000000

deploy/config/log-config-pv-proc.properties

@@ -5,9 +5,6 @@
 # - Each logger has additivity FALSE so that the root logger does not
 #   repeat to the console.
 #
-# - Note that for now, each logger is individually configured, and it is
-#   not possible to inherit anything useful.
-#
 
 # Root logger logs to the console.
 log4cplus.rootLogger=FATAL, console
@@ -17,96 +14,79 @@ log4cplus.appender.console=log4cplus::ConsoleAppender
 log4cplus.appender.console.layout=log4cplus::PatternLayout
 log4cplus.appender.console.layout.ConversionPattern=%d{%FT%T}Z - %m%n
 
-# named loggers
-log4cplus.logger.Verifier=INFO, verifier
-log4cplus.additivity.Verifier=FALSE
-log4cplus.logger.Reception=INFO, reception
-log4cplus.additivity.Reception=FALSE
-log4cplus.logger.VerifierInstrumentation=INFO, verifierinstrumentation
-log4cplus.additivity.VerifierInstrumentation=FALSE
-
-log4cplus.logger.AEOrdering=DEBUG, aeordering
-log4cplus.additivity.AEOrdering=FALSE
-log4cplus.logger.AESequenceDC=DEBUG, aesequencedc
-log4cplus.additivity.AESequenceDC=FALSE
-log4cplus.logger.AEReception=DEBUG, aereception
-log4cplus.additivity.AEReception=FALSE
-log4cplus.logger.IStore=DEBUG, istore
-log4cplus.additivity.IStore=FALSE
-log4cplus.logger.JobManagement=DEBUG, jobmanagement
-log4cplus.additivity.JobManagement=FALSE
-log4cplus.logger.VerificationGateway=DEBUG, verificationgateway
-log4cplus.additivity.VerificationGateway=FALSE
-
-log4cplus.appender.verifier.File=/var/log/verifier/Verifier.log
-log4cplus.appender.verifier=log4cplus::RollingFileAppender
-log4cplus.appender.verifier.AsyncAppend=TRUE
-log4cplus.appender.verifier.MaxFileSize=50MB
-log4cplus.appender.verifier.MaxBackupIndex=5
-log4cplus.appender.verifier.layout=log4cplus::PatternLayout
-log4cplus.appender.verifier.layout.ConversionPattern=%d{%FT%T.%q}Z - %m%n
+# hierarchical loggers
+log4cplus.logger.pv=INFO, pv_appender
+log4cplus.additivity.pv=FALSE
 
-log4cplus.appender.reception.File=/var/log/reception/Reception.log
-log4cplus.appender.reception=log4cplus::RollingFileAppender
-log4cplus.appender.reception.AsyncAppend=TRUE
-log4cplus.appender.reception.MaxFileSize=50MB
-log4cplus.appender.reception.MaxBackupIndex=5
-log4cplus.appender.reception.layout=log4cplus::PatternLayout
-log4cplus.appender.reception.layout.ConversionPattern=%d{%FT%T.%q}Z - %m%n
+# domain loggers
+log4cplus.logger.pv.jm=INFO, pv_appender
+#log4cplus.logger.pv.jm=DEBUG, pv_appender, pv_debug_appender
+log4cplus.additivity.pv.jm=FALSE
+log4cplus.logger.pv.aeo=INFO, pv_appender
+#log4cplus.logger.pv.aeo=DEBUG, pv_appender, pv_debug_appender
+log4cplus.additivity.pv.aeo=FALSE
+log4cplus.logger.pv.sv=INFO, pv_appender
+#log4cplus.logger.pv.sv=DEBUG, pv_appender, pv_debug_appender
+log4cplus.additivity.pv.sv=FALSE
+log4cplus.logger.pv.istore=INFO, pv_appender
+#log4cplus.logger.pv.istore=DEBUG, pv_appender, pv_debug_appender
+log4cplus.additivity.pv.istore=FALSE
+log4cplus.logger.pv.vg=INFO, pv_appender
+#log4cplus.logger.pv.vg=DEBUG, pv_appender, pv_debug_appender
+log4cplus.additivity.pv.vg=FALSE
+log4cplus.logger.pv.fr=INFO, pv_appender
+#log4cplus.logger.pv.fr=DEBUG, pv_appender, pv_debug_appender
+log4cplus.additivity.pv.fr=FALSE
 
-log4cplus.appender.verifierinstrumentation.File=/var/log/verifier/VerifierInstrumentation.log
-log4cplus.appender.verifierinstrumentation=log4cplus::RollingFileAppender
-log4cplus.appender.verifierinstrumentation.AsyncAppend=TRUE
-log4cplus.appender.verifierinstrumentation.MaxFileSize=50MB
-log4cplus.appender.verifierinstrumentation.MaxBackupIndex=5
-log4cplus.appender.verifierinstrumentation.layout=log4cplus::PatternLayout
-log4cplus.appender.verifierinstrumentation.layout.ConversionPattern=%d{%FT%T.%q}Z - %m%n
+# audit event logger
+# Enable by setting to INFO.
+log4cplus.logger.pv.vg.audit=FATAL, pv_vg_audit_appender
+log4cplus.additivity.pv.vg.audit=FALSE
 
-log4cplus.appender.aeordering.File=/var/log/verifier/AEOrdering.log
-log4cplus.appender.aeordering=log4cplus::RollingFileAppender
-log4cplus.appender.aeordering.AsyncAppend=TRUE
-log4cplus.appender.aeordering.MaxFileSize=50MB
-log4cplus.appender.aeordering.MaxBackupIndex=5
-log4cplus.appender.aeordering.layout=log4cplus::PatternLayout
-log4cplus.appender.aeordering.layout.ConversionPattern=%d{%FT%T.%q}Z - %m%n
+# benchmarking logger
+# Enable by setting to INFO.
+log4cplus.logger.pv.benchmark=FATAL, pv_benchmark_appender
+log4cplus.additivity.pv.benchmark=FALSE
 
-log4cplus.appender.aesequencedc.File=/var/log/verifier/AESequenceDC.log
-log4cplus.appender.aesequencedc=log4cplus::RollingFileAppender
-log4cplus.appender.aesequencedc.AsyncAppend=TRUE
-log4cplus.appender.aesequencedc.MaxFileSize=50MB
-log4cplus.appender.aesequencedc.MaxBackupIndex=5
-log4cplus.appender.aesequencedc.layout=log4cplus::PatternLayout
-log4cplus.appender.aesequencedc.layout.ConversionPattern=%d{%FT%T.%q}Z - %m%n
+log4cplus.appender.pv_appender=log4cplus::RollingFileAppender
+log4cplus.appender.pv_appender.File=/var/log/protocol_verifier/pv.log
+log4cplus.appender.pv_appender.Threshold=INFO
+log4cplus.appender.pv_appender.AsyncAppend=TRUE
+log4cplus.appender.pv_appender.MaxFileSize=50MB
+log4cplus.appender.pv_appender.MaxBackupIndex=5
+log4cplus.appender.pv_appender.layout=log4cplus::PatternLayout
+log4cplus.appender.pv_appender.layout.ConversionPattern=%d{%FT%T.%q}Z - %m%n
 
-log4cplus.appender.aereception.File=/var/log/reception/AEReception.log
-log4cplus.appender.aereception=log4cplus::RollingFileAppender
-log4cplus.appender.aereception.AsyncAppend=TRUE
-log4cplus.appender.aereception.MaxFileSize=50MB
-log4cplus.appender.aereception.MaxBackupIndex=5
-log4cplus.appender.aereception.layout=log4cplus::PatternLayout
-log4cplus.appender.aereception.layout.ConversionPattern=%d{%FT%T.%q}Z - %m%n
+log4cplus.appender.pv_debug_appender=log4cplus::RollingFileAppender
+log4cplus.appender.pv_debug_appender.File=/var/log/protocol_verifier/pv_debug.log
+log4cplus.appender.pv_debug_appender.filters.1=log4cplus::spi::LogLevelRangeFilter
+log4cplus.appender.pv_debug_appender.filters.1.LogLevelMax=DEBUG
+log4cplus.appender.pv_debug_appender.filters.1.LogLevelMin=TRACE
+log4cplus.appender.pv_debug_appender.AsyncAppend=TRUE
+log4cplus.appender.pv_debug_appender.MaxFileSize=50MB
+log4cplus.appender.pv_debug_appender.MaxBackupIndex=5
+log4cplus.appender.pv_debug_appender.layout=log4cplus::PatternLayout
+log4cplus.appender.pv_debug_appender.layout.ConversionPattern=%d{%FT%T.%q}Z - %F:%L %m%n
 
-log4cplus.appender.istore.File=/var/log/verifier/IStore.log
-log4cplus.appender.istore=log4cplus::RollingFileAppender
-log4cplus.appender.istore.AsyncAppend=TRUE
-log4cplus.appender.istore.MaxFileSize=50MB
-log4cplus.appender.istore.MaxBackupIndex=5
-log4cplus.appender.istore.layout=log4cplus::PatternLayout
-log4cplus.appender.istore.layout.ConversionPattern=%d{%FT%T.%q}Z - %m%n
+log4cplus.appender.pv_vg_audit_appender=log4cplus::RollingFileAppender
+log4cplus.appender.pv_vg_audit_appender.File=/var/log/protocol_verifier/pv_vg_audit.log
+log4cplus.appender.pv_vg_audit_appender.AsyncAppend=TRUE
+log4cplus.appender.pv_vg_audit_appender.MaxFileSize=50MB
+log4cplus.appender.pv_vg_audit_appender.MaxBackupIndex=5
+log4cplus.appender.pv_vg_audit_appender.layout=log4cplus::PatternLayout
+log4cplus.appender.pv_vg_audit_appender.layout.ConversionPattern=%d{%FT%T.%q}Z - %m%n
 
-log4cplus.appender.jobmanagement.File=/var/log/verifier/JobManagement.log
-log4cplus.appender.jobmanagement=log4cplus::RollingFileAppender
-log4cplus.appender.jobmanagement.AsyncAppend=TRUE
-log4cplus.appender.jobmanagement.MaxFileSize=50MB
-log4cplus.appender.jobmanagement.MaxBackupIndex=5
-log4cplus.appender.jobmanagement.layout=log4cplus::PatternLayout
-log4cplus.appender.jobmanagement.layout.ConversionPattern=%d{%FT%T.%q}Z - %m%n
+log4cplus.appender.pv_benchmark_appender=log4cplus::RollingFileAppender
+log4cplus.appender.pv_benchmark_appender.File=/var/log/protocol_verifier/pv_benchmark.log
+log4cplus.appender.pv_benchmark_appender.AsyncAppend=TRUE
+log4cplus.appender.pv_benchmark_appender.MaxFileSize=50MB
+log4cplus.appender.pv_benchmark_appender.MaxBackupIndex=5
+log4cplus.appender.pv_benchmark_appender.layout=log4cplus::PatternLayout
+log4cplus.appender.pv_benchmark_appender.layout.ConversionPattern=%d{%FT%T.%q}Z - %m%n
 
-log4cplus.appender.verificationgateway.File=/var/log/verifier/VerificationGateway.log
-log4cplus.appender.verificationgateway=log4cplus::RollingFileAppender
-log4cplus.appender.verificationgateway.AsyncAppend=TRUE
-log4cplus.appender.verificationgateway.MaxFileSize=50MB
-log4cplus.appender.verificationgateway.MaxBackupIndex=5
-log4cplus.appender.verificationgateway.layout=log4cplus::PatternLayout
-log4cplus.appender.verificationgateway.layout.ConversionPattern=%d{%FT%T.%q}Z - %m%n
+#log4cplus.appender.pv_benchmark_appender=xtuml::KafkaAppender
+#log4cplus.appender.pv_benchmark_appender.topic=BenchmarkingProbe_service0
+#log4cplus.appender.pv_benchmark_appender.layout=log4cplus::PatternLayout
+#log4cplus.appender.pv_benchmark_appender.layout=xtuml::JsonLayout
+#log4cplus.appender.pv_benchmark_appender.layout.ConversionPattern=%d{%FT%T.%q}Z - %m