feat: App stats availability measurement

Currently concerned with when stats are seen as unavailable by the
client. Does not include when stats are set to zero by the Cloud
Controller when one or more required metrics are missing.

Co-authored-by: Rebecca Roberts <rebecca.roberts@broadcom.com>

README.md

@@ -12,12 +12,13 @@ It measures:
   by periodically pushing a very simple app.
 - app syslog availability,
   by periodically checking that app logs
-  drain to a syslog sink.
+  drain to a syslog sink,
+- app stats availability,
+  by periodically checking that app stats
+  are not unavailable.
 
-The CF Release Integration team uses it
-to monitor availability during migrations
-from `cf-release` to `cf-deployment`,
-and during upgrade deployments.
+It is often used to monitor availability
+during upgrade deployments.
 
 ## Installation
 
@@ -66,6 +67,7 @@ Here is an example config `json`:
     "allowed_failures": {
         "app_pushability": 2,
         "http_availability": 5,
+        "app_stats": 2,
         "recent_logs": 2,
         "streaming_logs": 2,
         "app_syslog_availability": 2

cfCmdGenerator/cfCmdGenerator.go

@@ -24,6 +24,7 @@ type CfCmdGenerator interface {
 	DeleteOrg(org string) cmdStartWaiter.CmdStartWaiter
 	DeleteQuota(quota string) cmdStartWaiter.CmdStartWaiter
 	LogOut() cmdStartWaiter.CmdStartWaiter
+	AppStats(appName string) cmdStartWaiter.CmdStartWaiter
 	RecentLogs(appName string) cmdStartWaiter.CmdStartWaiter
 	StreamLogs(ctx context.Context, appName string) cmdStartWaiter.CmdStartWaiter
 	MapRoute(appName, domain string, port int) cmdStartWaiter.CmdStartWaiter
@@ -175,6 +176,14 @@ func (c *cfCmdGenerator) LogOut() cmdStartWaiter.CmdStartWaiter {
 	)
 }
 
+func (c *cfCmdGenerator) AppStats(appName string) cmdStartWaiter.CmdStartWaiter {
+	return c.setCfHome(
+		exec.Command(
+			"cf", "app", appName,
+		),
+	)
+}
+
 func (c *cfCmdGenerator) RecentLogs(appName string) cmdStartWaiter.CmdStartWaiter {
 	return c.setCfHome(
 		exec.Command(

cfCmdGenerator/cfCmdGenerator_test.go

@@ -162,6 +162,14 @@ var _ = Describe("CfCmdGenerator", func() {
 		})
 	})
 
+	Describe("AppStats", func() {
+		It("Generates the correct command", func() {
+			expectedCmd := exec.Command("cf", "app", "appName")
+			cmd := generator.AppStats("appName")
+			expectCommandToBeEquivalent(cmd, expectedCmd, cfHomeEnvVar)
+		})
+	})
+
 	Describe("RecentLogs", func() {
 		It("Generates the correct command", func() {
 			expectedCmd := exec.Command("cf", "logs", "appName", "--recent")

cfWorkflow/cfWorkflow.go

@@ -24,6 +24,7 @@ type CfWorkflow interface {
 	Delete(cfCmdGenerator.CfCmdGenerator) []cmdStartWaiter.CmdStartWaiter
 	TearDown(cfCmdGenerator.CfCmdGenerator) []cmdStartWaiter.CmdStartWaiter
 	RecentLogs(cfCmdGenerator.CfCmdGenerator) []cmdStartWaiter.CmdStartWaiter
+	AppStats(cfCmdGenerator.CfCmdGenerator) []cmdStartWaiter.CmdStartWaiter
 	StreamLogs(context.Context, cfCmdGenerator.CfCmdGenerator) []cmdStartWaiter.CmdStartWaiter
 
 	MapSyslogRoute(cfCmdGenerator.CfCmdGenerator) []cmdStartWaiter.CmdStartWaiter
@@ -141,6 +142,15 @@ func (c *cfWorkflow) TearDown(ccg cfCmdGenerator.CfCmdGenerator) []cmdStartWaite
 	return ret
 }
 
+func (c *cfWorkflow) AppStats(ccg cfCmdGenerator.CfCmdGenerator) []cmdStartWaiter.CmdStartWaiter {
+	return []cmdStartWaiter.CmdStartWaiter{
+		ccg.Api(c.cf.API),
+		ccg.Auth(c.cf.AdminUser, c.cf.AdminPassword),
+		ccg.Target(c.org, c.space),
+		ccg.AppStats(c.appName),
+	}
+}
+
 func (c *cfWorkflow) RecentLogs(ccg cfCmdGenerator.CfCmdGenerator) []cmdStartWaiter.CmdStartWaiter {
 	return []cmdStartWaiter.CmdStartWaiter{
 		ccg.Api(c.cf.API),

cfWorkflow/cfWorkflow_test.go

@@ -213,6 +213,21 @@ var _ = Describe("CfWorkflow", func() {
 		})
 	})
 
+	Describe("AppStats", func() {
+		It("returns a set of commands to get stats for an app", func() {
+			cmds := cw.AppStats(ccg)
+
+			Expect(cmds).To(Equal(
+				[]cmdStartWaiter.CmdStartWaiter{
+					ccg.Api("jigglypuff.cf-app.com"),
+					ccg.Auth("pika", "chu"),
+					ccg.Target("someOrg", "someSpace"),
+					ccg.AppStats("doraApp"),
+				},
+			))
+		})
+	})
+
 	Describe("RecentLogs", func() {
 		It("returns a set of commands to get recent logs for an app", func() {
 			cmds := cw.RecentLogs(ccg)

config/config.go

@@ -36,6 +36,7 @@ type AllowedFailures struct {
 	HttpAvailability      int `json:"http_availability"`
 	RecentLogs            int `json:"recent_logs"`
 	StreamingLogs         int `json:"streaming_logs"`
+	AppStats              int `json:"app_stats"`
 	AppSyslogAvailability int `json:"app_syslog_availability"`
 	TCPAvailability       int `json:"tcp_availability"`
 }

main.go

@@ -97,7 +97,7 @@ func main() {
 		}
 		logger.Println("Finished preparing included syslog sink app")
 	}
-	orcTmpDir, recentLogsTmpDir, streamingLogsTmpDir, pushTmpDir, tcpTmpDir, sinkTmpDir, err := createTmpDirs()
+	orcTmpDir, recentLogsTmpDir, streamingLogsTmpDir, appStatsTmpDir, pushTmpDir, tcpTmpDir, sinkTmpDir, err := createTmpDirs()
 	if err != nil {
 		logger.Println("Failed to create temp dirs:", err)
 		performMeasurements = false
@@ -179,6 +179,7 @@ func main() {
 		pushWorkflowGeneratorFunc,
 		cfCmdGenerator.New(recentLogsTmpDir, *useBuildpackDetection),
 		cfCmdGenerator.New(streamingLogsTmpDir, *useBuildpackDetection),
+		cfCmdGenerator.New(appStatsTmpDir, *useBuildpackDetection),
 		pushCmdGenerator,
 		cfg.AllowedFailures,
 		authFailedRetryFunc,
@@ -250,33 +251,37 @@ func main() {
 	os.Exit(exitCode)
 }
 
-func createTmpDirs() (string, string, string, string, string, string, error) {
+func createTmpDirs() (string, string, string, string, string, string, string, error) {
 	orcTmpDir, err := os.MkdirTemp("", "uptimer")
 	if err != nil {
-		return "", "", "", "", "", "", err
+		return "", "", "", "", "", "", "", err
 	}
 	recentLogsTmpDir, err := os.MkdirTemp("", "uptimer")
 	if err != nil {
-		return "", "", "", "", "", "", err
+		return "", "", "", "", "", "", "", err
 	}
 	streamingLogsTmpDir, err := os.MkdirTemp("", "uptimer")
 	if err != nil {
-		return "", "", "", "", "", "", err
+		return "", "", "", "", "", "", "", err
+	}
+	appsStatsTmpDir, err := os.MkdirTemp("", "uptimer")
+	if err != nil {
+		return "", "", "", "", "", "", "", err
 	}
 	pushTmpDir, err := os.MkdirTemp("", "uptimer")
 	if err != nil {
-		return "", "", "", "", "", "", err
+		return "", "", "", "", "", "", "", err
 	}
 	tcpTmpDir, err := os.MkdirTemp("", "uptimer")
 	if err != nil {
-		return "", "", "", "", "", "", err
+		return "", "", "", "", "", "", "", err
 	}
 	sinkTmpDir, err := os.MkdirTemp("", "uptimer")
 	if err != nil {
-		return "", "", "", "", "", "", err
+		return "", "", "", "", "", "", "", err
 	}
 
-	return orcTmpDir, recentLogsTmpDir, streamingLogsTmpDir, pushTmpDir, tcpTmpDir, sinkTmpDir, nil
+	return orcTmpDir, recentLogsTmpDir, streamingLogsTmpDir, appsStatsTmpDir, pushTmpDir, tcpTmpDir, sinkTmpDir, nil
 }
 
 func prepareIncludedApp(name, source string) (string, error) {
@@ -328,7 +333,7 @@ func createMeasurements(
 	logger *log.Logger,
 	orcWorkflow cfWorkflow.CfWorkflow,
 	pushWorkFlowGeneratorFunc func() cfWorkflow.CfWorkflow,
-	recentLogsCmdGenerator, streamingLogsCmdGenerator, pushCmdGenerator cfCmdGenerator.CfCmdGenerator,
+	recentLogsCmdGenerator, streamingLogsCmdGenerator, pushCmdGenerator, appStatsCmdGenerator cfCmdGenerator.CfCmdGenerator,
 	allowedFailures config.AllowedFailures,
 	authFailedRetryFunc func(stdOut, stdErr string) bool,
 ) []measurement.Measurement {
@@ -380,6 +385,16 @@ func createMeasurements(
 		},
 	)
 
+	appStatsRunner, appStatsRunnerOutBuf, appStatsRunnerErrBuf := createBufferedRunner()
+	appStatsMeasurement := measurement.NewStatsAvailability(
+		func() []cmdStartWaiter.CmdStartWaiter {
+			return orcWorkflow.AppStats(appStatsCmdGenerator)
+		},
+		appStatsRunner,
+		appStatsRunnerOutBuf,
+		appStatsRunnerErrBuf,
+	)
+
 	return []measurement.Measurement{
 		measurement.NewPeriodic(
 			logger,
@@ -417,6 +432,15 @@ func createMeasurements(
 			allowedFailures.StreamingLogs,
 			authFailedRetryFunc,
 		),
+		measurement.NewPeriodic(
+			logger,
+			clock,
+			10*time.Second,
+			appStatsMeasurement,
+			measurement.NewResultSet(),
+			allowedFailures.AppStats,
+			authFailedRetryFunc,
+		),
 	}
 }
 

measurement/measurement.go

@@ -166,3 +166,19 @@ func NewAppPushability(
 		runnerErrBuf:                         runnerErrBuf,
 	}
 }
+
+func NewStatsAvailability(
+	statsAvailabilityCommandGeneratorFunc func() []cmdStartWaiter.CmdStartWaiter,
+	runner cmdRunner.CmdRunner,
+	runnerOutBuf *bytes.Buffer,
+	runnerErrBuf *bytes.Buffer,
+) BaseMeasurement {
+	return &statsAvailability{
+		name:                                  "Stats availability",
+		summaryPhrase:                         "retrieve stats for app",
+		statsAvailabilityCommandGeneratorFunc: statsAvailabilityCommandGeneratorFunc,
+		runner:                                runner,
+		runnerOutBuf:                          runnerOutBuf,
+		runnerErrBuf:                          runnerErrBuf,
+	}
+}

measurement/statsAvailability.go

@@ -0,0 +1,44 @@
+package measurement
+
+import (
+	"bytes"
+	"strings"
+
+	"github.com/cloudfoundry/uptimer/cmdRunner"
+	"github.com/cloudfoundry/uptimer/cmdStartWaiter"
+)
+
+type statsAvailability struct {
+	name                                  string
+	summaryPhrase                         string
+	statsAvailabilityCommandGeneratorFunc func() []cmdStartWaiter.CmdStartWaiter
+	runner                                cmdRunner.CmdRunner
+	runnerOutBuf                          *bytes.Buffer
+	runnerErrBuf                          *bytes.Buffer
+}
+
+func (s *statsAvailability) Name() string {
+	return s.name
+}
+
+func (s *statsAvailability) SummaryPhrase() string {
+	return s.summaryPhrase
+}
+
+func (s *statsAvailability) PerformMeasurement() (string, string, string, bool) {
+	defer s.runnerOutBuf.Reset()
+	defer s.runnerErrBuf.Reset()
+
+	if err := s.runner.RunInSequence(s.statsAvailabilityCommandGeneratorFunc()...); err != nil {
+		return err.Error(), s.runnerOutBuf.String(), s.runnerErrBuf.String(), false
+	}
+
+	if strings.Contains(s.runnerErrBuf.String(), "Stats server temporarily unavailable.") {
+		return "Stats server was unavailable",
+			s.runnerOutBuf.String(),
+			s.runnerErrBuf.String(),
+			false
+	}
+
+	return "", "", "", true
+}