Skip to content

Commit

Permalink
add stability tests
Browse files Browse the repository at this point in the history 
 - stability tests differentiate between ert and srt
 - refactor flags and names to make sense in a world with three units
 - add env var flags for passthrough to test suites
 - clean up test containers automatically

Co-authored-by: Nick Rohn <nrohn@vmware.com>
Co-authored-by: Gary Liu <garyliu@vmware.com>
  • Loading branch information
@notrepo05 @syslxg
notrepo05 and syslxg committed Jun 21, 2023
1 parent 104b38d commit dce2397
Show file tree
Hide file tree
Showing 4 changed files with 255 additions and 34 deletions.
95 changes: 70 additions & 25 deletions internal/commands/test_tile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,13 @@ func (l infoLog) Writer() io.Writer {

type TileTest struct {
Options struct {
TilePath string `short:"tp" long:"tile-path" default:"." description:"Path to the Tile directory (e.g., ~/workspace/tas/ist)."`
GingkoManifestFlags string `short:"gmf" long:"ginkgo-manifest-flags" default:"-r -p -slowSpecThreshold 15" description:"Flags to pass to the Ginkgo Manifest test suite."`
Verbose bool `short:"v" long:"verbose" default:"false" description:"Print info lines. This doesn't affect Ginkgo output."`
ManifestOnly bool ` long:"manifest-only" default:"false" description:"Run only Manifest tests."`
MigrationsOnly bool ` long:"migrations-only" default:"false" description:"Run only Migration tests."`
TilePath string `short:"tp" long:"tile-path" default:"." description:"Path to the Tile directory (e.g., ~/workspace/tas/ist)."`
GingkoFlags string `short:"gmf" long:"ginkgo-flags" default:"-r -p -slowSpecThreshold 15" description:"Flags to pass to the Ginkgo Manifest and Stability test suites."`
Verbose bool `short:"v" long:"verbose" default:"false" description:"Print info lines. This doesn't affect Ginkgo output."`
Manifest bool ` long:"manifest" default:"false" description:"Focus the Manifest tests."`
Migrations bool ` long:"migrations" default:"false" description:"Focus the Migration tests."`
Stability bool ` long:"stability" default:"false" description:"Focus the Stability tests."`
EnvironmentVars []string `short:"e" long:"environment-variable" description:"Pass environment variable to the test suites. For example --stability -e 'PRODUCT=srt'."`
}

logger *log.Logger
Expand Down Expand Up @@ -110,6 +112,10 @@ func (u TileTest) Execute(args []string) error {
if err != nil {
return fmt.Errorf("could not parse manifest-test flags: %s", err)
}
envMap, err := validateAndParseEnvVars(u.Options.EnvironmentVars)
if err != nil {
return fmt.Errorf("could not parse manifest-test flags: %s", err)
}

loggerWithInfo := infoLog{
logger: u.logger,
Expand Down Expand Up @@ -174,8 +180,7 @@ func (u TileTest) Execute(args []string) error {
if buildError != "" {
if strings.Contains(buildError, "exit code: 128") {
format := `Does your private key have access to the ops-manifest repo?\n error: %s
Automatically looking in %s for ssh keys. We use SSH_AUTH_SOCK environment variable
for the socket.`
Automatically looking in %s for ssh keys. SSH_AUTH_SOCK needs to be set.`
return fmt.Errorf(format, buildError, strings.Join(StandardSSHKeys, ", "))
}
return errors.New(buildError)
Expand All @@ -192,27 +197,34 @@ func (u TileTest) Execute(args []string) error {

loggerWithInfo.Info("Mounting", parentDir, "and testing", tileDir)

runAll := !u.Options.ManifestOnly && !u.Options.MigrationsOnly
runAll := !u.Options.Manifest && !u.Options.Migrations && !u.Options.Stability

var dockerCmds []string
if u.Options.ManifestOnly || runAll {
dockerCmds = append(dockerCmds, fmt.Sprintf("cd /tas/%s/test/manifest", tileDir))
dockerCmds = append(dockerCmds, fmt.Sprintf("PRODUCT=%s RENDERER=ops-manifest ginkgo %s", toProduct(tileDir), u.Options.GingkoManifestFlags))
}
if u.Options.MigrationsOnly || runAll {
if u.Options.Migrations || runAll {
dockerCmds = append(dockerCmds, fmt.Sprintf("cd /tas/%s/migrations", tileDir))
dockerCmds = append(dockerCmds, "npm install")
dockerCmds = append(dockerCmds, "npm test")
}
ginkgo := []string{}
if u.Options.Stability || runAll {
ginkgo = append(ginkgo, fmt.Sprintf("/tas/%s/test/stability", tileDir))
}
if u.Options.Manifest || runAll {
ginkgo = append(ginkgo, fmt.Sprintf("/tas/%s/test/manifest", tileDir))
}
if u.Options.Stability || u.Options.Manifest || runAll {
ginkgoCommand := fmt.Sprintf("cd /tas/%s && ginkgo %s %s", tileDir, u.Options.GingkoFlags, strings.Join(ginkgo, " "))
dockerCmds = append(dockerCmds, ginkgoCommand)
}

dockerCmd := strings.Join(dockerCmds, " && ")

envVars := getTileTestEnvVars(absRepoDir, tileDir)
loggerWithInfo.Info("Running:", dockerCmd)
envVars := getTileTestEnvVars(absRepoDir, tileDir, envMap)
fmt.Printf("%+v\n", envVarsToSlice(envVars))
createResp, err := u.mobi.ContainerCreate(u.ctx, &container.Config{
Image: "kiln_test_dependencies:vmware",
Cmd: []string{"/bin/bash", "-c", dockerCmd},
Env: envVars,
Env: envVarsToSlice(envVars),
Tty: true,
}, &container.HostConfig{
LogConfig: container.LogConfig{
Expand Down Expand Up @@ -276,6 +288,18 @@ func (u TileTest) Execute(args []string) error {
return nil
}

func validateAndParseEnvVars(environmentVarArgs []string) (environmentVars, error) {
envMap := make(environmentVars)
for _, envVar := range environmentVarArgs {
parts := strings.SplitN(envVar, "=", 2)
if len(parts) != 2 {
return nil, errors.New("Environment variables must have the format [key]=[value]")
}
envMap[parts[0]] = parts[1]
}
return envMap, nil
}

func toProduct(dir string) string {
switch dir {
case "tas":
Expand All @@ -287,21 +311,32 @@ func toProduct(dir string) string {
}
}

func getTileTestEnvVars(dir, productDir string) []string {
func getTileTestEnvVars(dir, productDir string, envMap environmentVars) environmentVars {
const fixturesFormat = "%s/test/manifest/fixtures"
metadataPath := fmt.Sprintf(fixturesFormat+"/tas_metadata.yml", dir)
configPath := fmt.Sprintf(fixturesFormat+"/tas_config.yml", dir)

_, configErr := os.Stat(configPath)
_, metadataErr := os.Stat(metadataPath)

envVarsMap := make(map[string]string)
if metadataErr == nil && configErr == nil {
return []string{
fmt.Sprintf("TAS_METADATA_PATH=%s", fmt.Sprintf(fixturesFormat+"/%s", "/tas/"+productDir, "tas_metadata.yml")),
fmt.Sprintf("TAS_CONFIG_FILE=%s", fmt.Sprintf(fixturesFormat+"/%s", "/tas/"+productDir, "tas_config.yml")),
}
envVarsMap["TAS_METADATA_PATH"] = fmt.Sprintf(fixturesFormat+"/%s", "/tas/"+productDir, "tas_metadata.yml")
envVarsMap["TAS_CONFIG_FILE"] = fmt.Sprintf(fixturesFormat+"/%s", "/tas/"+productDir, "tas_config.yml")
}

return nil
// no need to set for tas tile, since it defaults to ert.
// for ist and tasw, we need to set it, as there's no default.
if toProduct(productDir) != "ert" {
envVarsMap["PRODUCT"] = toProduct(productDir)
}
envVarsMap["RENDERER"] = "ops-manifest"

// overwrite with / include optional env vars
for k, v := range envMap {
envVarsMap[k] = v
}

return envVarsMap
}

func getTarReader(fileContents string) (*bufio.Reader, error) {
Expand Down Expand Up @@ -365,14 +400,24 @@ func (u TileTest) addMissingKeys() error {
return err
}

type environmentVars map[string]string

func envVarsToSlice(envVars environmentVars) []string {
convertedEnvVars := []string{}
for k, v := range envVars {
convertedEnvVars = append(convertedEnvVars, fmt.Sprintf("%s=%s", k, v))
}
return convertedEnvVars
}

type ErrorLine struct {
Error string `json:"error"`
}

func (u TileTest) Usage() jhanda.Usage {
return jhanda.Usage{
Description: "Tests the Manifest and Migrations for a Tile in a Docker container. Requires a Docker daemon to be running and ssh keys with access to Ops Manager's Git repository. For non-interactive use, either set the environment variable SSH_PASSWORD, or `ssh add` your identity before running.",
ShortDescription: "Tests the Manifest and Migrations for a Tile.",
Description: "Run the Manifest, Migrations, and Stability tests for a Tile in a Docker container. Requires a Docker daemon to be running and ssh keys with access to Ops Manager's Git repository. For non-interactive use, either set the environment variable SSH_PASSWORD, or `ssh add` your identity before running.",
ShortDescription: "Runs unit tests for a Tile.",
Flags: u.Options,
}
}
2 changes: 1 addition & 1 deletion internal/commands/test_tile_acceptance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ var _ = Describe("test", func() {

Expect(err).To(HaveOccurred())
Expect(testOutput.String()).NotTo(ContainSubstring("SUCCESS"))
Expect(testOutput.String()).To(ContainSubstring("Failure"))
Expect(testOutput.String()).To(Or(ContainSubstring("ERR!"), ContainSubstring("Failure")))
})
})
})
Expand Down
134 changes: 126 additions & 8 deletions internal/commands/test_tile_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"log"
"os"
"path/filepath"
"sort"
"strings"

"github.com/docker/docker/api/types"
Expand Down Expand Up @@ -75,7 +76,7 @@ var _ = Describe("kiln test docker", func() {
})
When("verbose is passed", func() {
It("succeeds and logs info", func() {
err := subjectUnderTest.Execute([]string{"--manifest-only", "--verbose", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"})
err := subjectUnderTest.Execute([]string{"--manifest", "--verbose", "--tile-path", helloTilePath, "--ginkgo-flags", "-r -slowSpecThreshold 1"})
Expect(err).To(BeNil())

By("logging helpful messages", func() {
Expand All @@ -92,21 +93,38 @@ var _ = Describe("kiln test docker", func() {
Expect(fakeMobyClient.ContainerCreateCallCount()).To(Equal(1))
_, config, _, _, _, _ := fakeMobyClient.ContainerCreateArgsForCall(0)
By("configuring the metadata and product config when they exist", func() {
Expect(config.Env).To(Equal([]string{
expected := []string{
"PRODUCT=hello-tile",
"RENDERER=ops-manifest",
"TAS_METADATA_PATH=/tas/hello-tile/test/manifest/fixtures/tas_metadata.yml",
"TAS_CONFIG_FILE=/tas/hello-tile/test/manifest/fixtures/tas_config.yml",
}))
}
actual := config.Env
sort.Strings(actual)
sort.Strings(expected)

Expect(actual).To(BeEquivalentTo(expected))
})
By("executing the tests", func() {
dockerCmd := fmt.Sprintf("cd /tas/%s/test/manifest && PRODUCT=%[1]s RENDERER=ops-manifest ginkgo -r -slowSpecThreshold 1", "hello-tile")
expected := []string{
"PRODUCT=hello-tile",
"RENDERER=ops-manifest",
"TAS_METADATA_PATH=/tas/hello-tile/test/manifest/fixtures/tas_metadata.yml",
"TAS_CONFIG_FILE=/tas/hello-tile/test/manifest/fixtures/tas_config.yml",
}
actual := config.Env
sort.Strings(actual)
sort.Strings(expected)

dockerCmd := "cd /tas/hello-tile && ginkgo -r -slowSpecThreshold 1 /tas/hello-tile/test/manifest"
Expect(config.Cmd).To(Equal(strslice.StrSlice{"/bin/bash", "-c", dockerCmd}))
})
})
})
})
When("verbose isn't passed", func() {
It("doesn't log info", func() {
err := subjectUnderTest.Execute([]string{"--manifest-only", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"})
err := subjectUnderTest.Execute([]string{"--manifest", "--tile-path", helloTilePath, "--ginkgo-flags", "-r -slowSpecThreshold 1"})
Expect(err).To(BeNil())

By("logging helpful messages", func() {
Expand Down Expand Up @@ -134,7 +152,7 @@ var _ = Describe("kiln test docker", func() {
})
It("returns an error", func() {
subjectUnderTest := commands.NewTileTest(logger, ctx, fakeMobyClient, fakeSshProvider)
err := subjectUnderTest.Execute([]string{"--manifest-only", "--verbose", "--tile-path", helloTilePath, "--ginkgo-manifest-flags", "-r -slowSpecThreshold 1"})
err := subjectUnderTest.Execute([]string{"--manifest", "--verbose", "--tile-path", helloTilePath, "--ginkgo-flags", "-r -slowSpecThreshold 1"})
Expect(err).To(HaveOccurred())

By("logging helpful messages", func() {
Expand All @@ -146,6 +164,94 @@ var _ = Describe("kiln test docker", func() {
})
})

When("stability tests should be successful", func() {
const (
testSuccessLogLine = "manifest tests completed successfully"
)
var fakeMobyClient *fakes.MobyClient
BeforeEach(func() {
fakeMobyClient = setupFakeMobyClient(testSuccessLogLine, 0)
})
When("executing tests", func() {
var subjectUnderTest commands.TileTest
BeforeEach(func() {
writer.Reset()
subjectUnderTest = commands.NewTileTest(logger, ctx, fakeMobyClient, fakeSshProvider)
})
When("verbose is passed", func() {
It("succeeds and logs info", func() {
err := subjectUnderTest.Execute([]string{"--stability", "--verbose", "--tile-path", helloTilePath, "--ginkgo-flags", "-r -slowSpecThreshold 1"})
Expect(err).To(BeNil())

By("logging helpful messages", func() {
logs := writer.String()
By("logging container information", func() {
Expect(logs).To(ContainSubstring("Building / restoring cached docker image"))
})
By("logging test lines", func() {
Expect(logs).To(ContainSubstring("manifest tests completed successfully"))
})
})

By("creating a test container", func() {
Expect(fakeMobyClient.ContainerCreateCallCount()).To(Equal(1))
_, config, _, _, _, _ := fakeMobyClient.ContainerCreateArgsForCall(0)

By("executing the tests", func() {
expected := []string{
"PRODUCT=hello-tile",
"RENDERER=ops-manifest",
"TAS_METADATA_PATH=/tas/hello-tile/test/manifest/fixtures/tas_metadata.yml",
"TAS_CONFIG_FILE=/tas/hello-tile/test/manifest/fixtures/tas_config.yml",
}
actual := config.Env
sort.Strings(actual)
sort.Strings(expected)

dockerCmd := "cd /tas/hello-tile && ginkgo -r -slowSpecThreshold 1 /tas/hello-tile/test/stability"
Expect(config.Cmd).To(Equal(strslice.StrSlice{"/bin/bash", "-c", dockerCmd}))
})
})
})
})
When("verbose isn't passed", func() {
It("doesn't log info", func() {
err := subjectUnderTest.Execute([]string{"--stability", "--tile-path", helloTilePath, "--ginkgo-flags", "-r -slowSpecThreshold 1"})
Expect(err).To(BeNil())

By("logging helpful messages", func() {
logs := writer.String()
By("logging container information", func() {
Expect(logs).NotTo(ContainSubstring("Building / restoring cached docker image"))
Expect(logs).NotTo(ContainSubstring("Info:"))
})
By("logging test lines", func() {
Expect(logs).To(ContainSubstring("manifest tests completed successfully"))
})
})
})
})
})
})

When("stability tests shouldn't be successful", func() {
const (
testSuccessLogLine = "stability tests completed successfully"
)
var fakeMobyClient *fakes.MobyClient

BeforeEach(func() {
fakeMobyClient = setupFakeMobyClient(testSuccessLogLine, 0)
})

It("exits with an error if env vars are incorrectly formatted", func() {
subjectUnderTest := commands.NewTileTest(logger, ctx, fakeMobyClient, fakeSshProvider)
err := subjectUnderTest.Execute([]string{"--stability", "--verbose", "--tile-path", helloTilePath, "--ginkgo-flags", "-r -slowSpecThreshold 1", "--environment-variable", "MISFORMATTED_ENV_VAR"})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("Environment variables must have the format [key]=[value]"))
})
})

When("all tests are run", func() {
var fakeMobyClient *fakes.MobyClient
BeforeEach(func() {
Expand All @@ -167,7 +273,18 @@ var _ = Describe("kiln test docker", func() {
_, config, _, _, _, _ := fakeMobyClient.ContainerCreateArgsForCall(0)

By("executing the tests", func() {
dockerCmd := "cd /tas/hello-tile/test/manifest && PRODUCT=hello-tile RENDERER=ops-manifest ginkgo -r -p -slowSpecThreshold 15 && cd /tas/hello-tile/migrations && npm install && npm test"
dockerCmd := "cd /tas/hello-tile/migrations && npm install && npm test && cd /tas/hello-tile && ginkgo -r -p -slowSpecThreshold 15 /tas/hello-tile/test/stability /tas/hello-tile/test/manifest"
actual := config.Env
expected := []string{
"TAS_CONFIG_FILE=/tas/hello-tile/test/manifest/fixtures/tas_config.yml",
"PRODUCT=hello-tile",
"RENDERER=ops-manifest",
"TAS_METADATA_PATH=/tas/hello-tile/test/manifest/fixtures/tas_metadata.yml",
}

sort.Strings(expected)
sort.Strings(actual)
Expect(actual).To(Equal(expected))
Expect(config.Cmd).To(Equal(strslice.StrSlice{"/bin/bash", "-c", dockerCmd}))
})
})
Expand All @@ -191,7 +308,7 @@ var _ = Describe("kiln test docker", func() {
})

It("succeeds and logs info", func() {
err := subjectUnderTest.Execute([]string{"--migrations-only", "--verbose", "--tile-path", helloTilePath})
err := subjectUnderTest.Execute([]string{"--migrations", "--verbose", "--tile-path", helloTilePath})
Expect(err).To(BeNil())

By("logging helpful messages", func() {
Expand Down Expand Up @@ -228,6 +345,7 @@ var _ = Describe("kiln test docker", func() {
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("Docker daemon is not running"))
})

})
})

Expand Down
Loading

0 comments on commit dce2397

Please sign in to comment.