diff --git a/healthcheck/pkg/db/addr.go b/cmd/mongodb-healthcheck/db/config.go similarity index 81% rename from healthcheck/pkg/db/addr.go rename to cmd/mongodb-healthcheck/db/config.go index 29541b0ea3..898501b2e0 100644 --- a/healthcheck/pkg/db/addr.go +++ b/cmd/mongodb-healthcheck/db/config.go @@ -14,13 +14,11 @@ package db -import "strconv" +import ( + "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo" +) -type Addr struct { - Host string - Port int -} - -func (a Addr) String() string { - return a.Host + ":" + strconv.Itoa(a.Port) +type Config struct { + mongo.Config + SSL *SSLConfig } diff --git a/healthcheck/tools/db/db.go b/cmd/mongodb-healthcheck/db/db.go similarity index 56% rename from healthcheck/tools/db/db.go rename to cmd/mongodb-healthcheck/db/db.go index 19eff43a50..f9d7c53f27 100644 --- a/healthcheck/tools/db/db.go +++ b/cmd/mongodb-healthcheck/db/db.go @@ -16,11 +16,8 @@ package db import ( "context" - "time" "github.com/pkg/errors" - mgo "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" logf "sigs.k8s.io/controller-runtime/pkg/log" "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo" @@ -39,44 +36,20 @@ func Dial(ctx context.Context, conf *Config) (mongo.Client, error) { log := logf.FromContext(ctx) log.V(1).Info("Connecting to mongodb", "hosts", conf.Hosts, "ssl", conf.SSL.Enabled, "ssl_insecure", conf.SSL.Insecure) - opts := options.Client(). - SetHosts(conf.Hosts). - SetReplicaSet(conf.ReplSetName). - SetAuth(options.Credential{Password: conf.Password, Username: conf.Username}). - SetTLSConfig(conf.TLSConf). - SetConnectTimeout(10 * time.Second). - SetServerSelectionTimeout(10 * time.Second) - if conf.Username != "" && conf.Password != "" { log.V(1).Info("Enabling authentication for session", "user", conf.Username) } - client, err := mgo.Connect(ctx, opts) + cl, err := mongo.Dial(&conf.Config) if err != nil { - return nil, errors.Wrap(err, "connect to mongo replica set") - } - - if err := client.Ping(ctx, nil); err != nil { - if err := client.Disconnect(ctx); err != nil { - return nil, errors.Wrap(err, "disconnect client") - } - - opts := options.Client(). - SetHosts(conf.Hosts). - SetTLSConfig(conf.TLSConf). - SetConnectTimeout(10 * time.Second). - SetServerSelectionTimeout(10 * time.Second). - SetDirect(true) - - client, err = mgo.Connect(ctx, opts) + cfg := conf.Config + cfg.Direct = true + cfg.ReplSetName = "" + cl, err = mongo.Dial(&cfg) if err != nil { - return nil, errors.Wrap(err, "connect to mongo replica set with direct") - } - - if err := client.Ping(ctx, nil); err != nil { - return nil, errors.Wrap(err, "ping mongo") + return nil, errors.Wrap(err, "filed to dial mongo") } } - return mongo.ToInterface(client), nil + return cl, nil } diff --git a/healthcheck/tools/db/ssl.go b/cmd/mongodb-healthcheck/db/ssl.go similarity index 86% rename from healthcheck/tools/db/ssl.go rename to cmd/mongodb-healthcheck/db/ssl.go index b0c84b6de5..0c38fe8f9b 100644 --- a/healthcheck/tools/db/ssl.go +++ b/cmd/mongodb-healthcheck/db/ssl.go @@ -23,8 +23,6 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" ) -var lastSSLErr error - type SSLConfig struct { Enabled bool PEMKeyFile string @@ -42,11 +40,6 @@ func (sc *SSLConfig) loadCaCertificate() (*x509.CertPool, error) { return certificates, nil } -// LastSSLError returns the last error related to the DB connection SSL handshake -func LastSSLError() error { - return lastSSLErr -} - func (cnf *Config) configureTLS() error { log := logf.Log @@ -60,7 +53,7 @@ func (cnf *Config) configureTLS() error { // Configure client cert if len(cnf.SSL.PEMKeyFile) != 0 { - if err := isFileExists(cnf.SSL.PEMKeyFile); err != nil { + if _, err := os.Stat(cnf.SSL.PEMKeyFile); err != nil { return errors.Wrapf(err, "check if file with name %s exists", cnf.SSL.PEMKeyFile) } @@ -75,7 +68,7 @@ func (cnf *Config) configureTLS() error { // Configure CA cert if len(cnf.SSL.CAFile) != 0 { - if err := isFileExists(cnf.SSL.CAFile); err != nil { + if _, err := os.Stat(cnf.SSL.CAFile); err != nil { return errors.Wrapf(err, "check if file with name %s exists", cnf.SSL.CAFile) } @@ -91,8 +84,3 @@ func (cnf *Config) configureTLS() error { cnf.TLSConf = config return nil } - -func isFileExists(name string) error { - _, err := os.Stat(name) - return err -} diff --git a/healthcheck/tools/db/ssl_test.go b/cmd/mongodb-healthcheck/db/ssl_test.go similarity index 100% rename from healthcheck/tools/db/ssl_test.go rename to cmd/mongodb-healthcheck/db/ssl_test.go diff --git a/healthcheck/health.go b/cmd/mongodb-healthcheck/healthcheck/health.go similarity index 72% rename from healthcheck/health.go rename to cmd/mongodb-healthcheck/healthcheck/health.go index 6e79b4f3cb..9e8707ede8 100644 --- a/healthcheck/health.go +++ b/cmd/mongodb-healthcheck/healthcheck/health.go @@ -22,61 +22,17 @@ import ( "github.com/pkg/errors" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" + logf "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/percona/percona-server-mongodb-operator/healthcheck/tools/db" + "github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/db" "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo" ) -// OkMemberStates is a slice of acceptable replication member states -var OkMemberStates = []mongo.MemberState{ - mongo.MemberStatePrimary, - mongo.MemberStateSecondary, - mongo.MemberStateRecovering, - mongo.MemberStateArbiter, - mongo.MemberStateStartup2, - mongo.MemberStateRollback, -} - var ErrNoReplsetConfigStr string = "(NotYetInitialized) no replset config has been received" -// getSelfMemberState returns the replication state of the local MongoDB member -func getSelfMemberState(rsStatus *mongo.Status) *mongo.MemberState { - member := rsStatus.GetSelf() - if member == nil || member.Health != mongo.MemberHealthUp { - return nil - } - return &member.State -} - -// isStateOk checks if a replication member state matches one of the acceptable member states in 'OkMemberStates' -func isStateOk(memberState *mongo.MemberState, okMemberStates []mongo.MemberState) bool { - for _, state := range okMemberStates { - if *memberState == state { - return true - } - } - return false -} - -// HealthCheck checks the replication member state of the local MongoDB member -func HealthCheck(client mongo.Client, okMemberStates []mongo.MemberState) (State, *mongo.MemberState, error) { - rsStatus, err := client.RSStatus(context.TODO()) - if err != nil { - return StateFailed, nil, errors.Wrap(err, "get replica set status") - } - - state := getSelfMemberState(&rsStatus) - if state == nil { - return StateFailed, state, errors.New("found no member state for self in replica set status") - } - if isStateOk(state, okMemberStates) { - return StateOk, state, nil - } - - return StateFailed, state, errors.Errorf("member has unhealthy replication state: %d", state) -} - func HealthCheckMongosLiveness(ctx context.Context, cnf *db.Config) (err error) { + log := logf.FromContext(ctx).WithName("HealthCheckMongosLiveness") + client, err := db.Dial(ctx, cnf) if err != nil { return errors.Wrap(err, "connection error") @@ -93,6 +49,7 @@ func HealthCheckMongosLiveness(ctx context.Context, cnf *db.Config) (err error) } if isMasterResp.Msg != "isdbgrid" { + log.V(1).Info("Wrong isMaster msg", "msg", isMasterResp.Msg) return errors.New("wrong msg") } @@ -100,6 +57,8 @@ func HealthCheckMongosLiveness(ctx context.Context, cnf *db.Config) (err error) } func HealthCheckMongodLiveness(ctx context.Context, cnf *db.Config, startupDelaySeconds int64) (_ *mongo.MemberState, err error) { + log := logf.FromContext(ctx).WithName("HealthCheckMongodLiveness") + client, err := db.Dial(ctx, cnf) if err != nil { return nil, errors.Wrap(err, "connection error") @@ -134,6 +93,7 @@ func HealthCheckMongodLiveness(ctx context.Context, cnf *db.Config, startupDelay // to die before they added to a replset if res.Err().Error() == ErrNoReplsetConfigStr { state := mongo.MemberStateUnknown + log.V(1).Info("replSetGetStatus failed", "err", res.Err().Error(), "state", state) return &state, nil } return nil, errors.Wrap(res.Err(), "get replsetGetStatus response") @@ -162,6 +122,7 @@ func HealthCheckMongodLiveness(ctx context.Context, cnf *db.Config, startupDelay oplogRs := OplogRs{} if !isMasterResp.IsArbiter { + log.V(1).Info("Getting \"oplog.rs\" info") res := client.Database("local").RunCommand(ctx, bson.D{ {Key: "collStats", Value: "oplog.rs"}, {Key: "scale", Value: 1024 * 1024 * 1024}, // scale size to gigabytes @@ -172,8 +133,8 @@ func HealthCheckMongodLiveness(ctx context.Context, cnf *db.Config, startupDelay if err := res.Decode(&oplogRs); err != nil { return nil, errors.Wrap(err, "decode oplog.rs info") } - if oplogRs.Ok == 0 { - return nil, errors.New(oplogRs.Errmsg) + if oplogRs.OK == 0 { + return nil, errors.Wrap(errors.New("non-ok response from getting oplog.rs info"), oplogRs.Errmsg) } } @@ -182,6 +143,7 @@ func HealthCheckMongodLiveness(ctx context.Context, cnf *db.Config, startupDelay storageSize = oplogRs.StorageSize } + log.V(1).Info("Checking state", "state", rsStatus.MyState, "storage size", storageSize) if err := CheckState(rsStatus, startupDelaySeconds, storageSize); err != nil { return &rsStatus.MyState, err } @@ -189,21 +151,14 @@ func HealthCheckMongodLiveness(ctx context.Context, cnf *db.Config, startupDelay return &rsStatus.MyState, nil } -type ServerStatus struct { - Ok int `bson:"ok" json:"ok"` - Errmsg string `bson:"errmsg,omitempty" json:"errmsg,omitempty"` -} - type OplogRs struct { - StorageSize int64 `bson:"storageSize" json:"storageSize"` - - Ok int `bson:"ok" json:"ok"` - Errmsg string `bson:"errmsg,omitempty" json:"errmsg,omitempty"` + mongo.OKResponse `bson:",inline"` + StorageSize int64 `bson:"storageSize" json:"storageSize"` } type ReplSetStatus struct { - mongo.Status `bson:",inline"` InitialSyncStatus InitialSyncStatus `bson:"initialSyncStatus" json:"initialSyncStatus"` + mongo.Status `bson:",inline"` } type InitialSyncStatus interface{} diff --git a/healthcheck/readiness.go b/cmd/mongodb-healthcheck/healthcheck/readiness.go similarity index 68% rename from healthcheck/readiness.go rename to cmd/mongodb-healthcheck/healthcheck/readiness.go index 3631499fa3..b58ed28c44 100644 --- a/healthcheck/readiness.go +++ b/cmd/mongodb-healthcheck/healthcheck/readiness.go @@ -20,13 +20,19 @@ import ( "github.com/pkg/errors" "go.mongodb.org/mongo-driver/bson" + logf "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/percona/percona-server-mongodb-operator/healthcheck/tools/db" + "github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/db" + "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo" ) -// ReadinessCheck runs a ping on a pmgo.SessionManager to check server readiness +// MongodReadinessCheck runs a ping on a pmgo.SessionManager to check server readiness func MongodReadinessCheck(ctx context.Context, addr string) error { + log := logf.FromContext(ctx).WithName("MongodReadinessCheck") + var d net.Dialer + + log.V(1).Info("Connecting to " + addr) conn, err := d.DialContext(ctx, "tcp", addr) if err != nil { return errors.Wrap(err, "dial") @@ -35,6 +41,8 @@ func MongodReadinessCheck(ctx context.Context, addr string) error { } func MongosReadinessCheck(ctx context.Context, cnf *db.Config) (err error) { + log := logf.FromContext(ctx).WithName("MongosReadinessCheck") + client, err := db.Dial(ctx, cnf) if err != nil { return errors.Wrap(err, "connection error") @@ -45,21 +53,23 @@ func MongosReadinessCheck(ctx context.Context, cnf *db.Config) (err error) { } }() - ss := ServerStatus{} + log.V(1).Info("Running listDatabases") + resp := mongo.OKResponse{} cur := client.Database("admin").RunCommand(ctx, bson.D{ {Key: "listDatabases", Value: 1}, {Key: "filter", Value: bson.D{{Key: "name", Value: "admin"}}}, - {Key: "nameOnly", Value: true}}) + {Key: "nameOnly", Value: true}, + }) if cur.Err() != nil { return errors.Wrap(cur.Err(), "run listDatabases") } - if err := cur.Decode(&ss); err != nil { + if err := cur.Decode(&resp); err != nil { return errors.Wrap(err, "decode listDatabases response") } - if ss.Ok == 0 { - return errors.New(ss.Errmsg) + if resp.OK == 0 { + return errors.Wrap(errors.New("non-ok response from listDatabases"), resp.Errmsg) } return nil diff --git a/cmd/mongodb-healthcheck/main.go b/cmd/mongodb-healthcheck/main.go index c0f5bb55b9..6fd4525922 100644 --- a/cmd/mongodb-healthcheck/main.go +++ b/cmd/mongodb-healthcheck/main.go @@ -16,21 +16,22 @@ package main import ( "context" + "io" "os" "os/signal" + "path/filepath" "strconv" "strings" "syscall" uzap "go.uber.org/zap" "go.uber.org/zap/zapcore" + "gopkg.in/natefinch/lumberjack.v2" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "github.com/percona/percona-server-mongodb-operator/healthcheck" - "github.com/percona/percona-server-mongodb-operator/healthcheck/pkg" - "github.com/percona/percona-server-mongodb-operator/healthcheck/tools/db" - "github.com/percona/percona-server-mongodb-operator/healthcheck/tools/tool" + "github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/tool" + "github.com/percona/percona-server-mongodb-operator/pkg/psmdb" ) var ( @@ -42,108 +43,34 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt) defer stop() - app := tool.New("Performs health and readiness checks for MongoDB", GitCommit, GitBranch) - - k8sCmd := app.Command("k8s", "Performs liveness check for MongoDB on Kubernetes") - livenessCmd := k8sCmd.Command("liveness", "Run a liveness check of MongoDB").Default() - _ = k8sCmd.Command("readiness", "Run a readiness check of MongoDB") - startupDelaySeconds := livenessCmd.Flag("startupDelaySeconds", "").Default("7200").Uint64() - component := k8sCmd.Flag("component", "").Default("mongod").String() - - opts := zap.Options{ - Encoder: getLogEncoder(), - Level: getLogLevel(), - } - log := zap.New(zap.UseFlagOptions(&opts)) - logf.SetLogger(log) - - restoreInProgress, err := fileExists("/opt/percona/restore-in-progress") - if err != nil { - log.Error(err, "check if restore in progress") - os.Exit(1) - } - - if restoreInProgress { - os.Exit(0) - } - - sleepForever, err := fileExists("/data/db/sleep-forever") - if err != nil { - log.Error(err, "check if sleep-forever file exists") - os.Exit(1) - } - if sleepForever { - os.Exit(0) - } - - cnf, err := db.NewConfig( - app, - pkg.EnvMongoDBClusterMonitorUser, - pkg.EnvMongoDBClusterMonitorPassword, - ) - - if err != nil { - log.Error(err, "new cfg") - os.Exit(1) + logPath := filepath.Join(psmdb.MongodDataVolClaimName, "logs", "mongodb-healthcheck.log") + logRotateWriter := lumberjack.Logger{ + Filename: logPath, + MaxSize: 100, + MaxAge: 1, + MaxBackups: 0, + Compress: true, } - - command, err := app.Parse(os.Args[1:]) - if err != nil { - log.Error(err, "Cannot parse command line") - os.Exit(1) + logOpts := zap.Options{ + Encoder: getLogEncoder(), + Level: getLogLevel(), + DestWriter: io.MultiWriter(os.Stderr, &logRotateWriter), } - switch command { - case "k8s liveness": - log.Info("Running Kubernetes liveness check for", "component", component) - switch *component { - - case "mongod": - memberState, err := healthcheck.HealthCheckMongodLiveness(ctx, cnf, int64(*startupDelaySeconds)) - if err != nil { - log.Error(err, "Member failed Kubernetes liveness check") - os.Exit(1) - } - log.Info("Member passed Kubernetes liveness check with replication state", "state", memberState) - - case "mongos": - err := healthcheck.HealthCheckMongosLiveness(ctx, cnf) - if err != nil { - log.Error(err, "Member failed Kubernetes liveness check") - os.Exit(1) - } - log.Info("Member passed Kubernetes liveness check") - } + log := zap.New(zap.UseFlagOptions(&logOpts)) + logf.SetLogger(log) + ctx = logf.IntoContext(ctx, log) - case "k8s readiness": - log.Info("Running Kubernetes readiness check for component", "component", component) - switch *component { - - case "mongod": - err := healthcheck.MongodReadinessCheck(ctx, cnf.Hosts[0]) - if err != nil { - log.Error(err, "Member failed Kubernetes readiness check") - os.Exit(1) - } - case "mongos": - err := healthcheck.MongosReadinessCheck(ctx, cnf) - if err != nil { - log.Error(err, "Member failed Kubernetes readiness check") - os.Exit(1) - } - } - } -} + log.Info("Running mongodb-healthcheck", "commit", GitCommit, "branch", GitBranch) -func fileExists(name string) (bool, error) { - _, err := os.Stat(name) - if err != nil { - if os.IsNotExist(err) { - return false, nil + app := tool.New("Performs health and readiness checks for MongoDB", GitCommit, GitBranch) + if err := app.Run(ctx); err != nil { + log.Error(err, "Failed to perform check") + if err := logRotateWriter.Rotate(); err != nil { + log.Error(err, "failed to rotate logs") } - return false, err + os.Exit(1) } - return true, nil } func getLogEncoder() zapcore.Encoder { @@ -154,11 +81,11 @@ func getLogEncoder() zapcore.Encoder { return consoleEnc } - useJson, err := strconv.ParseBool(s) + useJSON, err := strconv.ParseBool(s) if err != nil { return consoleEnc } - if !useJson { + if !useJSON { return consoleEnc } @@ -166,9 +93,11 @@ func getLogEncoder() zapcore.Encoder { } func getLogLevel() zapcore.LevelEnabler { + defaultLogLevel := zapcore.DebugLevel + l, found := os.LookupEnv("LOG_LEVEL") if !found { - return zapcore.InfoLevel + return defaultLogLevel } switch strings.ToUpper(l) { @@ -179,6 +108,6 @@ func getLogLevel() zapcore.LevelEnabler { case "ERROR": return zapcore.ErrorLevel default: - return zapcore.InfoLevel + return defaultLogLevel } } diff --git a/healthcheck/tools/db/config.go b/cmd/mongodb-healthcheck/tool/config.go similarity index 76% rename from healthcheck/tools/db/config.go rename to cmd/mongodb-healthcheck/tool/config.go index a132827abc..907cddfecb 100644 --- a/healthcheck/tools/db/config.go +++ b/cmd/mongodb-healthcheck/tool/config.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package db +package tool import ( "fmt" @@ -20,9 +20,9 @@ import ( "time" "github.com/alecthomas/kingpin" - "github.com/percona/percona-server-mongodb-operator/healthcheck/pkg" - "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo" "github.com/pkg/errors" + + "github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/db" ) var ( @@ -33,31 +33,16 @@ var ( DefaultMongoDBTimeoutDuration = time.Duration(5) * time.Second ) -type Config struct { - mongo.Config - SSL *SSLConfig -} - -func getDefaultMongoDBAddress() string { - hostname := DefaultMongoDBHost - - mongodbPort := os.Getenv(pkg.EnvMongoDBPort) - if mongodbPort != "" { - return hostname + ":" + mongodbPort - } - return hostname + ":" + DefaultMongoDBPort -} - -func NewConfig(app *kingpin.Application, envUser string, envPassword string) (*Config, error) { - conf := &Config{} +func NewConfig(app *kingpin.Application, envUser string, envPassword string) (*db.Config, error) { + conf := &db.Config{} app.Flag( "address", "mongodb server address (hostname:port), defaults to '$TASK_NAME.$FRAMEWORK_HOST:$MONGODB_PORT' if the env vars are available and SSL is used, if not the default is '"+DefaultMongoDBHost+":"+DefaultMongoDBPort+"'", ).Default(getDefaultMongoDBAddress()).StringsVar(&conf.Hosts) app.Flag( "replset", - "mongodb replica set name, overridden by env var "+pkg.EnvMongoDBReplset, - ).Envar(pkg.EnvMongoDBReplset).StringVar(&conf.ReplSetName) + "mongodb replica set name, overridden by env var "+EnvMongoDBReplset, + ).Envar(EnvMongoDBReplset).StringVar(&conf.ReplSetName) usernameFile := fmt.Sprintf("/etc/users-secret/%s", envUser) if _, err := os.Stat(usernameFile); err == nil { @@ -93,25 +78,35 @@ func NewConfig(app *kingpin.Application, envUser string, envPassword string) (*C return nil, errors.Wrap(err, "failed to get password") } - ssl := &SSLConfig{} + ssl := &db.SSLConfig{} app.Flag( "ssl", - "enable SSL secured mongodb connection, overridden by env var "+pkg.EnvMongoDBNetSSLEnabled, - ).Envar(pkg.EnvMongoDBNetSSLEnabled).BoolVar(&ssl.Enabled) + "enable SSL secured mongodb connection, overridden by env var "+EnvMongoDBNetSSLEnabled, + ).Envar(EnvMongoDBNetSSLEnabled).BoolVar(&ssl.Enabled) app.Flag( "sslPEMKeyFile", - "path to client SSL Certificate file (including key, in PEM format), overridden by env var "+pkg.EnvMongoDBNetSSLPEMKeyFile, - ).Envar(pkg.EnvMongoDBNetSSLPEMKeyFile).StringVar(&ssl.PEMKeyFile) + "path to client SSL Certificate file (including key, in PEM format), overridden by env var "+EnvMongoDBNetSSLPEMKeyFile, + ).Envar(EnvMongoDBNetSSLPEMKeyFile).StringVar(&ssl.PEMKeyFile) app.Flag( "sslCAFile", - "path to SSL Certificate Authority file (in PEM format), overridden by env var "+pkg.EnvMongoDBNetSSLCAFile, - ).Envar(pkg.EnvMongoDBNetSSLCAFile).StringVar(&ssl.CAFile) + "path to SSL Certificate Authority file (in PEM format), overridden by env var "+EnvMongoDBNetSSLCAFile, + ).Envar(EnvMongoDBNetSSLCAFile).StringVar(&ssl.CAFile) app.Flag( "sslInsecure", - "skip validation of the SSL certificate and hostname, overridden by env var "+pkg.EnvMongoDBNetSSLInsecure, - ).Envar(pkg.EnvMongoDBNetSSLInsecure).BoolVar(&ssl.Insecure) + "skip validation of the SSL certificate and hostname, overridden by env var "+EnvMongoDBNetSSLInsecure, + ).Envar(EnvMongoDBNetSSLInsecure).BoolVar(&ssl.Insecure) conf.SSL = ssl return conf, nil } + +func getDefaultMongoDBAddress() string { + hostname := DefaultMongoDBHost + + mongodbPort := os.Getenv(EnvMongoDBPort) + if mongodbPort != "" { + return hostname + ":" + mongodbPort + } + return hostname + ":" + DefaultMongoDBPort +} diff --git a/healthcheck/pkg/env.go b/cmd/mongodb-healthcheck/tool/env.go similarity index 59% rename from healthcheck/pkg/env.go rename to cmd/mongodb-healthcheck/tool/env.go index 3301c8e522..9cfafc0f14 100644 --- a/healthcheck/pkg/env.go +++ b/cmd/mongodb-healthcheck/tool/env.go @@ -12,40 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pkg +package tool const ( - // general - EnvServiceName = "SERVICE_NAME" EnvMongoDBPort = "MONGODB_PORT" - EnvMongoDBIp = "MONGODB_IP" EnvMongoDBReplset = "MONGODB_REPLSET" - // backup user - EnvMongoDBBackupUser = "MONGODB_BACKUP_USER" - EnvMongoDBBackupPassword = "MONGODB_BACKUP_PASSWORD" - - // clusterAdmin user - EnvMongoDBClusterAdminUser = "MONGODB_CLUSTER_ADMIN_USER" - EnvMongoDBClusterAdminPassword = "MONGODB_CLUSTER_ADMIN_PASSWORD" - // clusterMonitor user EnvMongoDBClusterMonitorUser = "MONGODB_CLUSTER_MONITOR_USER" EnvMongoDBClusterMonitorPassword = "MONGODB_CLUSTER_MONITOR_PASSWORD" - // userAdmin user - EnvMongoDBUserAdminUser = "MONGODB_USER_ADMIN_USER" - EnvMongoDBUserAdminPassword = "MONGODB_USER_ADMIN_PASSWORD" - // mongodb ssl EnvMongoDBNetSSLEnabled = "MONGODB_NET_SSL_ENABLED" EnvMongoDBNetSSLInsecure = "MONGODB_NET_SSL_INSECURE" EnvMongoDBNetSSLPEMKeyFile = "MONGODB_NET_SSL_PEM_KEY_FILE" EnvMongoDBNetSSLCAFile = "MONGODB_NET_SSL_CA_FILE" - - // replset init - EnvInitInitiateDelay = "INIT_INITIATE_DELAY" - EnvInitMaxConnectTries = "INIT_MAX_CONNECT_TRIES" - EnvInitMaxInitReplsetTries = "INIT_MAX_INIT_REPLSET_TRIES" - EnvInitRetrySleep = "INIT_RETRY_SLEEP" ) diff --git a/cmd/mongodb-healthcheck/tool/tool.go b/cmd/mongodb-healthcheck/tool/tool.go new file mode 100644 index 0000000000..bafdb77308 --- /dev/null +++ b/cmd/mongodb-healthcheck/tool/tool.go @@ -0,0 +1,126 @@ +// Copyright 2018 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tool + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/alecthomas/kingpin" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/percona/percona-backup-mongodb/pbm/errors" + + "github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/healthcheck" +) + +// Author is the author used by kingpin +const Author = "Percona LLC." + +// New sets up a new kingpin.Application +func New(help, commit, branch string) *App { + app := kingpin.New(filepath.Base(os.Args[0]), help) + app.Author(Author) + app.Version(fmt.Sprintf( + "%s version %s\ngit commit %s, branch %s\ngo version %s", + app.Name, Version, commit, branch, runtime.Version(), + )) + return &App{app} +} + +type App struct { + *kingpin.Application +} + +func (app *App) Run(ctx context.Context) error { + log := logf.FromContext(ctx) + + k8sCmd := app.Command("k8s", "Performs liveness check for MongoDB on Kubernetes") + livenessCmd := k8sCmd.Command("liveness", "Run a liveness check of MongoDB").Default() + readinessCmd := k8sCmd.Command("readiness", "Run a readiness check of MongoDB") + startupDelaySeconds := livenessCmd.Flag("startupDelaySeconds", "").Default("7200").Uint64() + component := k8sCmd.Flag("component", "").Default("mongod").String() + + _, err := os.Stat("/opt/percona/restore-in-progress") + if err != nil && !os.IsNotExist(err) { + return errors.Wrap(err, "check if restore in progress file exists") + } + if err == nil { + return nil + } + + _, err = os.Stat("/data/db/sleep-forever") + if err != nil && !os.IsNotExist(err) { + return errors.Wrap(err, "check if sleep-forever file exists") + } + if err == nil { + return nil + } + + cnf, err := NewConfig( + app.Application, + EnvMongoDBClusterMonitorUser, + EnvMongoDBClusterMonitorPassword, + ) + if err != nil { + return errors.Wrap(err, "new cfg") + } + + command, err := app.Parse(os.Args[1:]) + if err != nil { + return errors.Wrap(err, "cannot parse command line") + } + + switch command { + case livenessCmd.FullCommand(): + log.Info("Running Kubernetes liveness check for", "component", component) + switch *component { + + case "mongod": + memberState, err := healthcheck.HealthCheckMongodLiveness(ctx, cnf, int64(*startupDelaySeconds)) + if err != nil { + return errors.Wrap(err, "member failed Kubernetes liveness check") + } + log.Info("Member passed Kubernetes liveness check with replication state", "state", memberState) + + case "mongos": + err := healthcheck.HealthCheckMongosLiveness(ctx, cnf) + if err != nil { + return errors.Wrap(err, "member failed Kubernetes liveness check") + } + log.Info("Member passed Kubernetes liveness check") + } + + case readinessCmd.FullCommand(): + log.Info("Running Kubernetes readiness check for component", "component", component) + switch *component { + + case "mongod": + err := healthcheck.MongodReadinessCheck(ctx, cnf.Hosts[0]) + if err != nil { + return errors.Wrap(err, "member failed Kubernetes readiness check") + } + case "mongos": + err := healthcheck.MongosReadinessCheck(ctx, cnf) + if err != nil { + return errors.Wrap(err, "member failed Kubernetes readiness check") + } + } + } + return nil +} diff --git a/healthcheck/version.go b/cmd/mongodb-healthcheck/tool/version.go similarity index 96% rename from healthcheck/version.go rename to cmd/mongodb-healthcheck/tool/version.go index 41fdfe775c..fd4dd86a01 100644 --- a/healthcheck/version.go +++ b/cmd/mongodb-healthcheck/tool/version.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package healthcheck +package tool const ( Version = "0.5.0" diff --git a/e2e-tests/functions b/e2e-tests/functions index 6030169b0e..bd5cddefc8 100755 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -26,7 +26,7 @@ namespace="${test_name}-${RANDOM}" replica_namespace="${test_name}-replica-${RANDOM}" conf_dir=$(realpath $test_dir/../conf || :) src_dir=$(realpath $test_dir/../..) -logs_dir=$(realpath $test_dir/../logs) +logs_dir=$(realpath $test_dir/../logs || :) if [[ ${ENABLE_LOGGING} == "true" ]]; then if [ ! -d "${logs_dir}" ]; then diff --git a/e2e-tests/liveness/run b/e2e-tests/liveness/run index 26fc837dd2..31eaf0e9a4 100755 --- a/e2e-tests/liveness/run +++ b/e2e-tests/liveness/run @@ -18,6 +18,26 @@ spinup_psmdb ${cluster}-rs0 $test_dir/conf/${cluster}-rs0.yml desc 'check if statefulset created with expected config' compare_kubectl "statefulset/${cluster}-rs0" +# There should be some failed liveness checks during the password change. +desc 'change MONGODB_CLUSTER_MONITOR_PASSWORD' +patch_secret "some-users" "MONGODB_CLUSTER_MONITOR_PASSWORD" "$newpassencrypted" +sleep 20 +wait_for_running "${cluster}-rs0" "3" + +desc 'check liveness logs' + +current_log_file=$(kubectl_bin exec -it $cluster-rs0-0 -- bash -c 'ls /data/db/mongod-data/logs' | grep -c mongodb-healthcheck.log) +if [[ $current_log_file != "1" ]]; then + echo "mongodb-healthcheck.log doesn't exist" + exit 1 +fi + +rotated_log_files=$(kubectl_bin exec -it $cluster-rs0-0 -- bash -c 'ls /data/db/mongod-data/logs' | grep -c "mongodb-healthcheck-.*\.log\.gz") +if [[ $rotated_log_files != "1" ]]; then + echo "mongodb-healthcheck.log doesn't exist" + exit 1 +fi + desc 'change liveness config' postfix="-changed" apply_cluster $test_dir/conf/${cluster}-rs0$postfix.yml diff --git a/e2e-tests/pvc-resize/run b/e2e-tests/pvc-resize/run index d820506371..c5d02910d7 100755 --- a/e2e-tests/pvc-resize/run +++ b/e2e-tests/pvc-resize/run @@ -37,10 +37,10 @@ function apply_resourcequota() { echo "Applying resourcequota for default storageclass ${default_sc} with quota ${quota}" - cat ${test_dir}/conf/resourcequota.yml | - sed "s/STORAGECLASS/${default_sc}/" | - sed "s/QUOTA/${quota}/" | - kubectl_bin apply -f - + cat ${test_dir}/conf/resourcequota.yml \ + | sed "s/STORAGECLASS/${default_sc}/" \ + | sed "s/QUOTA/${quota}/" \ + | kubectl_bin apply -f - } function wait_cluster_status() { @@ -96,7 +96,7 @@ for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=mongod -o name); retry=0 echo -n "Waiting for pvc/${pvc} to be resized" until [[ $(kubectl_bin get ${pvc} -o jsonpath={.status.capacity.storage}) == "2Gi" ]]; do - if [[ $retry -ge 60 ]]; then + if [[ $retry -ge 120 ]]; then echo echo "pvc/${pvc} was not resized, max retries exceeded" exit 1 @@ -139,7 +139,7 @@ echo echo -n "Waiting for pvc/mongod-data-some-name-rs0-0 to be resized" until [[ $(kubectl_bin get pvc mongod-data-some-name-rs0-0 -o jsonpath={.status.capacity.storage}) == "3Gi" ]]; do - if [[ $retry -ge 60 ]]; then + if [[ $retry -ge 120 ]]; then echo echo "pvc/mongod-data-some-name-rs0-0 was not resized, max retries exceeded" exit 1 @@ -162,7 +162,7 @@ for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=mongod -o name); retry=0 echo -n "Waiting for pvc/${pvc} to be resized" until [[ $(kubectl_bin get ${pvc} -o jsonpath={.status.capacity.storage}) == "3Gi" ]]; do - if [[ $retry -ge 60 ]]; then + if [[ $retry -ge 120 ]]; then echo echo "pvc/${pvc} was not resized, max retries exceeded" exit 1 diff --git a/go.mod b/go.mod index 7ac1a9d613..ea90a95a3a 100644 --- a/go.mod +++ b/go.mod @@ -117,6 +117,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect diff --git a/go.sum b/go.sum index 4613821b69..9ffc26d64c 100644 --- a/go.sum +++ b/go.sum @@ -701,6 +701,8 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/healthcheck/healthcheck.go b/healthcheck/healthcheck.go deleted file mode 100644 index d838ad545a..0000000000 --- a/healthcheck/healthcheck.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2018 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package healthcheck - -type State int - -const ( - StateOk State = iota - StateFailed -) - -// ExitCode returns an integer reflecting the State, to be used as an exit code -func (s State) ExitCode() int { - return int(s) -} diff --git a/healthcheck/pkg/default.go b/healthcheck/pkg/default.go deleted file mode 100644 index 6a7944f7ef..0000000000 --- a/healthcheck/pkg/default.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2018 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pkg - -const ( - DefaultServiceName = "percona-server-mongodb" -) diff --git a/healthcheck/pkg/pod/k8s/default.go b/healthcheck/pkg/pod/k8s/default.go deleted file mode 100644 index 25bea8fe4f..0000000000 --- a/healthcheck/pkg/pod/k8s/default.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2018 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package k8s - -const ( - DefaultNamespace = "psmdb" - DefaultInitDelay = "0s" -) diff --git a/healthcheck/pkg/pod/k8s/env.go b/healthcheck/pkg/pod/k8s/env.go deleted file mode 100644 index 18e950cc41..0000000000 --- a/healthcheck/pkg/pod/k8s/env.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2018 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package k8s - -const ( - EnvNamespace = "NAMESPACE" -) diff --git a/healthcheck/pkg/pod/k8s/pods.go b/healthcheck/pkg/pod/k8s/pods.go deleted file mode 100644 index 6863dab8ae..0000000000 --- a/healthcheck/pkg/pod/k8s/pods.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2018 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package k8s - -import ( - "errors" - "os" - "sync" - - "github.com/percona/percona-server-mongodb-operator/healthcheck/pkg" - "github.com/percona/percona-server-mongodb-operator/healthcheck/pkg/pod" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" -) - -const ( - EnvKubernetesHost = "KUBERNETES_SERVICE_HOST" - EnvKubernetesPort = "KUBERNETES_SERVICE_PORT" -) - -func getPodReplsetName(pod *corev1.Pod) (string, error) { - for _, container := range pod.Spec.Containers { - for _, env := range container.Env { - if env.Name == pkg.EnvMongoDBReplset { - return env.Value, nil - } - } - } - return "", errors.New("could not find mongodb replset name") -} - -// CustomResourceState represents the state of a single -// Kubernetes CR for PSMDB -type CustomResourceState struct { - Name string - Pods []corev1.Pod - Services []corev1.Service - ServicesExpose bool - Statefulsets []appsv1.StatefulSet -} - -func (cr *CustomResourceState) getServiceFromPod(pod *corev1.Pod) *corev1.Service { - serviceName := pod.Name - for i, svc := range cr.Services { - if svc.Name != serviceName { - continue - } - return &cr.Services[i] - } - return nil -} - -func (cr *CustomResourceState) getStatefulSetFromPod(pod *corev1.Pod) *appsv1.StatefulSet { - replsetName, err := getPodReplsetName(pod) - if err != nil { - return nil - } - setServiceName := cr.Name + "-" + replsetName - for i, statefulset := range cr.Statefulsets { - if statefulset.Spec.ServiceName != setServiceName { - continue - } - return &cr.Statefulsets[i] - } - return nil -} - -func NewPods(namespace string) *Pods { - return &Pods{ - namespace: namespace, - crs: make(map[string]*CustomResourceState), - } -} - -type Pods struct { - sync.Mutex - namespace string - crs map[string]*CustomResourceState -} - -func (p *Pods) Name() string { - return "k8s" -} - -func (p *Pods) URL() string { - host := os.Getenv(EnvKubernetesHost) - port := os.Getenv(EnvKubernetesPort) - if host == "" || port == "" { - return "" - } - return "tcp://" + host + ":" + port -} - -// Delete deletes the state of a single PSMDB CR -func (p *Pods) Delete(crState *CustomResourceState) { - p.Lock() - defer p.Unlock() - delete(p.crs, crState.Name) -} - -// Update updates the state of a single PSMDB CR -func (p *Pods) Update(crState *CustomResourceState) { - p.Lock() - defer p.Unlock() - p.crs[crState.Name] = crState -} - -// Pods returns all available pod names -func (p *Pods) Pods() ([]string, error) { - p.Lock() - defer p.Unlock() - - pods := make([]string, 0) - for _, cr := range p.crs { - for _, pod := range cr.Pods { - if pod.Status.Phase != corev1.PodRunning && pod.Status.Phase != corev1.PodPending { - continue - } - pods = append(pods, pod.Name) - } - } - return pods, nil -} - -// GetTasks returns tasks for a single pod by name -func (p *Pods) GetTasks(podName string) ([]pod.Task, error) { - p.Lock() - defer p.Unlock() - - tasks := make([]pod.Task, 0) - for _, cr := range p.crs { - for i := range cr.Pods { - pod := &cr.Pods[i] - if pod.Name != podName { - continue - } - tasks = append(tasks, NewTask(p.namespace, cr, pod)) - } - } - return tasks, nil -} diff --git a/healthcheck/pkg/pod/k8s/task.go b/healthcheck/pkg/pod/k8s/task.go deleted file mode 100644 index ebd775afb0..0000000000 --- a/healthcheck/pkg/pod/k8s/task.go +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright 2018 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package k8s - -import ( - "errors" - "fmt" - "github.com/percona/percona-server-mongodb-operator/healthcheck/pkg/db" - "github.com/percona/percona-server-mongodb-operator/healthcheck/pkg/pod" - corev1 "k8s.io/api/core/v1" - "strings" -) - -const ( - mongodContainerName = "mongod" - mongodArbiterContainerName = "mongod-arbiter" - mongodBackupContainerName = "mongod-backup" - mongosContainerName = "mongos" - mongodbPortName = "mongodb" - clusterServiceDNSSuffix = "svc.cluster.local" -) - -func GetMongoHost(pod, service, replset, namespace string) string { - return strings.Join([]string{pod, service + "-" + replset, namespace, clusterServiceDNSSuffix}, ".") -} - -type TaskState struct { - status corev1.PodStatus -} - -func NewTaskState(status corev1.PodStatus) *TaskState { - return &TaskState{status} -} - -func (ts TaskState) String() string { - return strings.ToUpper(string(ts.status.Phase)) -} - -type Task struct { - namespace string - pod *corev1.Pod - cr *CustomResourceState -} - -func NewTask(namespace string, cr *CustomResourceState, pod *corev1.Pod) *Task { - return &Task{ - namespace: namespace, - pod: pod, - cr: cr, - } -} - -func (t *Task) Service() string { - return t.cr.Name -} - -func (t *Task) State() pod.TaskState { - return NewTaskState(t.pod.Status) -} - -func (t *Task) HasState() bool { - return t.pod.Status.Phase != "" -} - -func (t *Task) Name() string { - return t.pod.Name -} - -func (t *Task) IsRunning() bool { - if t.pod.Status.Phase != corev1.PodRunning { - return false - } - for _, container := range t.pod.Status.ContainerStatuses { - if container.State.Running == nil { - return false - } - } - return true -} - -func (t *Task) IsUpdating() bool { - statefulset := t.cr.getStatefulSetFromPod(t.pod) - if statefulset == nil { - return false - } - status := statefulset.Status - if status.CurrentRevision != status.UpdateRevision { - return true - } - return status.ReadyReplicas != status.CurrentReplicas -} - -func (t *Task) IsTaskType(taskType pod.TaskType) bool { - var containerName string - switch taskType { - case pod.TaskTypeMongod: - containerName = mongodContainerName - case pod.TaskTypeMongodBackup: - containerName = mongodBackupContainerName - case pod.TaskTypeMongos: - containerName = mongosContainerName - case pod.TaskTypeArbiter: - containerName = mongodArbiterContainerName - default: - return false - } - for _, container := range t.pod.Spec.Containers { - if container.Name == containerName { - return true - } - } - return false -} - -func (t *Task) GetMongoAddr() (*db.Addr, error) { - if t.cr.ServicesExpose { - service := t.cr.getServiceFromPod(t.pod) - if service != nil { - addr, err := t.getServiceAddr() - if err != nil { - return nil, err - } - return addr, nil - } - return nil, errors.New("service not ready yet") - } - - for _, container := range t.pod.Spec.Containers { - for _, port := range container.Ports { - if port.Name != mongodbPortName { - continue - } - replset, err := t.GetMongoReplsetName() - if err != nil { - return nil, err - } - addr := &db.Addr{ - Host: GetMongoHost(t.pod.Name, t.cr.Name, replset, t.namespace), - Port: int(port.HostPort), - } - if addr.Port == 0 { - addr.Port = int(port.ContainerPort) - } - return addr, nil - } - } - return nil, errors.New("could not find mongodb address") -} - -func (t *Task) GetMongoReplsetName() (string, error) { - return getPodReplsetName(t.pod) -} - -func (t *Task) getServiceAddr() (*db.Addr, error) { - addr := &db.Addr{} - service := t.cr.getServiceFromPod(t.pod) - - switch service.Spec.Type { - case corev1.ServiceTypeClusterIP: - addr.Host = service.Spec.ClusterIP - for _, p := range service.Spec.Ports { - if p.Name != mongodbPortName { - continue - } - addr.Port = int(p.Port) - } - return addr, nil - - case corev1.ServiceTypeLoadBalancer: - host, err := getIngressPoint(*service) - if err != nil { - return nil, fmt.Errorf("can't get service %s address: %v", service.Name, err) - } - addr.Host = host - - for _, p := range service.Spec.Ports { - if p.Name != mongodbPortName { - continue - } - addr.Port = int(p.Port) - } - return addr, nil - - case corev1.ServiceTypeNodePort: - addr.Host = t.pod.Status.HostIP - for _, p := range service.Spec.Ports { - if p.Name != mongodbPortName { - continue - } - addr.Port = int(p.NodePort) - } - } - - return nil, fmt.Errorf("could not find mongodb service address") -} - -func getIngressPoint(service corev1.Service) (string, error) { - if service.Status.LoadBalancer.Ingress == nil || len(service.Status.LoadBalancer.Ingress) == 0 { - return "", fmt.Errorf("can't get service %s ingress", service.Name) - } - ip := service.Status.LoadBalancer.Ingress[0].IP - hostname := service.Status.LoadBalancer.Ingress[0].Hostname - - if ip != "" { - return ip, nil - } - - if hostname != "" { - return hostname, nil - } - - return "", fmt.Errorf("can't get service %s ingress", service.Name) -} diff --git a/healthcheck/pkg/pod/mocks/Source.go b/healthcheck/pkg/pod/mocks/Source.go deleted file mode 100644 index c5fb5b6a29..0000000000 --- a/healthcheck/pkg/pod/mocks/Source.go +++ /dev/null @@ -1,84 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. -package mocks - -import pod "github.com/percona/percona-server-mongodb-operator/healthcheck/pkg/pod" -import mock "github.com/stretchr/testify/mock" - -// Source is an autogenerated mock type for the Source type -type Source struct { - mock.Mock -} - -// GetTasks provides a mock function with given fields: podName -func (_m *Source) GetTasks(podName string) ([]pod.Task, error) { - ret := _m.Called(podName) - - var r0 []pod.Task - if rf, ok := ret.Get(0).(func(string) []pod.Task); ok { - r0 = rf(podName) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]pod.Task) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(podName) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Name provides a mock function with given fields: -func (_m *Source) Name() string { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} - -// Pods provides a mock function with given fields: -func (_m *Source) Pods() ([]string, error) { - ret := _m.Called() - - var r0 []string - if rf, ok := ret.Get(0).(func() []string); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// URL provides a mock function with given fields: -func (_m *Source) URL() string { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} diff --git a/healthcheck/pkg/pod/mocks/Task.go b/healthcheck/pkg/pod/mocks/Task.go deleted file mode 100644 index 32f4c1070d..0000000000 --- a/healthcheck/pkg/pod/mocks/Task.go +++ /dev/null @@ -1,155 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. -package mocks - -import db "github.com/percona/percona-server-mongodb-operator/healthcheck/pkg/db" -import mock "github.com/stretchr/testify/mock" -import pod "github.com/percona/percona-server-mongodb-operator/healthcheck/pkg/pod" - -// Task is an autogenerated mock type for the Task type -type Task struct { - mock.Mock -} - -// GetMongoAddr provides a mock function with given fields: -func (_m *Task) GetMongoAddr() (*db.Addr, error) { - ret := _m.Called() - - var r0 *db.Addr - if rf, ok := ret.Get(0).(func() *db.Addr); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*db.Addr) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetMongoReplsetName provides a mock function with given fields: -func (_m *Task) GetMongoReplsetName() (string, error) { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// HasState provides a mock function with given fields: -func (_m *Task) HasState() bool { - ret := _m.Called() - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// IsRunning provides a mock function with given fields: -func (_m *Task) IsRunning() bool { - ret := _m.Called() - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// IsTaskType provides a mock function with given fields: taskType -func (_m *Task) IsTaskType(taskType pod.TaskType) bool { - ret := _m.Called(taskType) - - var r0 bool - if rf, ok := ret.Get(0).(func(pod.TaskType) bool); ok { - r0 = rf(taskType) - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// IsUpdating provides a mock function with given fields: -func (_m *Task) IsUpdating() bool { - ret := _m.Called() - - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// Name provides a mock function with given fields: -func (_m *Task) Name() string { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} - -// Service provides a mock function with given fields: -func (_m *Task) Service() string { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} - -// State provides a mock function with given fields: -func (_m *Task) State() pod.TaskState { - ret := _m.Called() - - var r0 pod.TaskState - if rf, ok := ret.Get(0).(func() pod.TaskState); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(pod.TaskState) - } - } - - return r0 -} diff --git a/healthcheck/pkg/pod/mocks/TaskState.go b/healthcheck/pkg/pod/mocks/TaskState.go deleted file mode 100644 index 8b14c2336b..0000000000 --- a/healthcheck/pkg/pod/mocks/TaskState.go +++ /dev/null @@ -1,23 +0,0 @@ -// Code generated by mockery v1.0.0. DO NOT EDIT. -package mocks - -import mock "github.com/stretchr/testify/mock" - -// TaskState is an autogenerated mock type for the TaskState type -type TaskState struct { - mock.Mock -} - -// String provides a mock function with given fields: -func (_m *TaskState) String() string { - ret := _m.Called() - - var r0 string - if rf, ok := ret.Get(0).(func() string); ok { - r0 = rf() - } else { - r0 = ret.Get(0).(string) - } - - return r0 -} diff --git a/healthcheck/pkg/pod/pod.go b/healthcheck/pkg/pod/pod.go deleted file mode 100644 index 3b2f919d13..0000000000 --- a/healthcheck/pkg/pod/pod.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2018 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pod - -import "sync" - -type Pods struct { - sync.Mutex - pods []string -} - -func NewPods() *Pods { - return &Pods{} -} - -func (p *Pods) Get() []string { - p.Lock() - defer p.Unlock() - return p.pods -} - -func (p *Pods) Set(pods []string) { - p.Lock() - defer p.Unlock() - p.pods = pods -} - -func (p *Pods) Has(name string) bool { - p.Lock() - defer p.Unlock() - for _, podName := range p.pods { - if name == podName { - return true - } - } - return false -} - -type Source interface { - Name() string - URL() string - Pods() ([]string, error) - GetTasks(podName string) ([]Task, error) -} diff --git a/healthcheck/pkg/pod/task.go b/healthcheck/pkg/pod/task.go deleted file mode 100644 index e1c8eccf2c..0000000000 --- a/healthcheck/pkg/pod/task.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2018 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pod - -import ( - "github.com/percona/percona-server-mongodb-operator/healthcheck/pkg/db" -) - -type TaskType string - -var ( - TaskTypeMongod TaskType = "mongod" - TaskTypeMongodBackup TaskType = "mongod-backup" - TaskTypeArbiter TaskType = "arbiter" - TaskTypeConfigSvr TaskType = "configsvr" - TaskTypeMongos TaskType = "mongos" -) - -func (t TaskType) String() string { - return string(t) -} - -type TaskState interface { - String() string -} - -type Task interface { - Name() string - Service() string - State() TaskState - HasState() bool - IsRunning() bool - IsUpdating() bool - IsTaskType(taskType TaskType) bool - GetMongoAddr() (*db.Addr, error) - GetMongoReplsetName() (string, error) -} diff --git a/healthcheck/tools/testutils/testing.go b/healthcheck/tools/testutils/testing.go deleted file mode 100644 index ff276281b7..0000000000 --- a/healthcheck/tools/testutils/testing.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2018 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testutils - -import ( - "errors" - "os" - "testing" - "time" - - "gopkg.in/mgo.v2" -) - -const ( - envEnableDBTests = "ENABLE_MONGODB_TESTS" - envMongoDBReplsetName = "TEST_RS_NAME" - envMongoDBPrimaryPort = "TEST_PRIMARY_PORT" - envMongoDBSecondary1Port = "TEST_SECONDARY1_PORT" - envMongoDBSecondary2Port = "TEST_SECONDARY2_PORT" - envMongoDBAdminUser = "TEST_ADMIN_USER" - envMongoDBAdminPassword = "TEST_ADMIN_PASSWORD" -) - -var ( - enableDBTests = os.Getenv(envEnableDBTests) - MongodbReplsetName = os.Getenv(envMongoDBReplsetName) - MongodbHost = "127.0.0.1" - MongodbHostname = "localhost" - MongodbPrimaryPort = os.Getenv(envMongoDBPrimaryPort) - MongodbSecondary1Port = os.Getenv(envMongoDBSecondary1Port) - MongodbSecondary2Port = os.Getenv(envMongoDBSecondary2Port) - MongodbAdminUser = os.Getenv(envMongoDBAdminUser) - MongodbAdminPassword = os.Getenv(envMongoDBAdminPassword) - MongodbTimeout = time.Duration(10) * time.Second -) - -// Enabled returns a boolean reflecting whether testing against Mongodb should occur -func Enabled() bool { - return enableDBTests == "true" -} - -// getDialInfo returns a *mgo.DialInfo configured for testing -func getDialInfo(host, port string) (*mgo.DialInfo, error) { - if !Enabled() { - return nil, nil - } - if port == "" { - return nil, errors.New("Port argument is not set") - } else if host == "" { - return nil, errors.New("Host argument is not set") - } else if MongodbReplsetName == "" { - return nil, errors.New("Replica set name env var is not set") - } else if MongodbAdminUser == "" { - return nil, errors.New("Admin user env var is not set") - } else if MongodbAdminPassword == "" { - return nil, errors.New("Admin password env var is not set") - } - return &mgo.DialInfo{ - Addrs: []string{host + ":" + port}, - Direct: true, - Timeout: MongodbTimeout, - Username: MongodbAdminUser, - Password: MongodbAdminPassword, - ReplicaSetName: MongodbReplsetName, - }, nil -} - -// GetSession returns a *mgo.Session configured for testing against a MongoDB Primary -func GetSession(port string) (*mgo.Session, error) { - dialInfo, err := getDialInfo(MongodbHost, port) - if err != nil { - return nil, err - } - session, err := mgo.DialWithInfo(dialInfo) - if err != nil { - return nil, err - } - return session, err -} - -// DoSkipTest handles the conditional skipping of tests, based on the output of .Enabled() -func DoSkipTest(t *testing.T) { - if !Enabled() { - t.Skipf("Skipping test, env var %s is not 'true'", envEnableDBTests) - } -} diff --git a/healthcheck/tools/tool/tool.go b/healthcheck/tools/tool/tool.go deleted file mode 100644 index ba500e1901..0000000000 --- a/healthcheck/tools/tool/tool.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2018 Percona LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tool - -import ( - "fmt" - "os" - "path/filepath" - "runtime" - - "github.com/alecthomas/kingpin" - tools "github.com/percona/percona-server-mongodb-operator/healthcheck" -) - -// Author is the author used by kingpin -const Author = "Percona LLC." - -// New sets up a new kingpin.Application -func New(help, commit, branch string) *kingpin.Application { - app := kingpin.New(filepath.Base(os.Args[0]), help) - app.Author(Author) - app.Version(fmt.Sprintf( - "%s version %s\ngit commit %s, branch %s\ngo version %s", - app.Name, tools.Version, commit, branch, runtime.Version(), - )) - return app -} diff --git a/pkg/psmdb/mongo/mongo.go b/pkg/psmdb/mongo/mongo.go index d3ff87bf17..5f988d1f78 100644 --- a/pkg/psmdb/mongo/mongo.go +++ b/pkg/psmdb/mongo/mongo.go @@ -79,16 +79,27 @@ func Dial(conf *Config) (Client, error) { ctx, connectcancel := context.WithTimeout(context.Background(), 10*time.Second) defer connectcancel() + journal := true + wc := writeconcern.Majority() + wc.Journal = &journal opts := options.Client(). SetHosts(conf.Hosts). - SetReplicaSet(conf.ReplSetName). - SetAuth(options.Credential{ + SetWriteConcern(wc). + SetReadPreference(readpref.Primary()). + SetTLSConfig(conf.TLSConf). + SetDirect(conf.Direct). + SetConnectTimeout(10 * time.Second). + SetServerSelectionTimeout(10 * time.Second) + + if conf.ReplSetName != "" { + opts.SetReplicaSet(conf.ReplSetName) + } + if conf.Username != "" || conf.Password != "" { + opts.SetAuth(options.Credential{ Password: conf.Password, Username: conf.Username, - }). - SetWriteConcern(writeconcern.New(writeconcern.WMajority(), writeconcern.J(true))). - SetReadPreference(readpref.Primary()).SetTLSConfig(conf.TLSConf). - SetDirect(conf.Direct) + }) + } client, err := mongo.Connect(ctx, opts) if err != nil {