Skip to content

Commit

Permalink
feat: mongodb multi version support (#4913)
Browse files Browse the repository at this point in the history
  • Loading branch information
xuriwuyun authored Aug 31, 2023
1 parent 5de9d38 commit 01b3907
Show file tree
Hide file tree
Showing 12 changed files with 228 additions and 21 deletions.
6 changes: 4 additions & 2 deletions cmd/probe/internal/component/mongodb/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package mongodb

import (
"context"
"encoding/json"
"fmt"
"math/rand"
"strings"
Expand Down Expand Up @@ -127,7 +128,8 @@ func (mgr *Manager) InitiateReplSet(ctx context.Context, cluster *dcs.Cluster) e
}
defer client.Disconnect(context.TODO()) //nolint:errcheck

mgr.Logger.Infof("Initial Replset Config: %v", config)
configJSON, _ := json.Marshal(config)
mgr.Logger.Infof("Initial Replset Config: %s", string(configJSON))
response := client.Database("admin").RunCommand(ctx, bson.M{"replSetInitiate": config})
if response.Err() != nil {
return response.Err()
Expand Down Expand Up @@ -657,7 +659,7 @@ func (mgr *Manager) HasOtherHealthyMembers(ctx context.Context, cluster *dcs.Clu
}

for _, member := range rsStatus.Members {
if member.State != 1 {
if member.Health != 1 {
continue
}
memberName := strings.Split(member.Name, ".")[0]
Expand Down
2 changes: 1 addition & 1 deletion cmd/probe/internal/component/mongodb/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ type RSConfig struct {
Members ConfigMembers `bson:"members" json:"members"`
Configsvr bool `bson:"configsvr,omitempty" json:"configsvr,omitempty"`
ProtocolVersion int `bson:"protocolVersion,omitempty" json:"protocolVersion,omitempty"`
Settings Settings `bson:"settings,omitempty" json:"settings,omitempty"`
Settings Settings `bson:"-" json:"settings,omitempty"`
WriteConcernMajorityJournalDefault bool `bson:"writeConcernMajorityJournalDefault,omitempty" json:"writeConcernMajorityJournalDefault,omitempty"`
}

Expand Down
2 changes: 1 addition & 1 deletion deploy/mongodb/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ type: application

version: 0.7.0-alpha.0

appVersion: "5.0.14"
appVersion: "5.0"

home: https://www.mongodb.com
icon: https://bitnami.com/assets/stacks/mongodb/img/mongodb-stack-220x234.png
Expand Down
3 changes: 2 additions & 1 deletion deploy/mongodb/dataprotection/backup-info-collector.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
function get_current_time() {
curr_time=$(mongosh -u ${DB_USER} -p ${DB_PASSWORD} --port 27017 --host ${DB_HOST} --authenticationDatabase admin --eval 'db.isMaster().lastWrite.lastWriteDate.getTime()/1000' --quiet)
CLIENT=`which mongosh&&echo mongosh||echo mongo`
curr_time=$(${CLIENT} -u ${DB_USER} -p ${DB_PASSWORD} --port 27017 --host ${DB_HOST} --authenticationDatabase admin --eval 'db.isMaster().lastWrite.lastWriteDate.getTime()/1000' --quiet)
curr_time=$(date -d "@${curr_time}" -u '+%Y-%m-%dT%H:%M:%SZ')
echo $curr_time
}
Expand Down
3 changes: 2 additions & 1 deletion deploy/mongodb/dataprotection/pitr-backup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ check_oplog_push_process(){
echo $errorLog && exit 1
fi
# check role of the connected mongodb
isPrimary=$(mongosh -u ${DB_USER} -p ${DB_PASSWORD} --port 27017 --host ${DB_HOST} --authenticationDatabase admin --eval 'db.isMaster().ismaster' --quiet)
CLIENT=`which mongosh&&echo mongosh||echo mongo`
isPrimary=$(${CLIENT} -u ${DB_USER} -p ${DB_PASSWORD} --port 27017 --host ${DB_HOST} --authenticationDatabase admin --eval 'db.isMaster().ismaster' --quiet)
if [ "${isPrimary}" != "true" ]; then
echo "isPrimary: ${isPrimary}"
retryTimes=$(($retryTimes+1))
Expand Down
9 changes: 5 additions & 4 deletions deploy/mongodb/scripts/replicaset-restore.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@ RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-");
RPL_SET_NAME=${RPL_SET_NAME%-};
MODE=$1
mongod $MODE --bind_ip_all --port $PORT --dbpath $MONGODB_ROOT/db --directoryperdb --logpath $MONGODB_ROOT/logs/mongodb.log --logappend --pidfilepath $MONGODB_ROOT/tmp/mongodb.pid&
until mongosh --quiet --port $PORT --host $host --eval "print('peer is ready')"; do sleep 1; done
export CLIENT=`which mongosh&&echo mongosh||echo mongo`
until $CLIENT --quiet --port $PORT --host $host --eval "print('peer is ready')"; do sleep 1; done
PID=`cat $MONGODB_ROOT/tmp/mongodb.pid`

mongosh --quiet --port $PORT local --eval "db.system.replset.deleteOne({})"
mongosh --quiet --port $PORT local --eval "db.system.replset.find()"
mongosh --quiet --port $PORT admin --eval 'db.dropUser("root", {w: "majority", wtimeout: 4000})' || true
$CLIENT --quiet --port $PORT local --eval "db.system.replset.deleteOne({})"
$CLIENT --quiet --port $PORT local --eval "db.system.replset.find()"
$CLIENT --quiet --port $PORT admin --eval 'db.dropUser("root", {w: "majority", wtimeout: 4000})' || true
kill $PID
wait $PID
9 changes: 5 additions & 4 deletions deploy/mongodb/scripts/replicaset-setup.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,16 @@ mkdir -p $MONGODB_ROOT/tmp

BACKUPFILE=$MONGODB_ROOT/db/mongodb.backup
PORT_FOR_RESTORE=27027
CLIENT=`which mongosh&&echo mongosh||echo mongo`
if [ -f $BACKUPFILE ]
then
mongod --bind_ip_all --port $PORT_FOR_RESTORE --dbpath $MONGODB_ROOT/db --directoryperdb --logpath $MONGODB_ROOT/logs/mongodb.log --logappend --pidfilepath $MONGODB_ROOT/tmp/mongodb.pid&
until mongosh --quiet --port $PORT_FOR_RESTORE --host $host --eval "print('restore process is ready')"; do sleep 1; done
until $CLIENT --quiet --port $PORT_FOR_RESTORE --host $host --eval "print('restore process is ready')"; do sleep 1; done
PID=`cat $MONGODB_ROOT/tmp/mongodb.pid`

mongosh --quiet --port $PORT_FOR_RESTORE local --eval "db.system.replset.deleteOne({})"
mongosh --quiet --port $PORT_FOR_RESTORE local --eval "db.system.replset.find()"
mongosh --quiet --port $PORT_FOR_RESTORE admin --eval 'db.dropUser("root", {w: "majority", wtimeout: 4000})' || true
$CLIENT --quiet --port $PORT_FOR_RESTORE local --eval "db.system.replset.deleteOne({})"
$CLIENT --quiet --port $PORT_FOR_RESTORE local --eval "db.system.replset.find()"
$CLIENT --quiet --port $PORT_FOR_RESTORE admin --eval 'db.dropUser("root", {w: "majority", wtimeout: 4000})' || true
kill $PID
wait $PID
rm $BACKUPFILE
Expand Down
96 changes: 95 additions & 1 deletion deploy/mongodb/templates/clusterversion.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ apiVersion: apps.kubeblocks.io/v1alpha1
kind: ClusterVersion
metadata:
name: mongodb-{{ default .Chart.AppVersion .Values.clusterVersionOverride }}
annotations:
kubeblocks.io/is-default-cluster-version: "true"
labels:
{{- include "mongodb.labels" . | nindent 4 }}
spec:
Expand All @@ -18,4 +20,96 @@ spec:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
switchoverSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}

---
apiVersion: apps.kubeblocks.io/v1alpha1
kind: ClusterVersion
metadata:
name: mongodb-4.0
labels:
{{- include "mongodb.labels" . | nindent 4 }}
spec:
clusterDefinitionRef: mongodb
componentVersions:
- componentDefRef: mongodb
versionsContext:
containers:
- name: mongodb
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:4.0
imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }}
systemAccountSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:4.0
switchoverSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:4.0

---
apiVersion: apps.kubeblocks.io/v1alpha1
kind: ClusterVersion
metadata:
name: mongodb-4.2
labels:
{{- include "mongodb.labels" . | nindent 4 }}
spec:
clusterDefinitionRef: mongodb
componentVersions:
- componentDefRef: mongodb
versionsContext:
containers:
- name: mongodb
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:4.2
imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }}
systemAccountSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:4.2
switchoverSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:4.2

---
apiVersion: apps.kubeblocks.io/v1alpha1
kind: ClusterVersion
metadata:
name: mongodb-4.4
labels:
{{- include "mongodb.labels" . | nindent 4 }}
spec:
clusterDefinitionRef: mongodb
componentVersions:
- componentDefRef: mongodb
versionsContext:
containers:
- name: mongodb
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:4.4
imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }}
systemAccountSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:4.4
switchoverSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:4.4

---
apiVersion: apps.kubeblocks.io/v1alpha1
kind: ClusterVersion
metadata:
name: mongodb-6.0
labels:
{{- include "mongodb.labels" . | nindent 4 }}
spec:
clusterDefinitionRef: mongodb
componentVersions:
- componentDefRef: mongodb
versionsContext:
containers:
- name: mongodb
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:6.0
imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }}
systemAccountSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:6.0
switchoverSpec:
cmdExecutorConfig:
image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:6.0
11 changes: 7 additions & 4 deletions deploy/mongodb/templates/scriptstemplate.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ data:
switchover-check-role.sh: |-
#!/bin/sh
check_role() {
local role=$(mongosh --quiet --eval "rs.isMaster().ismaster" "$1" --username "$USERNAME" --password "$PASSWORD")
CLIENT=`which mongosh&&echo mongosh||echo mongo`
local role=$($CLIENT --quiet --eval "rs.isMaster().ismaster" "$1" --username "$USERNAME" --password "$PASSWORD")
if [ "${role}" = "true" ]; then
echo "Primary"
else
Expand Down Expand Up @@ -51,6 +52,7 @@ data:
CANDIDATE_URI="mongodb://$KB_SWITCHOVER_CANDIDATE_FQDN:27017"
USERNAME=$MONGODB_ROOT_USER
PASSWORD=$MONGODB_ROOT_PASSWORD
CLIENT=`which mongosh&&echo mongosh||echo mongo`
CANDIDATE_HOST="$KB_SWITCHOVER_CANDIDATE_FQDN.$KB_NAMESPACE.svc.cluster.local:27017"
. /scripts/switchover-check-role.sh
. /scripts/switchover-verify.sh
Expand All @@ -62,7 +64,7 @@ data:
exit 1
fi
echo "Switchover to new primary: $CANDIDATE_HOST"
mongosh --quiet --eval "conf=rs.config();conf.members.forEach(member => member.priority = 1);const candidateHost = '$CANDIDATE_HOST';const member = conf.members.find(member => member.host === candidateHost);if (member) {member.priority = 2;};rs.reconfig(conf)" "$URI" --username "$USERNAME" --password "$PASSWORD"
$CLIENT --quiet --eval "conf=rs.config();conf.members.forEach(member => member.priority = 1);const candidateHost = '$CANDIDATE_HOST';const member = conf.members.find(member => member.host === candidateHost);if (member) {member.priority = 2;};rs.reconfig(conf)" "$URI" --username "$USERNAME" --password "$PASSWORD"
echo "Checking candidate instance role after switchover..."
verify
}
Expand All @@ -75,6 +77,7 @@ data:
URI="mongodb://$KB_CONSENSUS_LEADER_POD_FQDN:27017"
USERNAME=$MONGODB_ROOT_USER
PASSWORD=$MONGODB_ROOT_PASSWORD
CLIENT=`which mongosh&&echo mongosh||echo mongo`
OLD_LEADER_HOST="$KB_CONSENSUS_LEADER_POD_FQDN.$KB_NAMESPACE.svc.cluster.local:27017"
. /scripts/switchover-check-role.sh
. /scripts/switchover-verify.sh
Expand All @@ -86,14 +89,14 @@ data:
exit 1
fi
echo "Switchover without candidate, try to select a new primary randomly ..."
local CANDIDATE_HOST=$(mongosh --quiet --eval "conf=rs.config();const candidateHost = '$OLD_LEADER_HOST';const member=conf.members.find(member => member.host !== candidateHost);if (member) {console.log(member.host)}" "$URI" --username "$USERNAME" --password "$PASSWORD")
local CANDIDATE_HOST=$($CLIENT --quiet --eval "conf=rs.config();const candidateHost = '$OLD_LEADER_HOST';const member=conf.members.find(member => member.host !== candidateHost);if (member) {console.log(member.host)}" "$URI" --username "$USERNAME" --password "$PASSWORD")
local CANDIDATE_URI="mongodb://$CANDIDATE_HOST"
if [ -z "$CANDIDATE_HOST" ]; then
echo "Failed to select a new candidate primary, exit"
exit 1
fi
echo "Switchover to new primary: $CANDIDATE_HOST"
mongosh --quiet --eval "conf=rs.config();conf.members.forEach(member => member.priority = 1);const candidateHost = '$CANDIDATE_HOST';const member = conf.members.find(member => member.host === candidateHost);if (member) {member.priority = 2;};rs.reconfig(conf)" "$URI" --username "$USERNAME" --password "$PASSWORD"
$CLIENT --quiet --eval "conf=rs.config();conf.members.forEach(member => member.priority = 1);const candidateHost = '$CANDIDATE_HOST';const member = conf.members.find(member => member.host === candidateHost);if (member) {member.priority = 2;};rs.reconfig(conf)" "$URI" --username "$USERNAME" --password "$PASSWORD"
echo "Checking candidate instance role after switchover..."
verify
}
Expand Down
104 changes: 104 additions & 0 deletions docker/Dockerfile-tools.bk
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
# Build the kubeblocks tools binaries
# includes kbcli, kubectl, and manager tools.

## docker buildx build injected build-args:
#BUILDPLATFORM — matches the current machine. (e.g. linux/amd64)
#BUILDOS — os component of BUILDPLATFORM, e.g. linux
#BUILDARCH — e.g. amd64, arm64, riscv64
#BUILDVARIANT — used to set build ARM variant, e.g. v7
#TARGETPLATFORM — The value set with --platform flag on build
#TARGETOS - OS component from --platform, e.g. linux
#TARGETARCH - Architecture from --platform, e.g. arm64
#TARGETVARIANT - used to set target ARM variant, e.g. v7

ARG GO_VERSION=1.20

FROM --platform=${BUILDPLATFORM} registry.cn-hangzhou.aliyuncs.com/xuriwuyun/golang:${GO_VERSION} as builder
ARG TARGETOS
ARG TARGETARCH
ARG GOPROXY
ARG GOPROXY=https://goproxy.cn
ARG LD_FLAGS="-s -w"

ENV GONOPROXY=github.com/apecloud
ENV GONOSUMDB=github.com/apecloud
ENV GOPRIVATE=github.com/apecloud
ENV GOPROXY=${GOPROXY}

WORKDIR /src

# Copy the Go Modules manifests
#COPY go.mod go.mod
#COPY go.sum go.sum
# cache deps before building and copying source so that we don't need to re-download as much
# and so that source changes don't invalidate our downloaded layer
# RUN go mod download

# Copy the go source
#COPY internal/ internal/
#COPY controllers/ controllers/
#COPY cmd/reloader/ cmd/reloader/
#COPY cmd/probe/ cmd/probe/
#COPY externalapis/ externalapis/
#COPY version/ version/
#COPY cmd/cli/ cmd/cli/
#COPY apis/ apis/
#COPY test/testdata/testdata.go test/testdata/testdata.go
RUN --mount=type=bind,target=. \
--mount=type=cache,target=/go/pkg/mod \
go mod download
# Build
RUN --mount=type=bind,target=. \
--mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg/mod \
go env && \
CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/killer cmd/reloader/container_killer/killer.go

RUN --mount=type=bind,target=. \
--mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg/mod \
CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/reloader cmd/reloader/main.go

RUN --mount=type=bind,target=. \
--mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg/mod \
CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/config_render cmd/reloader/template/*.go

RUN --mount=type=bind,target=. \
--mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg/mod \
CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/probe cmd/probe/main.go

RUN --mount=type=bind,target=. \
--mount=type=cache,target=/root/.cache/go-build \
--mount=type=cache,target=/go/pkg/mod \
CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/sqlctl cmd/probe/sqlctl/main.go

#RUN --mount=type=bind,target=. \
# --mount=type=cache,target=/root/.cache/go-build \
# --mount=type=cache,target=/go/pkg/mod \
# CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -tags="containers_image_openpgp" -a -o /out/kbcli cmd/cli/main.go

# Use alpine with tag 20230329 is corresponding to "edge" tag (latest release to date is 3.18) as of 20230625
FROM docker.io/alpine:edge as dist
ARG APK_MIRROR

# install tools via apk
# ENV APK_MIRROR=${APK_MIRROR}
# RUN if [ -n "${APK_MIRROR}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${APK_MIRROR}/g" /etc/apk/repositories; fi
# RUN apk add --no-cache curl kubectl helm --allow-untrusted \
# && rm -rf /var/cache/apk/*

# copy kubeblocks tools
COPY config/probe config/probe
COPY --from=builder /out/killer /bin
COPY --from=builder /out/reloader /bin
COPY --from=builder /out/config_render /bin
COPY --from=builder /out/probe /bin
COPY --from=builder /out/sqlctl /bin
#COPY --from=builder /out/kbcli /bin

# mkdir kbcli config dir and helm cache dir.
# RUN mkdir /.kbcli && chown -R 65532:65532 /.kbcli \
# && mkdir /.cache && chown -R 65532:65532 /.cache
USER 65532:65532
2 changes: 1 addition & 1 deletion internal/controller/builder/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -681,7 +681,7 @@ func buildActionFromCharacterType(characterType string, isConsensus bool) []work
{
Image: "registry.cn-hangzhou.aliyuncs.com/apecloud/mongo:5.0.14",
Command: []string{
"Status=$(mongosh -u $KB_RSM_USERNAME -p $KB_RSM_PASSWORD 127.0.0.1:27017 --quiet --eval \"JSON.stringify(rs.status())\") &&",
"Status=$(export CLIENT=`which mongosh&&echo mongosh||echo mongo`; $CLIENT -u $KB_RSM_USERNAME -p $KB_RSM_PASSWORD 127.0.0.1:27017 --quiet --eval \"JSON.stringify(rs.status())\") &&",
"MyState=$(echo $Status | jq '.myState') &&",
"echo $Status | jq \".members[] | select(.state == ($MyState | tonumber)) | .stateStr\" |tr '[:upper:]' '[:lower:]' | xargs echo -n",
},
Expand Down
2 changes: 1 addition & 1 deletion internal/sqlchannel/engine/mongodb.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func (r mongodb) ConnectCommand(connectInfo *AuthInfo) []string {
userPass = connectInfo.UserPasswd
}

mongodbCmd := []string{fmt.Sprintf("%s mongodb://%s:%s@$KB_POD_FQDN:27017/admin?replicaSet=$KB_CLUSTER_COMP_NAME", r.info.Client, userName, userPass)}
mongodbCmd := []string{fmt.Sprintf("export CLIENT=`which mongosh&&echo %s||echo mongo`; $CLIENT mongodb://%s:%s@$KB_POD_FQDN:27017/admin?replicaSet=$KB_CLUSTER_COMP_NAME", r.info.Client, userName, userPass)}

return []string{"sh", "-c", strings.Join(mongodbCmd, " ")}
}
Expand Down

0 comments on commit 01b3907

Please sign in to comment.