Merge pull request #10 from redcraft-org/feature/s3-backups

Add S3 backups support

.env.example

@@ -16,6 +16,14 @@ S3_REGION=fr-par
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 
+# S3 backup is used to backup directories specified in "directories_to_backup" in the rcsm_config.json of each server
+S3_BACKUP_ENABLED=false
+S3_BACKUP_ENDPOINT=https://s3.fr-par.scw.cloud
+S3_BACKUP_BUCKET=redcraft-backups
+S3_BACKUP_REGION=fr-par
+AWS_BACKUP_ACCESS_KEY_ID=
+AWS_BACKUP_SECRET_ACCESS_KEY=
+
 # This defines where servers are stored and how they should run
 MINECRAFT_SERVERS_DIRECTORY=/opt/minecraft
 MINECRAFT_SERVERS_TO_CREATE="test1;test2"

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
 
       - uses: actions/setup-go@v2
         with:
-          go-version: '^1.14'
+          go-version: '^1.19'
 
       - name: Install dependencies
         run: go install

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
 
       - uses: actions/setup-go@v2
         with:
-          go-version: '^1.14'
+          go-version: '^1.19'
 
       - name: Install dependencies
         run: go install

go.mod

@@ -1,31 +1,33 @@
 module github.com/redcraft-org/redcraft_server_management
 
-go 1.15
+go 1.19
 
 require (
-	github.com/acroca/go-symbols v0.1.1
 	github.com/aws/aws-sdk-go v1.35.14
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/cweill/gotests v1.5.3
-	github.com/davidrjenni/reftools v0.0.0-20191222082827-65925cf01315
-	github.com/fatih/gomodifytags v1.11.0
-	github.com/go-delve/delve v1.5.0
-	github.com/go-redis/redis v6.15.9+incompatible
 	github.com/go-redis/redis/v8 v8.3.2
-	github.com/godoctor/godoctor v0.0.0-20200702010311-8433dcb3dc61
-	github.com/google/go-querystring v1.0.0
-	github.com/haya14busa/goplay v1.0.0
 	github.com/joho/godotenv v1.3.0
-	github.com/josharian/impl v1.0.0
-	github.com/mdempsky/gocode v0.0.0-20200405233807-4acdcbdea79d
-	github.com/ramya-rao-a/go-outline v0.0.0-20200117021646-2a048b4510eb
+	github.com/otiai10/copy v1.9.0
 	github.com/rhysd/go-github-selfupdate v1.2.2
-	github.com/rogpeppe/godef v1.1.2
-	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
-	github.com/stamblerre/gocode v1.0.0
-	github.com/uudashr/gopkgs/v2 v2.1.2
-	golang.org/x/lint v0.0.0-20200302205851-738671d3881b
-	golang.org/x/tools v0.0.0-20201023174141-c8cfbd0f21e6
-	gopkg.in/bufio.v1 v1.0.0-20140618132640-567b2bfa514e // indirect
-	gopkg.in/redis.v2 v2.3.2
+)
+
+require (
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/golang/protobuf v1.4.2 // indirect
+	github.com/google/go-github/v30 v30.1.0 // indirect
+	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/tcnksm/go-gitconfig v0.1.2 // indirect
+	github.com/ulikunitz/xz v0.5.5 // indirect
+	go.opentelemetry.io/otel v0.13.0 // indirect
+	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
+	golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
+	golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 // indirect
+	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/appengine v1.3.0 // indirect
+	google.golang.org/protobuf v1.23.0 // indirect
 )

rcsm/backup.go

@@ -0,0 +1,174 @@
+package rcsm
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+var (
+	s3BackupClient     *s3.S3
+	s3BackupUploader   *s3manager.Uploader
+	s3BackupClientLock sync.Mutex
+)
+
+// BackupServerS3 creates a backup of the server and uploads it to S3
+func BackupServerS3(serverName string, directoriesToBackup []string) {
+	serverPath := path.Join(MinecraftServersDirectory, serverName)
+	backupFileName := fmt.Sprintf("%s.tar.gz", serverName)
+
+	// Create a .tar.gz file from the temporary file
+	var buf bytes.Buffer
+	compress(serverPath, &buf, directoriesToBackup)
+
+	// Create a temporary file to copy the directories to backup
+	tempFile, err := ioutil.TempFile("", backupFileName)
+	if err != nil {
+		TriggerLogEvent("severe", serverName, fmt.Sprintf("Unable to create temporary file for backup: %s", err))
+		return
+	}
+
+	fileToWrite, err := os.OpenFile(tempFile.Name(), os.O_CREATE|os.O_RDWR, os.FileMode(600))
+	if err != nil {
+		TriggerLogEvent("severe", serverName, fmt.Sprintf("Unable to open temporary file for backup: %s", err))
+		return
+	}
+	if _, err := io.Copy(fileToWrite, &buf); err != nil {
+		TriggerLogEvent("severe", serverName, fmt.Sprintf("Unable to write to temporary file for backup: %s", err))
+		return
+	}
+
+	// Upload the backup to S3
+	uploadBackup(serverName, tempFile.Name())
+
+	// Delete the temporary file
+	if err := os.Remove(tempFile.Name()); err != nil {
+		TriggerLogEvent("severe", serverName, fmt.Sprintf("Unable to delete temporary file for backup: %s", err))
+		return
+	}
+
+	TriggerLogEvent("info", serverName, "Backup complete")
+}
+
+func uploadBackup(serverName string, archivePath string) {
+	_, uploader := getS3BackupClient()
+
+	s3Bucket := S3BackupBucket
+	backupFileName := fmt.Sprintf("%s.tar.gz", serverName)
+	s3Location := fmt.Sprintf("s3://%s/%s", s3Bucket, backupFileName)
+
+	file, err := os.Open(archivePath)
+	if err != nil {
+		TriggerLogEvent("severe", serverName, fmt.Sprintf("Unable to open backup file for upload: %s", err))
+		return
+	}
+
+	TriggerLogEvent("info", serverName, fmt.Sprintf("Uploading backup to %s", s3Location))
+
+	// Upload the file to S3.
+	_, err = uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(s3Bucket),
+		Key:    aws.String(backupFileName),
+		Body:   file,
+	})
+	if err != nil {
+		TriggerLogEvent("severe", serverName, fmt.Sprintf("Unable to upload %q to %q, %v", backupFileName, s3Location, err))
+		return
+	}
+}
+
+func compress(src string, buf io.Writer, directoriesToBackup []string) error {
+	// tar > gzip > buf
+	zr := gzip.NewWriter(buf)
+	tw := tar.NewWriter(zr)
+
+	// Walk through every file in the folder
+	filepath.Walk(src, func(file string, fi os.FileInfo, err error) error {
+		fileOnlyPath := file[len(src):]
+
+		// Check if the file is in the list of directories to backup
+		isWhitelisted := false
+		for _, directoryToBackup := range directoriesToBackup {
+			if fileOnlyPath == directoryToBackup || strings.HasPrefix(fileOnlyPath, "/"+directoryToBackup+"/") {
+				isWhitelisted = true
+				break
+			}
+		}
+
+		// If the file is not in the list of directories to backup, skip it
+		if !isWhitelisted {
+			return nil
+		}
+
+		// Generate tar header
+		header, err := tar.FileInfoHeader(fi, file)
+		if err != nil {
+			return err
+		}
+
+		// Must provide real name (cf https://golang.org/src/archive/tar/common.go?#L626)
+		header.Name = filepath.ToSlash(file)
+
+		// Write header
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+
+		// If not a dir, write file content
+		if !fi.IsDir() {
+			data, err := os.Open(file)
+			if err != nil {
+				return err
+			}
+			if _, err := io.Copy(tw, data); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	// Produce tar
+	if err := tw.Close(); err != nil {
+		return err
+	}
+
+	// Produce gzip
+	if err := zr.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getS3BackupClient() (*s3.S3, *s3manager.Uploader) {
+	s3BackupClientLock.Lock()
+	defer s3BackupClientLock.Unlock()
+
+	if s3BackupClient == nil || s3BackupUploader == nil {
+		s3Session, err := session.NewSession(&aws.Config{
+			Region:   aws.String(S3BackupRegion),
+			Endpoint: aws.String(S3BackupEndpoint),
+		})
+		if err != nil {
+			TriggerLogEvent("fatal", "setup", fmt.Sprintf("Could not create an S3 backup client: %s", err))
+			os.Exit(1)
+		}
+
+		s3BackupClient = s3.New(s3Session)
+		s3BackupUploader = s3manager.NewUploader(s3Session)
+	}
+	return s3BackupClient, s3BackupUploader
+}

rcsm/config.go

@@ -10,7 +10,7 @@ import (
 
 var (
 	// Version is the current version of rcsm
-	Version string = "1.0.1"
+	Version string = "1.1.0"
 	// EnvFile is the path to the .env file config
 	EnvFile string = ".env"
 
@@ -41,6 +41,19 @@ var (
 	// AWSSecretAccessKey is the secret key for S3 authentication
 	AWSSecretAccessKey string = ""
 
+	// S3BackupEnabled specifies wether or not S3 is enabled to backup the files
+	S3BackupEnabled bool = false
+	// S3BackupEndpoint specifies the S3 endpoint if you use something else than AWS
+	S3BackupEndpoint string = ""
+	// S3BackupRegion specifies the region to use for the S3 bucket
+	S3BackupRegion string = ""
+	// S3BackupBucket specifies the bucket name for server templates
+	S3BackupBucket string = ""
+	// AWSBackupAccessKeyID is the key ID for S3 authentication
+	AWSBackupAccessKeyID string = ""
+	// AWSBackupSecretAccessKey is the secret key for S3 authentication
+	AWSBackupSecretAccessKey string = ""
+
 	// MinecraftServersDirectory is the directory where server directories are stored
 	MinecraftServersDirectory string = "/opt/minecraft"
 	// MinecraftServersToCreate is the servers you want to deploy if a template exists on S3
@@ -95,6 +108,13 @@ func ReadConfig() {
 	AWSAccessKeyID = ReadEnvString("AWS_ACCESS_KEY_ID", AWSAccessKeyID)
 	AWSSecretAccessKey = ReadEnvString("AWS_SECRET_ACCESS_KEY", AWSSecretAccessKey)
 
+	S3BackupEnabled = ReadEnvBool("S3_BACKUP_ENABLED", S3BackupEnabled)
+	S3BackupEndpoint = ReadEnvString("S3_BACKUP_ENDPOINT", S3BackupEndpoint)
+	S3BackupRegion = ReadEnvString("S3_BACKUP_REGION", S3BackupRegion)
+	S3BackupBucket = ReadEnvString("S3_BACKUP_BUCKET", S3BackupBucket)
+	AWSBackupAccessKeyID = ReadEnvString("AWS_BACKUP_ACCESS_KEY_ID", AWSBackupAccessKeyID)
+	AWSBackupSecretAccessKey = ReadEnvString("AWS_BACKUP_SECRET_ACCESS_KEY", AWSBackupSecretAccessKey)
+
 	MinecraftServersDirectory = ReadEnvString("MINECRAFT_SERVERS_DIRECTORY", MinecraftServersDirectory)
 	MinecraftServersToCreate = ReadEnvString("MINECRAFT_SERVERS_TO_CREATE", MinecraftServersToCreate)
 	MinecraftTmuxSessionPrefix = ReadEnvString("MINECRAFT_TMUX_SESSION_PREFIX", MinecraftTmuxSessionPrefix)

rcsm/remote.go

@@ -33,6 +33,8 @@ func parseRedisMessage(channel string, payload string) {
 			StopAllServers()
 		case "restart":
 			RestartAllServers()
+		case "backup":
+			BackupAllServers()
 		case "run":
 			RunCommandAllServers(redisCommand.Content)
 		}
@@ -44,6 +46,8 @@ func parseRedisMessage(channel string, payload string) {
 			StopServer(serverName)
 		case "restart":
 			RestartServer(serverName)
+		case "backup":
+			BackupServer(serverName)
 		case "run":
 			RunCommandServer(serverName, redisCommand.Content)
 		}

rcsm/servers.go

@@ -11,14 +11,15 @@ import (
 
 // MinecraftServer defines the stats about a server
 type MinecraftServer struct {
-	name         string
-	fullPath     string
-	running      bool
-	crashed      bool
-	restartTries int64
-	firstRetry   time.Time
-	StartCommand string `json:"start_command"`
-	StopCommand  string `json:"stop_command"`
+	name                string
+	fullPath            string
+	running             bool
+	crashed             bool
+	restartTries        int64
+	firstRetry          time.Time
+	StartCommand        string   `json:"start_command"`
+	StopCommand         string   `json:"stop_command"`
+	DirectoriesToBackup []string `json:"directories_to_backup"`
 }
 
 var (
@@ -126,6 +127,36 @@ func RestartServer(serverName string) {
 	startServer(server)
 }
 
+// BackupServer backups a server with a specified name
+func BackupServer(serverName string) {
+	if !S3BackupEnabled {
+		TriggerLogEvent("info", serverName, "Backup is disabled, skipping")
+		return
+	}
+
+	// Acquire lock on minecraftServers
+	minecraftServersLock.Lock()
+	defer minecraftServersLock.Unlock()
+
+	TriggerLogEvent("info", serverName, "Backing up server")
+
+	server := minecraftServers[serverName]
+
+	backupServer(server)
+}
+
+// BackupAllServers backups all servers
+func BackupAllServers() {
+	// Acquire lock on minecraftServers
+	minecraftServersLock.Lock()
+	defer minecraftServersLock.Unlock()
+
+	TriggerLogEvent("info", "rcsm", fmt.Sprintf("Backing up all servers"))
+	for _, server := range minecraftServers {
+		backupServer(server)
+	}
+}
+
 // RunCommandServer restarts a server with a specified name
 func RunCommandServer(serverName string, command string) {
 	// Acquire lock on minecraftServers
@@ -256,3 +287,7 @@ func runCommand(server MinecraftServer, command string) bool {
 
 	return !server.running
 }
+
+func backupServer(server MinecraftServer) {
+	BackupServerS3(server.name, server.DirectoriesToBackup)
+}