From 3ab2aa6824464ef35d9da7ea7f6491d374a8c456 Mon Sep 17 00:00:00 2001 From: peb-adr Date: Fri, 21 Jul 2023 07:57:47 +0200 Subject: [PATCH 1/4] rename env-vars (#225) --- pkg/config/default-config.yml | 49 +++++++++++++-------------- pkg/config/default-docker-compose.yml | 1 + pkg/initialdata/initialdata.go | 7 ++-- pkg/initialdata/initialdata_test.go | 3 +- pkg/server/server.go | 18 ++++------ pkg/setup/setup_test.go | 23 ++++++------- 6 files changed, 44 insertions(+), 57 deletions(-) diff --git a/pkg/config/default-config.yml b/pkg/config/default-config.yml index a4e2d49..77aa87b 100644 --- a/pkg/config/default-config.yml +++ b/pkg/config/default-config.yml @@ -29,11 +29,6 @@ defaultEnvironment: DATASTORE_READER_PORT: 9010 DATASTORE_WRITER_HOST: datastoreWriter DATASTORE_WRITER_PORT: 9011 - DATASTORE_DATABASE_HOST: postgres - DATASTORE_DATABASE_PORT: 5432 - DATASTORE_DATABASE_NAME: openslides - DATASTORE_DATABASE_USER: openslides - DATASTORE_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password AUTOUPDATE_HOST: autoupdate AUTOUPDATE_PORT: 9012 @@ -43,46 +38,45 @@ defaultEnvironment: VOTE_HOST: vote VOTE_PORT: 9013 - VOTE_DATABASE_HOST: postgres - VOTE_DATABASE_PORT: 5432 - VOTE_DATABASE_NAME: openslides - VOTE_DATABASE_USER: openslides - VOTE_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password - VOTE_REDIS_HOST: redis - VOTE_REDIS_PORT: 6379 CACHE_HOST: redis CACHE_PORT: 6379 - MESSAGE_BUS_HOST: redis MESSAGE_BUS_PORT: 6379 MEDIA_HOST: media MEDIA_PORT: 9006 - MEDIA_DATABASE_HOST: postgres - MEDIA_DATABASE_PORT: 5432 - MEDIA_DATABASE_NAME: openslides - MEDIA_DATABASE_USER: openslides - MEDIA_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password - MEDIA_BLOCK_SIZE: 4096 - MEDIA_PRESENTER_HOST: backendPresenter - MEDIA_PRESENTER_PORT: 9003 ICC_HOST: icc ICC_PORT: 9007 - ICC_REDIS_HOST: redis - ICC_REDIS_PORT: 6379 MANAGE_HOST: manage MANAGE_PORT: 9008 - MANAGE_AUTH_PASSWORD_FILE: /run/secrets/manage_auth_password - MANAGE_ACTION_HOST: backendManage + + DATABASE_HOST: postgres + DATABASE_PORT: 5432 + DATABASE_NAME: openslides + DATABASE_USER: openslides + DATABASE_PASSWORD_FILE: /run/secrets/postgres_password + MEDIA_DATABASE_HOST: postgres + MEDIA_DATABASE_PORT: 5432 + MEDIA_DATABASE_NAME: openslides + MEDIA_DATABASE_USER: openslides + MEDIA_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password + VOTE_DATABASE_HOST: postgres + VOTE_DATABASE_PORT: 5432 + VOTE_DATABASE_NAME: openslides + VOTE_DATABASE_USER: openslides + VOTE_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password INTERNAL_AUTH_PASSWORD_FILE: /run/secrets/internal_auth_password + MANAGE_AUTH_PASSWORD_FILE: /run/secrets/manage_auth_password + AUTH_TOKEN_KEY_FILE: /run/secrets/auth_token_key + AUTH_COOKIE_KEY_FILE: /run/secrets/auth_cookie_key + SUPERADMIN_PASSWORD_FILE: /run/secrets/superadmin OPENSLIDES_LOGLEVEL: info OPENSLIDES_DEVELOPMENT: "false" - SYSTEM_URL: localhost:8000 # You can extend or replace parts of the defaultEnvironment. @@ -97,6 +91,9 @@ services: datastoreReader: environment: NUM_WORKERS: 8 + manage: + environment: + ACTION_HOST: backendManage # All properties from the "defaults" section are available here. # diff --git a/pkg/config/default-docker-compose.yml b/pkg/config/default-docker-compose.yml index b951b3b..9368c1d 100644 --- a/pkg/config/default-docker-compose.yml +++ b/pkg/config/default-docker-compose.yml @@ -245,6 +245,7 @@ services: secrets: - auth_token_key - auth_cookie_key + - internal_auth_password {{- with .AdditionalContent }}{{ marshalContent 4 . }}{{- end }} {{- end }} diff --git a/pkg/initialdata/initialdata.go b/pkg/initialdata/initialdata.go index ce68bcb..3c140b1 100644 --- a/pkg/initialdata/initialdata.go +++ b/pkg/initialdata/initialdata.go @@ -5,13 +5,11 @@ import ( "encoding/json" "fmt" "os" - "path" "strings" "github.com/OpenSlides/openslides-manage-service/pkg/connection" "github.com/OpenSlides/openslides-manage-service/pkg/fehler" "github.com/OpenSlides/openslides-manage-service/pkg/setpassword" - "github.com/OpenSlides/openslides-manage-service/pkg/setup" "github.com/OpenSlides/openslides-manage-service/pkg/shared" "github.com/OpenSlides/openslides-manage-service/proto" "github.com/spf13/cobra" @@ -103,7 +101,7 @@ type backendAction interface { } // InitialData sets initial data in the datastore. -func InitialData(ctx context.Context, in *proto.InitialDataRequest, runPath string, ba backendAction) (*proto.InitialDataResponse, error) { +func InitialData(ctx context.Context, in *proto.InitialDataRequest, superadminSecretFile string, ba backendAction) (*proto.InitialDataResponse, error) { initialData := in.Data if initialData == nil { // The backend expects at least an empty object. @@ -133,8 +131,7 @@ func InitialData(ctx context.Context, in *proto.InitialDataRequest, runPath stri return nil, fmt.Errorf("requesting backend action %q: %w", name, err) } - p := path.Join(runPath, setup.SecretsDirName, setup.SuperadminFileName) - if err := SetSuperadminPassword(ctx, p, ba); err != nil { + if err := SetSuperadminPassword(ctx, superadminSecretFile, ba); err != nil { return nil, fmt.Errorf("setting superadmin password: %w", err) } diff --git a/pkg/initialdata/initialdata_test.go b/pkg/initialdata/initialdata_test.go index ba66a6a..7712e20 100644 --- a/pkg/initialdata/initialdata_test.go +++ b/pkg/initialdata/initialdata_test.go @@ -114,7 +114,8 @@ func TestInitialDataServerAll(t *testing.T) { // Run tests t.Run("running the first time", func(t *testing.T) { - resp, err := initialdata.InitialData(ctx, in, testDir, ma) + p := path.Join(testDir, setup.SecretsDirName, setup.SuperadminFileName) + resp, err := initialdata.InitialData(ctx, in, p, ma) if err != nil { t.Fatalf("running InitialData() failed: %v", err) } diff --git a/pkg/server/server.go b/pkg/server/server.go index e95b5a9..7cb3717 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -29,8 +29,6 @@ import ( "google.golang.org/grpc/metadata" ) -const runDir = "/run" - // Run starts the manage server. func Run(cfg *Config) error { logger, err := shared.NewLogger(cfg.OpenSlidesLoglevel) @@ -105,7 +103,7 @@ func (s *srv) InitialData(ctx context.Context, in *proto.InitialDataRequest) (*p return nil, fmt.Errorf("getting internal auth password from file: %w", err) } a := backendaction.New(s.config.manageBackendActionURL(), pw, backendaction.ActionRoute) - return initialdata.InitialData(ctx, in, runDir, a) + return initialdata.InitialData(ctx, in, s.config.SuperadminPasswordFile, a) } @@ -208,23 +206,19 @@ type Config struct { // variables. The first value is the name of the environment variable. After // a comma the default value can be given. If no default value is given, then // an empty string is used. The type of a env field has to be string. - Port string `env:"MANAGE_PORT,9008"` - ManageAuthPasswordFile string `env:"MANAGE_AUTH_PASSWORD_FILE,/run/secrets/manage_auth_password"` + Port string `env:"MANAGE_PORT,9008"` + ManageAuthPasswordFile string `env:"MANAGE_AUTH_PASSWORD_FILE,/run/secrets/manage_auth_password"` + InternalAuthPasswordFile string `env:"INTERNAL_AUTH_PASSWORD_FILE,/run/secrets/internal_auth_password"` + SuperadminPasswordFile string `env:"SUPERADMIN_PASSWORD_FILE,/run/secrets/superadmin"` - // Hint: The env var for the host is MANAGE_ACTION_HOST but the env vars for - // protocol and port don't have the MANAGE_ prefix because the backend - // itself does not distiguish between an common backend container and a - // manage backend container. So protocol and port are the same for all backend containers. ManageActionProtocol string `env:"ACTION_PROTOCOL,http"` - ManageActionHost string `env:"MANAGE_ACTION_HOST,backendManage"` + ManageActionHost string `env:"ACTION_HOST,backendManage"` ManageActionPort string `env:"ACTION_PORT,9002"` DatastoreReaderProtocol string `env:"DATASTORE_READER_PROTOCOL,http"` DatastoreReaderHost string `env:"DATASTORE_READER_HOST,datastore-reader"` DatastoreReaderPort string `env:"DATASTORE_READER_PORT,9010"` - InternalAuthPasswordFile string `env:"INTERNAL_AUTH_PASSWORD_FILE,/run/secrets/internal_auth_password"` - OpenSlidesDevelopment string `env:"OPENSLIDES_DEVELOPMENT,0"` OpenSlidesLoglevel string `env:"OPENSLIDES_LOGLEVEL,info"` } diff --git a/pkg/setup/setup_test.go b/pkg/setup/setup_test.go index 014cd48..818b84b 100644 --- a/pkg/setup/setup_test.go +++ b/pkg/setup/setup_test.go @@ -469,31 +469,29 @@ version: "3.4" x-default-environment: &default-environment ACTION_HOST: backendAction ACTION_PORT: "9002" + AUTH_COOKIE_KEY_FILE: /run/secrets/auth_cookie_key AUTH_HOST: auth AUTH_PORT: "9004" + AUTH_TOKEN_KEY_FILE: /run/secrets/auth_token_key AUTOUPDATE_HOST: autoupdate AUTOUPDATE_PORT: "9012" CACHE_HOST: redis CACHE_PORT: "6379" - DATASTORE_DATABASE_HOST: postgres - DATASTORE_DATABASE_NAME: openslides - DATASTORE_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password - DATASTORE_DATABASE_PORT: "5432" - DATASTORE_DATABASE_USER: openslides + DATABASE_HOST: postgres + DATABASE_NAME: openslides + DATABASE_PASSWORD_FILE: /run/secrets/postgres_password + DATABASE_PORT: "5432" + DATABASE_USER: openslides DATASTORE_READER_HOST: datastoreReader DATASTORE_READER_PORT: "9010" DATASTORE_WRITER_HOST: datastoreWriter DATASTORE_WRITER_PORT: "9011" ICC_HOST: icc ICC_PORT: "9007" - ICC_REDIS_HOST: redis - ICC_REDIS_PORT: "6379" INTERNAL_AUTH_PASSWORD_FILE: /run/secrets/internal_auth_password - MANAGE_ACTION_HOST: backendManage MANAGE_AUTH_PASSWORD_FILE: /run/secrets/manage_auth_password MANAGE_HOST: manage MANAGE_PORT: "9008" - MEDIA_BLOCK_SIZE: "4096" MEDIA_DATABASE_HOST: postgres MEDIA_DATABASE_NAME: openslides MEDIA_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password @@ -501,14 +499,13 @@ x-default-environment: &default-environment MEDIA_DATABASE_USER: openslides MEDIA_HOST: media MEDIA_PORT: "9006" - MEDIA_PRESENTER_HOST: backendPresenter - MEDIA_PRESENTER_PORT: "9003" MESSAGE_BUS_HOST: redis MESSAGE_BUS_PORT: "6379" OPENSLIDES_DEVELOPMENT: "false" OPENSLIDES_LOGLEVEL: info PRESENTER_HOST: backendPresenter PRESENTER_PORT: "9003" + SUPERADMIN_PASSWORD_FILE: /run/secrets/superadmin SYSTEM_URL: localhost:8000 VOTE_DATABASE_HOST: postgres VOTE_DATABASE_NAME: openslides @@ -517,8 +514,6 @@ x-default-environment: &default-environment VOTE_DATABASE_USER: openslides VOTE_HOST: vote VOTE_PORT: "9013" - VOTE_REDIS_HOST: redis - VOTE_REDIS_PORT: "6379" services: proxy: @@ -680,6 +675,7 @@ services: secrets: - auth_token_key - auth_cookie_key + - internal_auth_password vote: image: ghcr.io/openslides/openslides/openslides-vote:latest @@ -741,6 +737,7 @@ services: - backendManage environment: << : *default-environment + ACTION_HOST: backendManage networks: - frontend - data From 9ff78f7a5d39e8212c53ff053c1ecb4217fc3a24 Mon Sep 17 00:00:00 2001 From: Magnus Schieder <50337967+m-schieder@users.noreply.github.com> Date: Mon, 24 Jul 2023 16:01:45 +0200 Subject: [PATCH 2/4] Fix backendAction secrets (#226) --- pkg/config/default-docker-compose.yml | 1 + pkg/setup/setup_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/config/default-docker-compose.yml b/pkg/config/default-docker-compose.yml index 9368c1d..663d8b2 100644 --- a/pkg/config/default-docker-compose.yml +++ b/pkg/config/default-docker-compose.yml @@ -91,6 +91,7 @@ services: secrets: - auth_token_key - auth_cookie_key + - internal_auth_password - postgres_password {{- with .AdditionalContent }}{{ marshalContent 4 . }}{{- end }} {{- end }} diff --git a/pkg/setup/setup_test.go b/pkg/setup/setup_test.go index 818b84b..878f54f 100644 --- a/pkg/setup/setup_test.go +++ b/pkg/setup/setup_test.go @@ -574,6 +574,7 @@ services: secrets: - auth_token_key - auth_cookie_key + - internal_auth_password - postgres_password backendPresenter: From 9fc6907e848f0a17a280a90ef6c9a9673f456884 Mon Sep 17 00:00:00 2001 From: Joshua Sangmeister <33004050+jsangmeister@users.noreply.github.com> Date: Thu, 3 Aug 2023 14:41:12 +0200 Subject: [PATCH 3/4] Extend project automation (#227) --- .github/workflows/project-automation.yml | 32 +++++++++++++++++++ .github/workflows/project-issue-closed.yml | 14 ++++++++ .github/workflows/project-issue-opened.yml | 15 +++++++++ .../workflows/project-pull-request-closed.yml | 14 ++++++++ .../workflows/project-pull-request-opened.yml | 15 +++++++++ .../project-pull-request-review-requested.yml | 14 ++++++++ .github/workflows/set-project.yml | 23 ------------- 7 files changed, 104 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/project-automation.yml create mode 100644 .github/workflows/project-issue-closed.yml create mode 100644 .github/workflows/project-issue-opened.yml create mode 100644 .github/workflows/project-pull-request-closed.yml create mode 100644 .github/workflows/project-pull-request-opened.yml create mode 100644 .github/workflows/project-pull-request-review-requested.yml delete mode 100644 .github/workflows/set-project.yml diff --git a/.github/workflows/project-automation.yml b/.github/workflows/project-automation.yml new file mode 100644 index 0000000..5eb9e0a --- /dev/null +++ b/.github/workflows/project-automation.yml @@ -0,0 +1,32 @@ +name: Project automation +on: + workflow_call: + inputs: + resource_node_id: + required: true + type: string + status_value: + required: true + type: string + secrets: + AUTOMATION_APP_ID: + required: true + AUTOMATION_APP_INSTALLATION_ID: + required: true + AUTOMATION_APP_PRIVATE_KEY: + required: true + +jobs: + workflow_call: + name: Set status + runs-on: ubuntu-latest + steps: + - uses: leonsteinhaeuser/project-beta-automations@v2.1.0 + with: + gh_app_ID: ${{ secrets.AUTOMATION_APP_ID }} + gh_app_installation_ID: ${{ secrets.AUTOMATION_APP_INSTALLATION_ID }} + gh_app_secret_key: ${{ secrets.AUTOMATION_APP_PRIVATE_KEY }} + organization: OpenSlides + project_id: 2 + resource_node_id: ${{ inputs.resource_node_id }} + status_value: ${{ inputs.status_value }} diff --git a/.github/workflows/project-issue-closed.yml b/.github/workflows/project-issue-closed.yml new file mode 100644 index 0000000..eb6199d --- /dev/null +++ b/.github/workflows/project-issue-closed.yml @@ -0,0 +1,14 @@ +name: Project automation +on: + issues: + types: + - closed + +jobs: + issue_closed: + name: Issue closed + uses: ./.github/workflows/project-automation.yml + secrets: inherit + with: + resource_node_id: ${{ github.event.issue.node_id }} + status_value: "Done" diff --git a/.github/workflows/project-issue-opened.yml b/.github/workflows/project-issue-opened.yml new file mode 100644 index 0000000..068a5f0 --- /dev/null +++ b/.github/workflows/project-issue-opened.yml @@ -0,0 +1,15 @@ +name: Project automation +on: + issues: + types: + - opened + - reopened + +jobs: + issue_opened: + name: Issue opened + uses: ./.github/workflows/project-automation.yml + secrets: inherit + with: + resource_node_id: ${{ github.event.issue.node_id }} + status_value: "Backlog" diff --git a/.github/workflows/project-pull-request-closed.yml b/.github/workflows/project-pull-request-closed.yml new file mode 100644 index 0000000..c09de8b --- /dev/null +++ b/.github/workflows/project-pull-request-closed.yml @@ -0,0 +1,14 @@ +name: Project automation +on: + pull_request_target: + types: + - closed + +jobs: + pull_request_closed: + name: Pull request closed + uses: ./.github/workflows/project-automation.yml + secrets: inherit + with: + resource_node_id: ${{ github.event.pull_request.node_id }} + status_value: "Done" diff --git a/.github/workflows/project-pull-request-opened.yml b/.github/workflows/project-pull-request-opened.yml new file mode 100644 index 0000000..55901d1 --- /dev/null +++ b/.github/workflows/project-pull-request-opened.yml @@ -0,0 +1,15 @@ +name: Project automation +on: + pull_request_target: + types: + - opened + - reopened + +jobs: + pull_request_opened: + name: Pull request opened + uses: ./.github/workflows/project-automation.yml + secrets: inherit + with: + resource_node_id: ${{ github.event.pull_request.node_id }} + status_value: "Work in progress" diff --git a/.github/workflows/project-pull-request-review-requested.yml b/.github/workflows/project-pull-request-review-requested.yml new file mode 100644 index 0000000..698edd6 --- /dev/null +++ b/.github/workflows/project-pull-request-review-requested.yml @@ -0,0 +1,14 @@ +name: Project automation +on: + pull_request_target: + types: + - review_requested + +jobs: + pull_request_review_requested: + name: Pull request review requested + uses: ./.github/workflows/project-automation.yml + secrets: inherit + with: + resource_node_id: ${{ github.event.pull_request.node_id }} + status_value: "Review in progress" diff --git a/.github/workflows/set-project.yml b/.github/workflows/set-project.yml deleted file mode 100644 index dcc5055..0000000 --- a/.github/workflows/set-project.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -name: Set project -on: - issues: - types: [opened] - pull_request_target: - types: [opened] - -jobs: - set-project: - name: 'Set project' - runs-on: ubuntu-latest - steps: - - uses: tibdex/github-app-token@v1 - id: generate-token - with: - app_id: ${{ secrets.AUTOMATION_APP_ID }} - private_key: ${{ secrets.AUTOMATION_APP_PRIVATE_KEY }} - - - uses: actions/add-to-project@v0.5.0 - with: - project-url: https://github.com/orgs/OpenSlides/projects/2 - github-token: ${{ steps.generate-token.outputs.token }} From f58d71b82d2dabfe11710b91120f9e9e139c8b65 Mon Sep 17 00:00:00 2001 From: peb-adr Date: Tue, 15 Aug 2023 13:16:02 +0200 Subject: [PATCH 4/4] Add auth-*-key secrets to media (#230) Required since https://github.com/OpenSlides/openslides-media-service/pull/70 --- pkg/config/default-docker-compose.yml | 2 ++ pkg/setup/setup_test.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pkg/config/default-docker-compose.yml b/pkg/config/default-docker-compose.yml index 663d8b2..3321b8f 100644 --- a/pkg/config/default-docker-compose.yml +++ b/pkg/config/default-docker-compose.yml @@ -303,6 +303,8 @@ services: - frontend - data secrets: + - auth_token_key + - auth_cookie_key - postgres_password {{- with .AdditionalContent }}{{ marshalContent 4 . }}{{- end }} {{- end }} diff --git a/pkg/setup/setup_test.go b/pkg/setup/setup_test.go index 878f54f..3c5580c 100644 --- a/pkg/setup/setup_test.go +++ b/pkg/setup/setup_test.go @@ -713,6 +713,8 @@ services: - frontend - data secrets: + - auth_token_key + - auth_cookie_key - postgres_password icc: