diff --git a/.github/workflows/project-automation.yml b/.github/workflows/project-automation.yml new file mode 100644 index 0000000..5eb9e0a --- /dev/null +++ b/.github/workflows/project-automation.yml @@ -0,0 +1,32 @@ +name: Project automation +on: + workflow_call: + inputs: + resource_node_id: + required: true + type: string + status_value: + required: true + type: string + secrets: + AUTOMATION_APP_ID: + required: true + AUTOMATION_APP_INSTALLATION_ID: + required: true + AUTOMATION_APP_PRIVATE_KEY: + required: true + +jobs: + workflow_call: + name: Set status + runs-on: ubuntu-latest + steps: + - uses: leonsteinhaeuser/project-beta-automations@v2.1.0 + with: + gh_app_ID: ${{ secrets.AUTOMATION_APP_ID }} + gh_app_installation_ID: ${{ secrets.AUTOMATION_APP_INSTALLATION_ID }} + gh_app_secret_key: ${{ secrets.AUTOMATION_APP_PRIVATE_KEY }} + organization: OpenSlides + project_id: 2 + resource_node_id: ${{ inputs.resource_node_id }} + status_value: ${{ inputs.status_value }} diff --git a/.github/workflows/project-issue-closed.yml b/.github/workflows/project-issue-closed.yml new file mode 100644 index 0000000..eb6199d --- /dev/null +++ b/.github/workflows/project-issue-closed.yml @@ -0,0 +1,14 @@ +name: Project automation +on: + issues: + types: + - closed + +jobs: + issue_closed: + name: Issue closed + uses: ./.github/workflows/project-automation.yml + secrets: inherit + with: + resource_node_id: ${{ github.event.issue.node_id }} + status_value: "Done" diff --git a/.github/workflows/project-issue-opened.yml b/.github/workflows/project-issue-opened.yml new file mode 100644 index 0000000..068a5f0 --- /dev/null +++ b/.github/workflows/project-issue-opened.yml @@ -0,0 +1,15 @@ +name: Project automation +on: + issues: + types: + - opened + - reopened + +jobs: + issue_opened: + name: Issue opened + uses: ./.github/workflows/project-automation.yml + secrets: inherit + with: + resource_node_id: ${{ github.event.issue.node_id }} + status_value: "Backlog" diff --git a/.github/workflows/project-pull-request-closed.yml b/.github/workflows/project-pull-request-closed.yml new file mode 100644 index 0000000..c09de8b --- /dev/null +++ b/.github/workflows/project-pull-request-closed.yml @@ -0,0 +1,14 @@ +name: Project automation +on: + pull_request_target: + types: + - closed + +jobs: + pull_request_closed: + name: Pull request closed + uses: ./.github/workflows/project-automation.yml + secrets: inherit + with: + resource_node_id: ${{ github.event.pull_request.node_id }} + status_value: "Done" diff --git a/.github/workflows/project-pull-request-opened.yml b/.github/workflows/project-pull-request-opened.yml new file mode 100644 index 0000000..55901d1 --- /dev/null +++ b/.github/workflows/project-pull-request-opened.yml @@ -0,0 +1,15 @@ +name: Project automation +on: + pull_request_target: + types: + - opened + - reopened + +jobs: + pull_request_opened: + name: Pull request opened + uses: ./.github/workflows/project-automation.yml + secrets: inherit + with: + resource_node_id: ${{ github.event.pull_request.node_id }} + status_value: "Work in progress" diff --git a/.github/workflows/project-pull-request-review-requested.yml b/.github/workflows/project-pull-request-review-requested.yml new file mode 100644 index 0000000..698edd6 --- /dev/null +++ b/.github/workflows/project-pull-request-review-requested.yml @@ -0,0 +1,14 @@ +name: Project automation +on: + pull_request_target: + types: + - review_requested + +jobs: + pull_request_review_requested: + name: Pull request review requested + uses: ./.github/workflows/project-automation.yml + secrets: inherit + with: + resource_node_id: ${{ github.event.pull_request.node_id }} + status_value: "Review in progress" diff --git a/.github/workflows/release.yml b/.github/workflows/release-create.yml similarity index 52% rename from .github/workflows/release.yml rename to .github/workflows/release-create.yml index b8d0b65..0c03a91 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release-create.yml @@ -1,7 +1,20 @@ -name: GitHub Release +name: GitHub Release (reusable) on: - push: - branches: [main] + workflow_call: + inputs: + branch: + type: string + default: main + tag: + type: string + default: dev + title: + type: string + default: Development Build + prerelease: + type: boolean + default: true + jobs: release: name: Create GitHub Release and upload binary @@ -14,6 +27,8 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v2 + with: + ref: "${{ inputs.branch }}" - name: Build binaries run: go build ./cmd/openslides @@ -22,8 +37,8 @@ jobs: uses: marvinpinto/action-automatic-releases@latest with: repo_token: "${{ secrets.GITHUB_TOKEN }}" - automatic_release_tag: "latest" - prerelease: true - title: "Development Build" + automatic_release_tag: "${{ inputs.tag }}" + prerelease: "${{ inputs.prerelease }}" + title: "${{ inputs.title }}" files: | openslides diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml new file mode 100644 index 0000000..e4d4718 --- /dev/null +++ b/.github/workflows/release-dev.yml @@ -0,0 +1,9 @@ +name: GitHub Release (dev) +on: + push: + branches: [main] + +jobs: + release-dev: + name: "Create dev Release" + uses: ./.github/workflows/release-create.yml diff --git a/.github/workflows/release-stable.yml b/.github/workflows/release-stable.yml new file mode 100644 index 0000000..4eef664 --- /dev/null +++ b/.github/workflows/release-stable.yml @@ -0,0 +1,24 @@ +name: GitHub Release (stable) +on: + repository_dispatch: + types: [stable-update] + +jobs: + release-stable: + name: "Create GitHub Release (4.x.x) and upload binary" + if: startsWith(github.event.client_payload.branch, 'stable/4') + uses: ./.github/workflows/release-create.yml + with: + branch: "${{ github.event.client_payload.branch }}" + tag: "${{ github.event.client_payload.version }}" + title: Stable Build + prerelease: false + release-stable-latest: + name: "Create GitHub Release (latest) and upload binary" + if: startsWith(github.event.client_payload.branch, 'stable/4') + uses: ./.github/workflows/release-create.yml + with: + branch: "${{ github.event.client_payload.branch }}" + tag: latest + title: Stable Build + prerelease: false diff --git a/.github/workflows/set-project.yml b/.github/workflows/set-project.yml deleted file mode 100644 index dcc5055..0000000 --- a/.github/workflows/set-project.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -name: Set project -on: - issues: - types: [opened] - pull_request_target: - types: [opened] - -jobs: - set-project: - name: 'Set project' - runs-on: ubuntu-latest - steps: - - uses: tibdex/github-app-token@v1 - id: generate-token - with: - app_id: ${{ secrets.AUTOMATION_APP_ID }} - private_key: ${{ secrets.AUTOMATION_APP_PRIVATE_KEY }} - - - uses: actions/add-to-project@v0.5.0 - with: - project-url: https://github.com/orgs/OpenSlides/projects/2 - github-token: ${{ steps.generate-token.outputs.token }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 423ede2..fd52760 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,3 @@ jobs: - name: go test run: go test -timeout 10s -race ./... - - - name: go test for special system tests - run: OPENSLIDES_MANAGE_SERVICE_SYSTEM_TEST=true go test -timeout 300s -v pkg/client/system_test.go diff --git a/pkg/config/config.go b/pkg/config/config.go index 5cba70c..84e49b3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -239,6 +239,7 @@ func NewYmlConfig(configFiles [][]byte) (*YmlConfig, error) { "autoupdate", "auth", "vote", + "search", "redis", "media", "icc", diff --git a/pkg/config/default-config.yml b/pkg/config/default-config.yml index a4e2d49..dfcde26 100644 --- a/pkg/config/default-config.yml +++ b/pkg/config/default-config.yml @@ -29,11 +29,6 @@ defaultEnvironment: DATASTORE_READER_PORT: 9010 DATASTORE_WRITER_HOST: datastoreWriter DATASTORE_WRITER_PORT: 9011 - DATASTORE_DATABASE_HOST: postgres - DATASTORE_DATABASE_PORT: 5432 - DATASTORE_DATABASE_NAME: openslides - DATASTORE_DATABASE_USER: openslides - DATASTORE_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password AUTOUPDATE_HOST: autoupdate AUTOUPDATE_PORT: 9012 @@ -41,48 +36,51 @@ defaultEnvironment: AUTH_HOST: auth AUTH_PORT: 9004 + SEARCH_HOST: search + SEARCH_PORT: 9050 + RESTRICTER_URL: http://autoupdate:9012/internal/autoupdate + VOTE_HOST: vote VOTE_PORT: 9013 - VOTE_DATABASE_HOST: postgres - VOTE_DATABASE_PORT: 5432 - VOTE_DATABASE_NAME: openslides - VOTE_DATABASE_USER: openslides - VOTE_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password - VOTE_REDIS_HOST: redis - VOTE_REDIS_PORT: 6379 CACHE_HOST: redis CACHE_PORT: 6379 - MESSAGE_BUS_HOST: redis MESSAGE_BUS_PORT: 6379 MEDIA_HOST: media MEDIA_PORT: 9006 - MEDIA_DATABASE_HOST: postgres - MEDIA_DATABASE_PORT: 5432 - MEDIA_DATABASE_NAME: openslides - MEDIA_DATABASE_USER: openslides - MEDIA_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password - MEDIA_BLOCK_SIZE: 4096 - MEDIA_PRESENTER_HOST: backendPresenter - MEDIA_PRESENTER_PORT: 9003 ICC_HOST: icc ICC_PORT: 9007 - ICC_REDIS_HOST: redis - ICC_REDIS_PORT: 6379 MANAGE_HOST: manage MANAGE_PORT: 9008 - MANAGE_AUTH_PASSWORD_FILE: /run/secrets/manage_auth_password - MANAGE_ACTION_HOST: backendManage + + DATABASE_HOST: postgres + DATABASE_PORT: 5432 + DATABASE_NAME: openslides + DATABASE_USER: openslides + DATABASE_PASSWORD_FILE: /run/secrets/postgres_password + MEDIA_DATABASE_HOST: postgres + MEDIA_DATABASE_PORT: 5432 + MEDIA_DATABASE_NAME: openslides + MEDIA_DATABASE_USER: openslides + MEDIA_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password + VOTE_DATABASE_HOST: postgres + VOTE_DATABASE_PORT: 5432 + VOTE_DATABASE_NAME: openslides + VOTE_DATABASE_USER: openslides + VOTE_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password INTERNAL_AUTH_PASSWORD_FILE: /run/secrets/internal_auth_password + MANAGE_AUTH_PASSWORD_FILE: /run/secrets/manage_auth_password + AUTH_TOKEN_KEY_FILE: /run/secrets/auth_token_key + AUTH_COOKIE_KEY_FILE: /run/secrets/auth_cookie_key + SUPERADMIN_PASSWORD_FILE: /run/secrets/superadmin OPENSLIDES_LOGLEVEL: info OPENSLIDES_DEVELOPMENT: "false" - SYSTEM_URL: localhost:8000 # You can extend or replace parts of the defaultEnvironment. @@ -97,6 +95,9 @@ services: datastoreReader: environment: NUM_WORKERS: 8 + manage: + environment: + ACTION_HOST: backendManage # All properties from the "defaults" section are available here. # diff --git a/pkg/config/default-docker-compose.yml b/pkg/config/default-docker-compose.yml index b951b3b..5d2a08b 100644 --- a/pkg/config/default-docker-compose.yml +++ b/pkg/config/default-docker-compose.yml @@ -15,6 +15,7 @@ services: - backendAction - backendPresenter - autoupdate + - search - auth - media - icc @@ -54,6 +55,7 @@ services: - backendAction - backendPresenter - autoupdate + - search - auth - media - icc @@ -91,6 +93,7 @@ services: secrets: - auth_token_key - auth_cookie_key + - internal_auth_password - postgres_password {{- with .AdditionalContent }}{{ marshalContent 4 . }}{{- end }} {{- end }} @@ -227,6 +230,30 @@ services: {{- end }} + {{- with .Services.search }} + + search: + image: {{ .ContainerRegistry }}/openslides-search:{{ .Tag }} + {{- if checkFlag $.DisableDependsOn }}{{ else }} + depends_on: + - datastoreReader + - postgres + - autoupdate + {{- end }} + environment: + << : *default-environment + {{- with .Environment }}{{ marshalContent 6 . }}{{- end }} + networks: + - frontend + - data + secrets: + - auth_token_key + - auth_cookie_key + - postgres_password + {{- with .AdditionalContent }}{{ marshalContent 4 . }}{{- end }} + {{- end }} + + {{- with .Services.auth }} auth: @@ -245,6 +272,7 @@ services: secrets: - auth_token_key - auth_cookie_key + - internal_auth_password {{- with .AdditionalContent }}{{ marshalContent 4 . }}{{- end }} {{- end }} @@ -301,6 +329,8 @@ services: - frontend - data secrets: + - auth_token_key + - auth_cookie_key - postgres_password {{- with .AdditionalContent }}{{ marshalContent 4 . }}{{- end }} {{- end }} diff --git a/pkg/initialdata/initialdata.go b/pkg/initialdata/initialdata.go index ce68bcb..3c140b1 100644 --- a/pkg/initialdata/initialdata.go +++ b/pkg/initialdata/initialdata.go @@ -5,13 +5,11 @@ import ( "encoding/json" "fmt" "os" - "path" "strings" "github.com/OpenSlides/openslides-manage-service/pkg/connection" "github.com/OpenSlides/openslides-manage-service/pkg/fehler" "github.com/OpenSlides/openslides-manage-service/pkg/setpassword" - "github.com/OpenSlides/openslides-manage-service/pkg/setup" "github.com/OpenSlides/openslides-manage-service/pkg/shared" "github.com/OpenSlides/openslides-manage-service/proto" "github.com/spf13/cobra" @@ -103,7 +101,7 @@ type backendAction interface { } // InitialData sets initial data in the datastore. -func InitialData(ctx context.Context, in *proto.InitialDataRequest, runPath string, ba backendAction) (*proto.InitialDataResponse, error) { +func InitialData(ctx context.Context, in *proto.InitialDataRequest, superadminSecretFile string, ba backendAction) (*proto.InitialDataResponse, error) { initialData := in.Data if initialData == nil { // The backend expects at least an empty object. @@ -133,8 +131,7 @@ func InitialData(ctx context.Context, in *proto.InitialDataRequest, runPath stri return nil, fmt.Errorf("requesting backend action %q: %w", name, err) } - p := path.Join(runPath, setup.SecretsDirName, setup.SuperadminFileName) - if err := SetSuperadminPassword(ctx, p, ba); err != nil { + if err := SetSuperadminPassword(ctx, superadminSecretFile, ba); err != nil { return nil, fmt.Errorf("setting superadmin password: %w", err) } diff --git a/pkg/initialdata/initialdata_test.go b/pkg/initialdata/initialdata_test.go index ba66a6a..7712e20 100644 --- a/pkg/initialdata/initialdata_test.go +++ b/pkg/initialdata/initialdata_test.go @@ -114,7 +114,8 @@ func TestInitialDataServerAll(t *testing.T) { // Run tests t.Run("running the first time", func(t *testing.T) { - resp, err := initialdata.InitialData(ctx, in, testDir, ma) + p := path.Join(testDir, setup.SecretsDirName, setup.SuperadminFileName) + resp, err := initialdata.InitialData(ctx, in, p, ma) if err != nil { t.Fatalf("running InitialData() failed: %v", err) } diff --git a/pkg/server/server.go b/pkg/server/server.go index e95b5a9..7cb3717 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -29,8 +29,6 @@ import ( "google.golang.org/grpc/metadata" ) -const runDir = "/run" - // Run starts the manage server. func Run(cfg *Config) error { logger, err := shared.NewLogger(cfg.OpenSlidesLoglevel) @@ -105,7 +103,7 @@ func (s *srv) InitialData(ctx context.Context, in *proto.InitialDataRequest) (*p return nil, fmt.Errorf("getting internal auth password from file: %w", err) } a := backendaction.New(s.config.manageBackendActionURL(), pw, backendaction.ActionRoute) - return initialdata.InitialData(ctx, in, runDir, a) + return initialdata.InitialData(ctx, in, s.config.SuperadminPasswordFile, a) } @@ -208,23 +206,19 @@ type Config struct { // variables. The first value is the name of the environment variable. After // a comma the default value can be given. If no default value is given, then // an empty string is used. The type of a env field has to be string. - Port string `env:"MANAGE_PORT,9008"` - ManageAuthPasswordFile string `env:"MANAGE_AUTH_PASSWORD_FILE,/run/secrets/manage_auth_password"` + Port string `env:"MANAGE_PORT,9008"` + ManageAuthPasswordFile string `env:"MANAGE_AUTH_PASSWORD_FILE,/run/secrets/manage_auth_password"` + InternalAuthPasswordFile string `env:"INTERNAL_AUTH_PASSWORD_FILE,/run/secrets/internal_auth_password"` + SuperadminPasswordFile string `env:"SUPERADMIN_PASSWORD_FILE,/run/secrets/superadmin"` - // Hint: The env var for the host is MANAGE_ACTION_HOST but the env vars for - // protocol and port don't have the MANAGE_ prefix because the backend - // itself does not distiguish between an common backend container and a - // manage backend container. So protocol and port are the same for all backend containers. ManageActionProtocol string `env:"ACTION_PROTOCOL,http"` - ManageActionHost string `env:"MANAGE_ACTION_HOST,backendManage"` + ManageActionHost string `env:"ACTION_HOST,backendManage"` ManageActionPort string `env:"ACTION_PORT,9002"` DatastoreReaderProtocol string `env:"DATASTORE_READER_PROTOCOL,http"` DatastoreReaderHost string `env:"DATASTORE_READER_HOST,datastore-reader"` DatastoreReaderPort string `env:"DATASTORE_READER_PORT,9010"` - InternalAuthPasswordFile string `env:"INTERNAL_AUTH_PASSWORD_FILE,/run/secrets/internal_auth_password"` - OpenSlidesDevelopment string `env:"OPENSLIDES_DEVELOPMENT,0"` OpenSlidesLoglevel string `env:"OPENSLIDES_LOGLEVEL,info"` } diff --git a/pkg/setup/setup_test.go b/pkg/setup/setup_test.go index 014cd48..49391f6 100644 --- a/pkg/setup/setup_test.go +++ b/pkg/setup/setup_test.go @@ -469,31 +469,29 @@ version: "3.4" x-default-environment: &default-environment ACTION_HOST: backendAction ACTION_PORT: "9002" + AUTH_COOKIE_KEY_FILE: /run/secrets/auth_cookie_key AUTH_HOST: auth AUTH_PORT: "9004" + AUTH_TOKEN_KEY_FILE: /run/secrets/auth_token_key AUTOUPDATE_HOST: autoupdate AUTOUPDATE_PORT: "9012" CACHE_HOST: redis CACHE_PORT: "6379" - DATASTORE_DATABASE_HOST: postgres - DATASTORE_DATABASE_NAME: openslides - DATASTORE_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password - DATASTORE_DATABASE_PORT: "5432" - DATASTORE_DATABASE_USER: openslides + DATABASE_HOST: postgres + DATABASE_NAME: openslides + DATABASE_PASSWORD_FILE: /run/secrets/postgres_password + DATABASE_PORT: "5432" + DATABASE_USER: openslides DATASTORE_READER_HOST: datastoreReader DATASTORE_READER_PORT: "9010" DATASTORE_WRITER_HOST: datastoreWriter DATASTORE_WRITER_PORT: "9011" ICC_HOST: icc ICC_PORT: "9007" - ICC_REDIS_HOST: redis - ICC_REDIS_PORT: "6379" INTERNAL_AUTH_PASSWORD_FILE: /run/secrets/internal_auth_password - MANAGE_ACTION_HOST: backendManage MANAGE_AUTH_PASSWORD_FILE: /run/secrets/manage_auth_password MANAGE_HOST: manage MANAGE_PORT: "9008" - MEDIA_BLOCK_SIZE: "4096" MEDIA_DATABASE_HOST: postgres MEDIA_DATABASE_NAME: openslides MEDIA_DATABASE_PASSWORD_FILE: /run/secrets/postgres_password @@ -501,14 +499,16 @@ x-default-environment: &default-environment MEDIA_DATABASE_USER: openslides MEDIA_HOST: media MEDIA_PORT: "9006" - MEDIA_PRESENTER_HOST: backendPresenter - MEDIA_PRESENTER_PORT: "9003" MESSAGE_BUS_HOST: redis MESSAGE_BUS_PORT: "6379" OPENSLIDES_DEVELOPMENT: "false" OPENSLIDES_LOGLEVEL: info PRESENTER_HOST: backendPresenter PRESENTER_PORT: "9003" + RESTRICTER_URL: http://autoupdate:9012/internal/autoupdate + SEARCH_HOST: search + SEARCH_PORT: "9050" + SUPERADMIN_PASSWORD_FILE: /run/secrets/superadmin SYSTEM_URL: localhost:8000 VOTE_DATABASE_HOST: postgres VOTE_DATABASE_NAME: openslides @@ -517,8 +517,6 @@ x-default-environment: &default-environment VOTE_DATABASE_USER: openslides VOTE_HOST: vote VOTE_PORT: "9013" - VOTE_REDIS_HOST: redis - VOTE_REDIS_PORT: "6379" services: proxy: @@ -528,6 +526,7 @@ services: - backendAction - backendPresenter - autoupdate + - search - auth - media - icc @@ -552,6 +551,7 @@ services: - backendAction - backendPresenter - autoupdate + - search - auth - media - icc @@ -579,6 +579,7 @@ services: secrets: - auth_token_key - auth_cookie_key + - internal_auth_password - postgres_password backendPresenter: @@ -667,6 +668,22 @@ services: - auth_cookie_key - postgres_password + search: + image: ghcr.io/openslides/openslides/openslides-search:latest + depends_on: + - datastoreReader + - postgres + - autoupdate + environment: + << : *default-environment + networks: + - frontend + - data + secrets: + - auth_token_key + - auth_cookie_key + - postgres_password + auth: image: ghcr.io/openslides/openslides/openslides-auth:latest depends_on: @@ -680,6 +697,7 @@ services: secrets: - auth_token_key - auth_cookie_key + - internal_auth_password vote: image: ghcr.io/openslides/openslides/openslides-vote:latest @@ -716,6 +734,8 @@ services: - frontend - data secrets: + - auth_token_key + - auth_cookie_key - postgres_password icc: @@ -741,6 +761,7 @@ services: - backendManage environment: << : *default-environment + ACTION_HOST: backendManage networks: - frontend - data