Skip to content

Commit

Permalink
feat: Gitea drone (#33)
Browse files Browse the repository at this point in the history
Allowing drone to sync with gitea.
Co-authored-by: Marc Went <marc.went+git@redkubes.com>
  • Loading branch information
Maurice Faber authored May 22, 2021
1 parent cd6d90d commit 2b2d2ff
Show file tree
Hide file tree
Showing 12 changed files with 144 additions and 47 deletions.
22 changes: 8 additions & 14 deletions .env.sample
Original file line number Diff line number Diff line change
@@ -1,21 +1,14 @@
#######################################################
# Sample development values
#
# Sxample cluster base domain used: dev.gke.otomi.cloud
#
# Tasks will complain when a value is missing ;)
#######################################################

# DEBUG="*"

# OPTIONAL: Token to publish GitHub packages.
NPM_TOKEN=

# Harbor task
HARBOR_USER='admin'
HARBOR_PASSWORD=''
HARBOR_BASE_URL='http://127.0.0.1:8083/api/v2.0'
OIDC_CLIENT_SECRET=''
OIDC_ENDPOINT='https://keycloak.dev.eks.otomi.cloud'
OIDC_ENDPOINT='https://keycloak.dev.gke.otomi.cloud'
OIDC_VERIFY_CERT='true' # set to false when staging certs are used on the cluster
TEAM_NAMES='["team-otomi","team-chai","team-dev","team-demo"]'

Expand All @@ -27,9 +20,9 @@ IDP_ALIAS='redkubes-azure'
IDP_GROUP_OTOMI_ADMIN='e69ded30-0882-4490-8e0f-2e67625a0693'
IDP_GROUP_TEAM_ADMIN='3c63814c-59df-46c3-9a69-d9e1c3611097'
IDP_GROUP_MAPPINGS_TEAMS='{"team-demo":"28010af7-9535-4265-8689-50f51f8f2c87","team-dev":"3d2e6df8-c7c7-4d51-aa6a-9b7b5330d915","team-otomi":"0efd2f6d-fb8b-49a9-9507-54cd6e92c348"}'
IDP_USERNAME_CLAIM_MAPPER="${CLAIM.email}"
IDP_USERNAME_CLAIM_MAPPER='${CLAIM.email}'
IDP_SUB_CLAIM_MAPPER='oid'
KEYCLOAK_ADDRESS='https://keycloak.dev.eks.otomi.cloud'
KEYCLOAK_ADDRESS='https://keycloak.dev.gke.otomi.cloud'
KEYCLOAK_ADMIN='admin'
KEYCLOAK_ADMIN_PASSWORD=''
KEYCLOAK_REALM='master'
Expand All @@ -38,8 +31,9 @@ IDP_OIDC_URL='https://login.microsoftonline.com/$TENANT_ID'
REDIRECT_URIS='["http://localhost:8084/*","https://auth.dev.gke.otomi.cloud/*","https://otomi.dev.gke.otomi.cloud/*","https://harbor.dev.gke.otomi.cloud/*","https://apps.dev.gke.otomi.cloud/*","https://apps.team-dev.dev.gke.otomi.cloud/*","https://apps.team-demo.dev.gke.otomi.cloud/*","https://apps.team-admin.dev.gke.otomi.cloud/*","https://apps.team-otomi.dev.gke.otomi.cloud/*","https://apps.team-chai.dev.gke.otomi.cloud/*"]'

# Gitea task
GITEA_REPO='values'
GITEA_USER='otomi'
GITEA_PASSWORD=''
GITEA_URL='http://127.0.0.1:8082'
DRONE_URL='http://drone.dev.eks.otomi.cloud'
DRONE_URL='https://drone.dev.gke.otomi.cloud'

# Drone task
DRONE_TOKEN=''
15 changes: 15 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,21 @@ npm run task:(gitea*|harbor|keycloak|certs-aws|...)-dev

Or you can start them in the vscode debugger.

**Skipping TLS (like for staging certs):**

Run the next line in your shell to skip TLS cert validation, like when using self-signed certs like letsencrypt staging:
```bash
export NODE_TLS_REJECT_UNAUTHORIZED='0'
```

**Setting debug level:**

For all packages to turn on debug:
```bash
export DEBUG='*'
```
To limit scope please read [the debug docs](https://github.com/visionmedia/debug).

## Unit tests

There are not many unit tests, as the tasks are *very* robust and idempotent. You can run them as always with:
Expand Down
8 changes: 3 additions & 5 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 5 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@
"js-yaml": "^3.14.0",
"lodash": "^4.17.21",
"lowdb": "^1.0.0",
"morgan": "^1.10.0"
"morgan": "^1.10.0",
"ts-custom-error": "^3.2.0"
},
"description": "Tasks needed by the Otomi Container Platform to glue all the pieces together.",
"devDependencies": {
Expand Down Expand Up @@ -69,7 +70,6 @@
"sinon-chai": "^3.5.0",
"standard-version": "^9.0.0",
"supertest": "^4.0.2",
"ts-custom-error": "^3.2.0",
"ts-node": "^8.10.2",
"ts-node-dev": "^1.0.0-pre.56",
"typescript": "^4.2.4"
Expand Down Expand Up @@ -118,7 +118,10 @@
"tasks:keycloak": "node dist/tasks/keycloak/keycloak.js",
"tasks:keycloak-dev": "ts-node-dev ./src/tasks/keycloak/keycloak.ts",
"tasks:gitea": "node dist/tasks/gitea/gitea.js && node dist/tasks/gitea/gitea-drone-oauth.js",
"tasks:gitea-add-users": "node dist/tasks/gitea/gitea-add-users-to-team.js",
"tasks:gitea-dev": "ts-node-dev ./src/tasks/gitea/gitea.ts",
"tasks:gitea-add-users-dev": "ts-node-dev ./src/tasks/gitea/gitea-add-users-to-team.ts",
"tasks:gitea-drone-auth": "node dist/tasks/gitea/gitea-drone-oauth.js",
"tasks:gitea-drone-auth-dev": "ts-node-dev ./src/tasks/gitea/gitea-drone-oauth.ts",
"tasks:harbor": "node dist/tasks/harbor/harbor.js",
"tasks:harbor-dev": "ts-node-dev ./src/tasks/harbor/harbor.ts",
Expand Down
5 changes: 5 additions & 0 deletions src/tasks/common.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
export const orgName = 'otomi'
export const teamNameOwners = 'Owners'
export const teamNameViewer = 'otomi-viewer'
export const repoName = 'values'
export const username = 'otomi-admin'
7 changes: 4 additions & 3 deletions src/tasks/drone/drone.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import * as drone from 'drone-node'
import { doApiCall, handleErrors } from '../../utils'
import { cleanEnv, DRONE_URL, DRONE_TOKEN } from '../../validators'
import { orgName, repoName } from '../common'

const env = cleanEnv({
DRONE_URL,
Expand All @@ -24,13 +25,13 @@ async function main(): Promise<void> {
// https://discourse.drone.io/t/not-found-from-machine-user/7073/4?u=morriz

// Sync repos
await doApiCall(errors, 'Syncing repos', () => client.syncRepos())
// await doApiCall(errors, 'Syncing repos', () => client.syncRepos())

// Connect repo
await doApiCall(errors, 'Connecting repo', () => client.enableRepo('otomi-admin', 'values'))
await doApiCall(errors, 'Connecting repo', () => client.enableRepo(orgName, repoName))

// Update repo: this preconfigures the repo so that it only needs activating
// await doApiCall(errors, 'Updating repo', () => client.updateRepo(env.DRONE_OWNER, env.DRONE_REPO, settings))
await doApiCall(errors, 'Updating repo', () => client.updateRepo(orgName, repoName, {}))

handleErrors(errors)
}
Expand Down
4 changes: 1 addition & 3 deletions src/tasks/gitea/common.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
/* eslint-disable import/prefer-default-export */
import { CustomError } from 'ts-custom-error'

export class GiteaDroneError extends CustomError {}
export const orgName = 'otomi'
export const repoName = 'values'
export const username = 'otomi-admin'
58 changes: 58 additions & 0 deletions src/tasks/gitea/gitea-add-users-to-team.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import { OrganizationApi, AdminApi, User, Team } from '@redkubes/gitea-client-node'
import { doApiCall } from '../../utils'
import { cleanEnv, GITEA_PASSWORD, GITEA_URL } from '../../validators'
import { orgName, teamNameOwners, username } from '../common'

const env = cleanEnv({
GITEA_PASSWORD,
GITEA_URL,
})
const errors: string[] = []
export default async function main(): Promise<void> {
let giteaUrl = env.GITEA_URL
if (giteaUrl.endsWith('/')) {
giteaUrl = giteaUrl.slice(0, -1)
}
// create the org
const adminApi = new AdminApi(username, env.GITEA_PASSWORD, `${giteaUrl}/api/v1`)

const users: User[] = await doApiCall(errors, `Get all users`, () => adminApi.adminGetAllUsers())

const orgApi = new OrganizationApi(username, env.GITEA_PASSWORD, `${giteaUrl}/api/v1`)
await Promise.allSettled(
[teamNameOwners].map(
async (teamName): Promise<any[]> => {
const orgTeams = (await doApiCall(errors, 'Find team ID if exists', () =>
orgApi.orgListTeams(orgName),
)) as Team[]
const teamID = orgTeams.find((team) => team.name === teamName)!.id as number

const membersInTeam = (await doApiCall(errors, 'Find all members in this team', () =>
orgApi.orgListTeamMembers(teamID),
)) as User[]
const userIDsInTeam = membersInTeam.map((member) => member.id)
const membersNotInTeam: User[] = users.filter((user) => !userIDsInTeam.includes(user.id))

// eslint-disable-next-line no-restricted-syntax
return Promise.allSettled(
membersNotInTeam.map((member) =>
doApiCall(errors, `Add ${member.login} to ${teamName}`, () =>
orgApi.orgAddTeamMember(teamID, member.login as string),
),
),
)
},
),
)

if (errors.length) {
console.error(`Errors found: ${JSON.stringify(errors, null, 2)}`)
process.exit(1)
} else {
console.info('Success!')
}
}
// Run main only on execution, not on import (like tests)
if (typeof require !== 'undefined' && require.main === module) {
main()
}
3 changes: 2 additions & 1 deletion src/tasks/gitea/gitea-drone-oauth.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ import { UserApi, CreateOAuth2ApplicationOptions } from '@redkubes/gitea-client-

import { cleanEnv, GITEA_PASSWORD, GITEA_URL, DRONE_URL } from '../../validators'
import { createSecret, doApiCall, getApiClient, getSecret } from '../../utils'
import { username, GiteaDroneError } from './common'
import { GiteaDroneError } from './common'
import { username } from '../common'

const env = cleanEnv({
GITEA_PASSWORD,
Expand Down
47 changes: 33 additions & 14 deletions src/tasks/gitea/gitea.ts
Original file line number Diff line number Diff line change
@@ -1,36 +1,55 @@
import { CreateRepoOption, UserApi } from '@redkubes/gitea-client-node'
import { doApiCall, handleErrors } from '../../utils'

import { OrganizationApi, CreateRepoOption, CreateOrgOption, CreateTeamOption } from '@redkubes/gitea-client-node'
import { doApiCall } from '../../utils'
import { cleanEnv, GITEA_PASSWORD, GITEA_URL } from '../../validators'
import { repoName, username } from './common'
import { orgName, repoName, username, teamNameViewer } from '../common'

const env = cleanEnv({
GITEA_PASSWORD,
GITEA_URL,
})

const errors: string[] = []

export async function createTeam(orgApi: OrganizationApi): Promise<void> {
const readOnlyTeam: CreateTeamOption = {
...new CreateTeamOption(),
canCreateOrgRepo: false,
name: teamNameViewer,
includesAllRepositories: true,
permission: CreateTeamOption.PermissionEnum.Read,
units: ['repo.code'],
}
return doApiCall(
errors,
`Creating team "${teamNameViewer}" in org "${orgName}"`,
() => orgApi.orgCreateTeam(orgName, readOnlyTeam),
422,
)
}

export default async function main(): Promise<void> {
let giteaUrl = env.GITEA_URL
if (giteaUrl.endsWith('/')) {
giteaUrl = giteaUrl.slice(0, -1)
}

// create the org
const userApi = new UserApi(username, env.GITEA_PASSWORD, `${giteaUrl}/api/v1`)
// const orgApi = new OrganizationApi(username, env.GITEA_PASSWORD, `${giteaUrl}/api/v1`)
// const orgOption = { ...new CreateOrgOption(), username: orgName, repoAdminChangeTeamAccess: true }
// await doApiCall(errors, `Creating org "${orgName}"`, () => orgApi.orgCreate(orgOption), 422)
const orgApi = new OrganizationApi(username, env.GITEA_PASSWORD, `${giteaUrl}/api/v1`)
const orgOption = { ...new CreateOrgOption(), username: orgName, repoAdminChangeTeamAccess: true }
await doApiCall(errors, `Creating org "${orgName}"`, () => orgApi.orgCreate(orgOption), 422)

// await createTeam(orgApi)
// create the org repo
const repoOption = { ...new CreateRepoOption(), autoInit: false, name: repoName, _private: true }
await doApiCall(errors, `Creating repo "${repoName}"`, () => userApi.createCurrentUserRepo(repoOption))
// await doApiCall(errors, `Creating org repo "${repoName}"`, () => orgApi.createOrgRepo(orgName, repoOption))
await doApiCall(errors, `Creating org repo "${repoName}"`, () => orgApi.createOrgRepo(orgName, repoOption))
// add the

handleErrors(errors)
if (errors.length) {
console.error(`Errors found: ${JSON.stringify(errors, null, 2)}`)
process.exit(1)
} else {
console.info('Success!')
}
}
// Run main only on execution, not on import (like tests)

if (typeof require !== 'undefined' && require.main === module) {
main()
}
10 changes: 6 additions & 4 deletions src/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import http from 'http'
import { findIndex, mapValues } from 'lodash'
import { CoreV1Api, KubeConfig, V1Secret, V1ObjectMeta, V1ServiceAccount } from '@kubernetes/client-node'

let apiClient
let apiClient: CoreV1Api

export function getApiClient(): CoreV1Api {
if (apiClient) return apiClient
Expand Down Expand Up @@ -32,13 +32,15 @@ export function ensure<T>(argument: T | undefined | null, message = 'This value

export async function createSecret(name: string, namespace: string, data: object): Promise<void> {
const b64enc = (val): string => Buffer.from(`${val}`).toString('base64')
const secret = {
const secret: V1Secret = {
...new V1Secret(),
metadata: { ...new V1ObjectMeta(), name },
data: mapValues(data, b64enc),
data: mapValues(data, b64enc) as {
[key: string]: string
},
}

await apiClient.createNamespacedSecret(namespace, secret)
await getApiClient().createNamespacedSecret(namespace, secret)
console.info(`New secret ${name} has been created in the namespace ${namespace}`)
}

Expand Down
5 changes: 4 additions & 1 deletion src/validators.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ export const DRONE_URL = str({ desc: 'The public url of the drone server' })
export const GITEA_PASSWORD = str({ desc: 'The gitea admin password' })
export const GITEA_URL = url({ desc: 'The gitea core service url' })
export const KEYCLOAK_ADDRESS = str({ desc: 'The Keycloak Server address' })
export const KEYCLOAK_ADMIN = str({ desc: 'Default admin username for KeyCloak Server' })
export const KEYCLOAK_ADMIN = str({ desc: 'Default admin username for KeyCloak Server', default: 'admin' })
export const KEYCLOAK_ADMIN_PASSWORD = str({ desc: 'Default password for admin' })
export const KEYCLOAK_CLIENT_ID = str({ desc: 'Default Keycloak Client', default: 'otomi' })
export const KEYCLOAK_CLIENT_SECRET = str({ desc: 'The keycloak client secret' })
Expand All @@ -51,3 +51,6 @@ export function cleanEnv<T>(
return process.env as Readonly<T> & CleanEnv & { readonly [varName: string]: string | undefined }
return clean(env, validators, options) as Readonly<T> & CleanEnv & { readonly [varName: string]: string | undefined }
}

// And to avoid npm trying to check for updates
process.env.NO_UPDATE_NOTIFIER = 'true'

0 comments on commit 2b2d2ff

Please sign in to comment.