diff --git a/backend/config/custom-environment-variables.json b/backend/config/custom-environment-variables.json index 509f4f312b..da1646d7aa 100644 --- a/backend/config/custom-environment-variables.json +++ b/backend/config/custom-environment-variables.json @@ -129,6 +129,7 @@ "appId": "CROWD_GITHUB_APP_ID", "clientId": "CROWD_GITHUB_CLIENT_ID", "clientSecret": "CROWD_GITHUB_CLIENT_SECRET", + "callbackUrl": "CROWD_GITHUB_CALLBACK_URL", "privateKey": "CROWD_GITHUB_PRIVATE_KEY", "webhookSecret": "CROWD_GITHUB_WEBHOOK_SECRET", "isCommitDataEnabled": "CROWD_GITHUB_IS_COMMIT_DATA_ENABLED" @@ -180,8 +181,7 @@ "secretAccessKey": "CROWD_OPENSEARCH_AWS_SECRET_ACCESS_KEY" }, "auth0": { - "domain": "CROWD_AUTH0_DOMAIN", "clientId": "CROWD_AUTH0_CLIENT_ID", - "cert": "CROWD_AUTH0_CERT" + "jwks": "CROWD_AUTH0_JWKS" } } diff --git a/backend/package-lock.json b/backend/package-lock.json index dce59e879c..876cd6719a 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -65,6 +65,7 @@ "html-to-text": "^8.2.1", "json2csv": "^5.0.7", "jsonwebtoken": "8.5.1", + "jwks-rsa": "^3.0.1", "lodash": "4.17.21", "moment": "2.29.4", "moment-timezone": "^0.5.34", @@ -74,6 +75,7 @@ "openapi-comment-parser": "^1.0.0", "passport": "0.6.0", "passport-facebook": "3.0.0", + "passport-github2": "^0.1.12", "passport-google-oauth": "2.0.0", "passport-google-oauth20": "^2.0.0", "passport-slack": "0.0.7", @@ -22200,6 +22202,7 @@ }, "node_modules/erlpack": { "version": "0.1.4", + "hasInstallScript": true, "license": "MIT", "dependencies": { "bindings": "^1.5.0", @@ -26517,6 +26520,14 @@ "version": "1.1.0", "license": "MIT" }, + "node_modules/jose": { + "version": "4.14.4", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.14.4.tgz", + "integrity": "sha512-j8GhLiKmUAh+dsFXlX1aJCbt5KMibuKb+d7j1JaOJG6s2UjX1PQlW+OKB/sD4a/5ZYF4RcmYmLSndOoU3Lt/3g==", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/js-beautify": { "version": "1.14.8", "license": "MIT", @@ -26767,6 +26778,22 @@ "safe-buffer": "^5.0.1" } }, + "node_modules/jwks-rsa": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-3.0.1.tgz", + "integrity": "sha512-UUOZ0CVReK1QVU3rbi9bC7N5/le8ziUj0A2ef1Q0M7OPD2KvjEYizptqIxGIo6fSLYDkqBrazILS18tYuRc8gw==", + "dependencies": { + "@types/express": "^4.17.14", + "@types/jsonwebtoken": "^9.0.0", + "debug": "^4.3.4", + "jose": "^4.10.4", + "limiter": "^1.1.5", + "lru-memoizer": "^2.1.4" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/jws": { "version": "4.0.0", "license": "MIT", @@ -26847,6 +26874,11 @@ "node": ">= 0.8.0" } }, + "node_modules/limiter": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz", + "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA==" + }, "node_modules/lines-and-columns": { "version": "1.2.4", "dev": true, @@ -26904,6 +26936,11 @@ "version": "4.3.0", "license": "MIT" }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" + }, "node_modules/lodash.debounce": { "version": "4.0.8", "dev": true, @@ -27056,6 +27093,29 @@ "yallist": "^3.0.2" } }, + "node_modules/lru-memoizer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/lru-memoizer/-/lru-memoizer-2.2.0.tgz", + "integrity": "sha512-QfOZ6jNkxCcM/BkIPnFsqDhtrazLRsghi9mBwFAzol5GCvj4EkFT899Za3+QwikCg5sRX8JstioBDwOxEyzaNw==", + "dependencies": { + "lodash.clonedeep": "^4.5.0", + "lru-cache": "~4.0.0" + } + }, + "node_modules/lru-memoizer/node_modules/lru-cache": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.2.tgz", + "integrity": "sha512-uQw9OqphAGiZhkuPlpFGmdTU2tEuhxTourM/19qGJrxBPHAr/f8BT1a0i/lOclESnGatdJG/UCkP9kZB/Lh1iw==", + "dependencies": { + "pseudomap": "^1.0.1", + "yallist": "^2.0.0" + } + }, + "node_modules/lru-memoizer/node_modules/yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" + }, "node_modules/lru-queue": { "version": "0.1.0", "license": "MIT", @@ -28864,6 +28924,17 @@ "node": ">= 0.4.0" } }, + "node_modules/passport-github2": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/passport-github2/-/passport-github2-0.1.12.tgz", + "integrity": "sha512-3nPUCc7ttF/3HSP/k9sAXjz3SkGv5Nki84I05kSQPo01Jqq1NzJACgMblCK0fGcv9pKCG/KXU3AJRDGLqHLoIw==", + "dependencies": { + "passport-oauth2": "1.x.x" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/passport-google-oauth": { "version": "2.0.0", "license": "MIT", @@ -33415,6 +33486,7 @@ }, "node_modules/zlib-sync": { "version": "0.1.8", + "hasInstallScript": true, "license": "MIT", "dependencies": { "nan": "^2.17.0" @@ -62452,6 +62524,11 @@ "join-component": { "version": "1.1.0" }, + "jose": { + "version": "4.14.4", + "resolved": "https://registry.npmjs.org/jose/-/jose-4.14.4.tgz", + "integrity": "sha512-j8GhLiKmUAh+dsFXlX1aJCbt5KMibuKb+d7j1JaOJG6s2UjX1PQlW+OKB/sD4a/5ZYF4RcmYmLSndOoU3Lt/3g==" + }, "js-beautify": { "version": "1.14.8", "requires": { @@ -62618,6 +62695,19 @@ "safe-buffer": "^5.0.1" } }, + "jwks-rsa": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/jwks-rsa/-/jwks-rsa-3.0.1.tgz", + "integrity": "sha512-UUOZ0CVReK1QVU3rbi9bC7N5/le8ziUj0A2ef1Q0M7OPD2KvjEYizptqIxGIo6fSLYDkqBrazILS18tYuRc8gw==", + "requires": { + "@types/express": "^4.17.14", + "@types/jsonwebtoken": "^9.0.0", + "debug": "^4.3.4", + "jose": "^4.10.4", + "limiter": "^1.1.5", + "lru-memoizer": "^2.1.4" + } + }, "jws": { "version": "4.0.0", "requires": { @@ -62668,6 +62758,11 @@ "type-check": "~0.4.0" } }, + "limiter": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz", + "integrity": "sha512-FWWMIEOxz3GwUI4Ts/IvgVy6LPvoMPgjMdQ185nN6psJyBJ4yOpzqm695/h5umdLJg2vW3GR5iG11MAkR2AzJA==" + }, "lines-and-columns": { "version": "1.2.4", "dev": true @@ -62705,6 +62800,11 @@ "lodash.camelcase": { "version": "4.3.0" }, + "lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" + }, "lodash.debounce": { "version": "4.0.8", "dev": true @@ -62804,6 +62904,31 @@ "yallist": "^3.0.2" } }, + "lru-memoizer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/lru-memoizer/-/lru-memoizer-2.2.0.tgz", + "integrity": "sha512-QfOZ6jNkxCcM/BkIPnFsqDhtrazLRsghi9mBwFAzol5GCvj4EkFT899Za3+QwikCg5sRX8JstioBDwOxEyzaNw==", + "requires": { + "lodash.clonedeep": "^4.5.0", + "lru-cache": "~4.0.0" + }, + "dependencies": { + "lru-cache": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.2.tgz", + "integrity": "sha512-uQw9OqphAGiZhkuPlpFGmdTU2tEuhxTourM/19qGJrxBPHAr/f8BT1a0i/lOclESnGatdJG/UCkP9kZB/Lh1iw==", + "requires": { + "pseudomap": "^1.0.1", + "yallist": "^2.0.0" + } + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==" + } + } + }, "lru-queue": { "version": "0.1.0", "requires": { @@ -63970,6 +64095,14 @@ "passport-oauth2": "1.x.x" } }, + "passport-github2": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/passport-github2/-/passport-github2-0.1.12.tgz", + "integrity": "sha512-3nPUCc7ttF/3HSP/k9sAXjz3SkGv5Nki84I05kSQPo01Jqq1NzJACgMblCK0fGcv9pKCG/KXU3AJRDGLqHLoIw==", + "requires": { + "passport-oauth2": "1.x.x" + } + }, "passport-google-oauth": { "version": "2.0.0", "requires": { diff --git a/backend/package.json b/backend/package.json index 14f4e4ffce..542765a516 100644 --- a/backend/package.json +++ b/backend/package.json @@ -99,6 +99,7 @@ "html-to-text": "^8.2.1", "json2csv": "^5.0.7", "jsonwebtoken": "8.5.1", + "jwks-rsa": "^3.0.1", "lodash": "4.17.21", "moment": "2.29.4", "moment-timezone": "^0.5.34", @@ -108,6 +109,7 @@ "openapi-comment-parser": "^1.0.0", "passport": "0.6.0", "passport-facebook": "3.0.0", + "passport-github2": "^0.1.12", "passport-google-oauth": "2.0.0", "passport-google-oauth20": "^2.0.0", "passport-slack": "0.0.7", diff --git a/backend/src/api/auth/authSocial.ts b/backend/src/api/auth/authSocial.ts index 77951211e3..97d9b29f0f 100644 --- a/backend/src/api/auth/authSocial.ts +++ b/backend/src/api/auth/authSocial.ts @@ -1,6 +1,6 @@ import passport from 'passport' import { getServiceChildLogger } from '@crowd/logging' -import { API_CONFIG, GOOGLE_CONFIG } from '../../conf' +import { API_CONFIG, GITHUB_CONFIG, GOOGLE_CONFIG } from '../../conf' import AuthService from '../../services/auth/authService' const log = getServiceChildLogger('AuthSocial') @@ -46,6 +46,26 @@ export default (app, routes) => { })(req, res) }) } + + if (GITHUB_CONFIG.clientId) { + routes.get( + '/auth/social/github', + passport.authenticate('github', { + scope: ['user:email', 'read:user'], + session: false, + }), + () => { + // The request will be redirected for authentication, so this + // function will not be called. + }, + ) + + routes.get('/auth/social/github/callback', (req, res) => { + passport.authenticate('github', (err, jwtToken) => { + handleCallback(res, err, jwtToken) + })(req, res) + }) + } } function handleCallback(res, err, jwtToken) { diff --git a/backend/src/api/auth/ssoCallback.ts b/backend/src/api/auth/ssoCallback.ts index b3ab5c3a71..a8b68e5172 100644 --- a/backend/src/api/auth/ssoCallback.ts +++ b/backend/src/api/auth/ssoCallback.ts @@ -1,31 +1,44 @@ import jwt from 'jsonwebtoken' +import jwksClient from 'jwks-rsa' import AuthService from '../../services/auth/authService' import { AUTH0_CONFIG } from '../../conf' import Error401 from '../../errors/Error401' +const jwks = jwksClient({ + jwksUri: AUTH0_CONFIG.jwks, + cache: true, + cacheMaxEntries: 5, + cacheMaxAge: 86400000, +}) + +async function getKey(header, callback) { + jwks.getSigningKey(header.kid, (err, key: any) => { + const signingKey = key.publicKey || key.rsaPublicKey + callback(null, signingKey) + }) +} + export default async (req, res) => { const { idToken, invitationToken, tenantId } = req.body try { const verifyToken = new Promise((resolve, reject) => { - const publicKey = AUTH0_CONFIG.cert.replaceAll('"', '').replace(/\\n/g, '\n') - jwt.verify(idToken, publicKey, { algorithms: ['RS256'] }, (err, decoded) => { - // If error verifying token + jwt.verify(idToken, getKey, { algorithms: ['RS256'] }, (err, decoded) => { if (err) { reject(new Error401()) } - // If token matches auth0 validation criteria - const { aud, iss } = decoded as any - if (aud !== AUTH0_CONFIG.clientId || !iss.includes(AUTH0_CONFIG.domain)) { + const { aud } = decoded as any + + if (aud !== AUTH0_CONFIG.clientId) { reject(new Error401()) } - // If token validation passed resolve(decoded) }) }) const data: any = await verifyToken + // Signin with data const token: string = await AuthService.signinFromSSO( 'auth0', diff --git a/backend/src/api/webhooks/github.ts b/backend/src/api/webhooks/github.ts index 8746f6b42a..dcafb145f5 100644 --- a/backend/src/api/webhooks/github.ts +++ b/backend/src/api/webhooks/github.ts @@ -35,7 +35,7 @@ export default async (req, res) => { await sendNodeWorkerMessage( integration.tenantId, - new NodeWorkerProcessWebhookMessage(integration.tenantId, result.id), + new NodeWorkerProcessWebhookMessage(integration.tenantId, result.id, undefined, true), ) await req.responseHandler.success(req, res, {}, 204) diff --git a/backend/src/conf/configTypes.ts b/backend/src/conf/configTypes.ts index a79c3e84d7..b9e0417feb 100644 --- a/backend/src/conf/configTypes.ts +++ b/backend/src/conf/configTypes.ts @@ -79,9 +79,8 @@ export interface ApiConfiguration { } export interface Auth0Configuration { - domain: string clientId: string - cert: string + jwks: string } export interface PlansConfiguration { @@ -144,6 +143,7 @@ export interface GithubConfiguration { webhookSecret: string isCommitDataEnabled: string globalLimit?: number + callbackUrl: string } export interface SendgridConfiguration { diff --git a/backend/src/cubejs/schema/Members.js b/backend/src/cubejs/schema/Members.js index d9e7bf567a..920cce0798 100644 --- a/backend/src/cubejs/schema/Members.js +++ b/backend/src/cubejs/schema/Members.js @@ -28,6 +28,7 @@ cube(`Members`, { Members.isBot, Members.isOrganization, Segments.id, + Activities.platform, ], timeDimension: Members.joinedAt, granularity: `day`, @@ -36,17 +37,54 @@ cube(`Members`, { }, }, - ActiveMembers: { + MembersByJoinedAtPure: { measures: [Members.count], dimensions: [ Members.score, Members.location, Members.tenantId, + Members.isTeamMember, + Members.isBot, + Members.isOrganization, + Segments.id, + ], + timeDimension: Members.joinedAt, + granularity: `day`, + refreshKey: { + every: `10 minute`, + }, + }, + + MembersByJoinedAtTags: { + measures: [Members.count], + dimensions: [ + Members.score, + Members.location, + Members.tenantId, + Members.isTeamMember, + Members.isBot, + Members.isOrganization, + Segments.id, Tags.name, + ], + timeDimension: Members.joinedAt, + granularity: `day`, + refreshKey: { + every: `10 minute`, + }, + }, + + MembersByJoinedAtPlatform: { + measures: [Members.count], + dimensions: [ + Members.score, + Members.location, + Members.tenantId, Members.isTeamMember, Members.isBot, Members.isOrganization, Segments.id, + Activities.platform, ], timeDimension: Members.joinedAt, granularity: `day`, @@ -55,9 +93,11 @@ cube(`Members`, { }, }, - MembersActivities: { + MembersByActivityPure: { measures: [Members.count], dimensions: [ + Members.score, + Members.location, Members.tenantId, Members.isTeamMember, Members.isBot, @@ -71,16 +111,17 @@ cube(`Members`, { }, }, - MActivitiesDupDimensions: { + MembersByActivityPlatform: { measures: [Members.count], dimensions: [ + Members.score, + Members.location, Members.tenantId, - Activities.platform, - Activities.type, Members.isTeamMember, Members.isBot, Members.isOrganization, Segments.id, + Activities.platform, ], timeDimension: Activities.date, granularity: `day`, @@ -89,17 +130,19 @@ cube(`Members`, { }, }, - MembersTags: { + MembersByActivityActivityType: { measures: [Members.count], dimensions: [ + Members.score, + Members.location, Members.tenantId, - Tags.name, Members.isTeamMember, Members.isBot, Members.isOrganization, Segments.id, + Activities.type, ], - timeDimension: Members.joinedAt, + timeDimension: Activities.date, granularity: `day`, refreshKey: { every: `10 minute`, diff --git a/backend/src/cubejs/schema/Organizations.js b/backend/src/cubejs/schema/Organizations.js index 1bffe9ba73..c49387734f 100644 --- a/backend/src/cubejs/schema/Organizations.js +++ b/backend/src/cubejs/schema/Organizations.js @@ -5,13 +5,25 @@ cube(`Organizations`, { preAggregations: { newOrganizations: { measures: [Organizations.count], - dimensions: [Organizations.tenantId, Members.isTeamMember, Members.isBot, Segments.id], + dimensions: [ + Organizations.tenantId, + Members.isTeamMember, + Members.isBot, + Segments.id, + Activities.platform, + ], timeDimension: Organizations.joinedAt, granularity: `day`, }, activeOrganizations: { measures: [Organizations.count], - dimensions: [Organizations.tenantId, Members.isTeamMember, Members.isBot, Segments.id], + dimensions: [ + Organizations.tenantId, + Members.isTeamMember, + Members.isBot, + Segments.id, + Activities.platform, + ], timeDimension: Activities.date, granularity: `day`, }, diff --git a/backend/src/database/migrations/U1688636697__employment-history.sql b/backend/src/database/migrations/U1688636697__employment-history.sql new file mode 100644 index 0000000000..3ed96a0207 --- /dev/null +++ b/backend/src/database/migrations/U1688636697__employment-history.sql @@ -0,0 +1,3 @@ +ALTER TABLE "memberOrganizations" DROP COLUMN "dateStart"; +ALTER TABLE "memberOrganizations" DROP COLUMN "dateEnd"; +ALTER TABLE "memberOrganizations" DROP COLUMN "title"; diff --git a/backend/src/database/migrations/U1688653004__convert-employment-history.sql b/backend/src/database/migrations/U1688653004__convert-employment-history.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/database/migrations/U1689175138__employment-history-segments.sql b/backend/src/database/migrations/U1689175138__employment-history-segments.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/database/migrations/V1688636697__employment-history.sql b/backend/src/database/migrations/V1688636697__employment-history.sql new file mode 100644 index 0000000000..6fb47b3ba6 --- /dev/null +++ b/backend/src/database/migrations/V1688636697__employment-history.sql @@ -0,0 +1,3 @@ +ALTER TABLE "memberOrganizations" ADD COLUMN "dateStart" TIMESTAMP WITH TIME ZONE NULL; +ALTER TABLE "memberOrganizations" ADD COLUMN "dateEnd" TIMESTAMP WITH TIME ZONE NULL; +ALTER TABLE "memberOrganizations" ADD COLUMN "title" VARCHAR(255) NULL; diff --git a/backend/src/database/migrations/V1688653004__convert-employment-history.sql b/backend/src/database/migrations/V1688653004__convert-employment-history.sql new file mode 100644 index 0000000000..050d9276fe --- /dev/null +++ b/backend/src/database/migrations/V1688653004__convert-employment-history.sql @@ -0,0 +1,73 @@ +DO $$ +DECLARE + _member RECORD; + _member_id UUID; + _org_id UUID; + _exp JSON; + _start_date TIMESTAMP WITH TIME ZONE; + _end_date TIMESTAMP WITH TIME ZONE; +BEGIN + CREATE OR REPLACE FUNCTION __convert_iso_string_to_timestamp(iso_string text) RETURNS timestamp AS $inner$ + BEGIN + IF iso_string = 'Present' THEN + RETURN NULL; + ELSIF (length(iso_string) = 4) THEN -- Only year + RETURN to_timestamp(iso_string, 'YYYY'); + ELSIF (length(iso_string) = 7) THEN -- Year and month + RETURN to_timestamp(iso_string, 'YYYY-MM'); + ELSE -- Full timestamp + RETURN iso_string::TIMESTAMP WITH TIME ZONE; + END IF; + END; + $inner$ LANGUAGE plpgsql; + + FOR _member IN SELECT id, "tenantId", attributes->'workExperiences'->'default' AS exp FROM members WHERE (attributes->>'workExperiences') IS NOT NULL + LOOP + _member_id := _member.id; + + FOR _exp IN SELECT * FROM json_array_elements(_member.exp::JSON) + LOOP + -- { + -- "title": "Senior Developer Evangelist", + -- "company": "Clarifai", + -- "endDate": "2019-05-01T00:00:00Z", + -- "location": "San Francisco, California", + -- "startDate": "2018-08-01T00:00:00Z" + -- } + + _org_id := (SELECT id FROM organizations WHERE "tenantId" = _member."tenantId" AND name ILIKE '%' || (_exp->>'company')::TEXT || '%' LIMIT 1); + IF _org_id IS NULL THEN + INSERT INTO organizations (id, "updatedAt", "createdAt", "tenantId", name) + VALUES (uuid_generate_v4(), NOW(), NOW(), _member."tenantId", _exp ->> 'company') + RETURNING id INTO _org_id; + + RAISE NOTICE 'Created org: %', _org_id; + ELSE + RAISE NOTICE 'Found org: %', _org_id; + END IF; + + _start_date := __convert_iso_string_to_timestamp(_exp ->> 'startDate'); + _end_date := __convert_iso_string_to_timestamp(_exp ->> 'endDate'); + + -- if row in `memberOrganizations` exists, skip + IF EXISTS (SELECT 1 FROM "memberOrganizations" WHERE "memberId" = _member_id AND "organizationId" = _org_id) THEN + UPDATE "memberOrganizations" + SET "updatedAt" = NOW(), + "dateStart" = _start_date, + "dateEnd" = _end_date, + "title" = _exp->>'title' + WHERE "memberId" = _member_id AND "organizationId" = _org_id; + RAISE NOTICE 'Found member org: %', _org_id; + ELSE + INSERT INTO "memberOrganizations" ("updatedAt", "createdAt", "memberId", "organizationId", "dateStart", "dateEnd", "title") + VALUES (NOW(), NOW(), _member_id, _org_id, _start_date, _end_date, _exp->>'title'); + RAISE NOTICE 'Created member org: %', _org_id; + END IF; + + RAISE NOTICE 'member id: %, exp: %', _member_id, _exp; + END LOOP; + END LOOP; + + DROP FUNCTION IF EXISTS __convert_iso_string_to_timestamp(text); +END $$ LANGUAGE PLPGSQL; + diff --git a/backend/src/database/migrations/V1689175138__employment-history-segments.sql b/backend/src/database/migrations/V1689175138__employment-history-segments.sql new file mode 100644 index 0000000000..ff9b2e044a --- /dev/null +++ b/backend/src/database/migrations/V1689175138__employment-history-segments.sql @@ -0,0 +1,11 @@ +INSERT INTO "organizationSegments" ("organizationId", "segmentId", "tenantId") +SELECT + mo."organizationId", + ms."segmentId", + ms."tenantId" +FROM "memberOrganizations" mo +JOIN members m ON mo."memberId" = m.id +JOIN "memberSegments" ms ON ms."memberId" = m.id +LEFT JOIN "organizationSegments" os ON os."organizationId" = mo."organizationId" +WHERE os."segmentId" IS NULL +ON CONFLICT DO NOTHING; diff --git a/backend/src/database/repositories/__tests__/memberRepository.test.ts b/backend/src/database/repositories/__tests__/memberRepository.test.ts index 5b13e1286a..1f35f87a27 100644 --- a/backend/src/database/repositories/__tests__/memberRepository.test.ts +++ b/backend/src/database/repositories/__tests__/memberRepository.test.ts @@ -2491,8 +2491,9 @@ describe('MemberRepository tests', () => { ) const member2 = members.rows.find((m) => m.username[PlatformType.SLACK].includes('test2')) expect(members.rows.length).toEqual(1) - expect(member2.organizations[0].name).toEqual('crowd.dev') - expect(member2.organizations[1].name).toEqual('pied piper') + expect(member2.organizations.map((o) => o.name)).toEqual( + expect.arrayContaining(['crowd.dev', 'pied piper']), + ) }) it('is successfully finding and counting all members, and scoreRange is gte than 7', async () => { @@ -3208,7 +3209,9 @@ describe('MemberRepository tests', () => { member1.createdAt = member1.createdAt.toISOString().split('T')[0] member1.updatedAt = member1.updatedAt.toISOString().split('T')[0] - member1.organizations = member1.organizations.map((i) => i.get({ plain: true })) + member1.organizations = member1.organizations.map((i) => + SequelizeTestUtils.objectWithoutKey(i.get({ plain: true }), ['memberOrganizations']), + ) // // sort member organizations by createdAt // member1.organizations.sort((a, b) => { diff --git a/backend/src/database/repositories/memberRepository.ts b/backend/src/database/repositories/memberRepository.ts index e998493441..2fa8bfd2e4 100644 --- a/backend/src/database/repositories/memberRepository.ts +++ b/backend/src/database/repositories/memberRepository.ts @@ -44,6 +44,7 @@ import { mapUsernameToIdentities, } from './types/memberTypes' import Error400 from '../../errors/Error400' +import OrganizationRepository from './organizationRepository' const { Op } = Sequelize @@ -127,9 +128,13 @@ class MemberRepository { await record.setTags(data.tags || [], { transaction, }) - await record.setOrganizations(data.organizations || [], { + + await MemberRepository.updateMemberOrganizations( + record, + data.organizations, transaction, - }) + options, + ) await record.setTasks(data.tasks || [], { transaction, @@ -653,11 +658,12 @@ class MemberRepository { }) } - if (data.organizations) { - await record.setOrganizations(data.organizations || [], { - transaction, - }) - } + await MemberRepository.updateMemberOrganizations( + record, + data.organizations, + transaction, + options, + ) if (data.noMerge) { await record.setNoMerge(data.noMerge || [], { @@ -947,6 +953,17 @@ class MemberRepository { attributes: ['id', 'name'], as: 'organizations', order: [['createdAt', 'ASC']], + through: { + attributes: [ + 'memberId', + 'organizationId', + 'createdAt', + 'updatedAt', + 'dateStart', + 'dateEnd', + 'title', + ], + }, }, { model: options.database.segment, @@ -981,6 +998,8 @@ class MemberRepository { } const data = record.get({ plain: returnPlain }) + MemberRepository.sortOrganizations(data.organizations) + const identities = (await this.getIdentities([data.id], options)).get(data.id) data.username = {} @@ -3137,8 +3156,9 @@ class MemberRepository { output.organizations = await record.getOrganizations({ transaction, order: [['createdAt', 'ASC']], - joinTableAttributes: [], + joinTableAttributes: ['dateStart', 'dateEnd', 'title'], }) + MemberRepository.sortOrganizations(output.organizations) output.tasks = await record.getTasks({ transaction, @@ -3181,6 +3201,87 @@ class MemberRepository { return output } + + static async updateMemberOrganizations( + record, + organizations, + transaction, + options: IRepositoryOptions, + ) { + if (!organizations) { + return + } + + await record.setOrganizations([], { transaction }) + for (const item of organizations) { + const org = typeof item === 'string' ? { id: item } : item + await MemberRepository.createOrUpdateWorkExperience( + { + memberId: record.id, + organizationId: org.id, + title: org.title, + dateStart: org.startDate, + dateEnd: org.endDate, + }, + { + transaction, + ...options, + }, + ) + await OrganizationRepository.includeOrganizationToSegments(org.id, { + transaction, + ...options, + }) + } + } + + static async createOrUpdateWorkExperience( + { memberId, organizationId, title, dateStart, dateEnd }, + options: IRepositoryOptions, + ) { + const seq = SequelizeRepository.getSequelize(options) + const transaction = SequelizeRepository.getTransaction(options) + + const query = ` + INSERT INTO "memberOrganizations" ("memberId", "organizationId", "createdAt", "updatedAt", "title", "dateStart", "dateEnd") + VALUES (:memberId, :organizationId, NOW(), NOW(), :title, :dateStart, :dateEnd) + ON CONFLICT ("memberId", "organizationId") DO UPDATE + SET "title" = :title, "dateStart" = :dateStart, "dateEnd" = :dateEnd + ` + + await seq.query(query, { + replacements: { + memberId, + organizationId, + title: title || null, + dateStart: dateStart || null, + dateEnd: dateEnd || null, + }, + type: QueryTypes.INSERT, + transaction, + }) + } + + static sortOrganizations(organizations) { + organizations.sort((a, b) => { + a = a.dataValues ? a.get({ plain: true }) : {} + b = b.dataValues ? b.get({ plain: true }) : {} + + const aDate = a.memberOrganizations?.dateStart + const bDate = b.memberOrganizations?.dateStart + + if (aDate && bDate) { + return bDate.getTime() - aDate.getTime() + } + if (!aDate && !bDate) { + return 0 + } + if (!bDate) { + return 1 + } + return -1 + }) + } } export default MemberRepository diff --git a/backend/src/database/repositories/segmentRepository.ts b/backend/src/database/repositories/segmentRepository.ts index 852c48138f..5c357eca28 100644 --- a/backend/src/database/repositories/segmentRepository.ts +++ b/backend/src/database/repositories/segmentRepository.ts @@ -493,7 +493,7 @@ class SegmentRepository extends RepositoryBase< } if (criteria.filter?.parentSlug) { - searchQuery += ` AND s."parentSlug" ilike :parent_slug ` + searchQuery += ` AND s."parentSlug" = :parent_slug ` } const projects = await this.options.database.sequelize.query( @@ -560,11 +560,11 @@ class SegmentRepository extends RepositoryBase< } if (criteria.filter?.parentSlug) { - searchQuery += ` AND s."parentSlug" ilike :parent_slug ` + searchQuery += ` AND s."parentSlug" = :parent_slug ` } if (criteria.filter?.grandparentSlug) { - searchQuery += ` AND s."grandparentSlug" ilike :grandparent_slug ` + searchQuery += ` AND s."grandparentSlug" = :grandparent_slug ` } const subprojects = await this.options.database.sequelize.query( diff --git a/backend/src/middlewares/passportStrategyMiddleware.ts b/backend/src/middlewares/passportStrategyMiddleware.ts index b9016d4c5d..3df0fd792f 100644 --- a/backend/src/middlewares/passportStrategyMiddleware.ts +++ b/backend/src/middlewares/passportStrategyMiddleware.ts @@ -1,8 +1,9 @@ import { getServiceLogger } from '@crowd/logging' import passport from 'passport' -import { GOOGLE_CONFIG, SLACK_CONFIG } from '../conf' +import { GOOGLE_CONFIG, SLACK_CONFIG, GITHUB_CONFIG } from '../conf' import { getGoogleStrategy } from '../services/auth/passportStrategies/googleStrategy' import { getSlackStrategy } from '../services/auth/passportStrategies/slackStrategy' +import { getGithubStrategy } from '../services/auth/passportStrategies/githubStrategy' const log = getServiceLogger() @@ -19,6 +20,10 @@ export async function passportStrategyMiddleware(req, res, next) { if (GOOGLE_CONFIG.clientId) { passport.use(getGoogleStrategy()) } + + if (GITHUB_CONFIG.clientId) { + passport.use(getGithubStrategy()) + } } catch (error) { log.error(error, 'Error getting some passport strategies!') } finally { diff --git a/backend/src/serverless/integrations/services/integrationProcessor.ts b/backend/src/serverless/integrations/services/integrationProcessor.ts index 6a6b8de053..0e4c0c5ef6 100644 --- a/backend/src/serverless/integrations/services/integrationProcessor.ts +++ b/backend/src/serverless/integrations/services/integrationProcessor.ts @@ -11,7 +11,6 @@ import { IntegrationTickProcessor } from './integrationTickProcessor' import { DiscordIntegrationService } from './integrations/discordIntegrationService' import { DiscourseIntegrationService } from './integrations/discourseIntegrationService' import { GithubIntegrationService } from './integrations/githubIntegrationService' -import { SlackIntegrationService } from './integrations/slackIntegrationService' import { TwitterIntegrationService } from './integrations/twitterIntegrationService' import { TwitterReachIntegrationService } from './integrations/twitterReachIntegrationService' import { WebhookProcessor } from './webhookProcessor' @@ -32,7 +31,6 @@ export class IntegrationProcessor extends LoggerBase { new DiscordIntegrationService(), new TwitterIntegrationService(), new TwitterReachIntegrationService(), - new SlackIntegrationService(), new GithubIntegrationService(), new DiscourseIntegrationService(), ] diff --git a/backend/src/serverless/integrations/services/integrations/slackIntegrationService.ts b/backend/src/serverless/integrations/services/integrations/slackIntegrationService.ts deleted file mode 100644 index 78d06b25d7..0000000000 --- a/backend/src/serverless/integrations/services/integrations/slackIntegrationService.ts +++ /dev/null @@ -1,586 +0,0 @@ -import moment from 'moment/moment' -import sanitizeHtml from 'sanitize-html' -import { SLACK_GRID, SLACK_MEMBER_ATTRIBUTES, SlackActivityType } from '@crowd/integrations' -import { RedisCache, getRedisClient } from '@crowd/redis' -import { timeout } from '@crowd/common' -import { IntegrationType, MemberAttributeName, PlatformType } from '@crowd/types' -import { SLACK_CONFIG, REDIS_CONFIG } from '../../../../conf' -import { - IIntegrationStream, - IPendingStream, - IProcessStreamResults, - IStepContext, - IStreamResultOperation, -} from '../../../../types/integration/stepResult' -import { SlackMessages } from '../../types/slackTypes' -import { IntegrationServiceBase } from '../integrationServiceBase' -import MemberAttributeSettingsService from '../../../../services/memberAttributeSettingsService' -import getChannels from '../../usecases/slack/getChannels' -import { Thread } from '../../types/iteratorTypes' -import getMessagesThreads from '../../usecases/slack/getMessagesInThreads' -import getMessages from '../../usecases/slack/getMessages' -import getTeam from '../../usecases/slack/getTeam' -import { AddActivitiesSingle, Member, PlatformIdentities } from '../../types/messageTypes' -import Operations from '../../../dbOperations/operations' -import getMember from '../../usecases/slack/getMember' -import getMembers from '../../usecases/slack/getMembers' - -/* eslint class-methods-use-this: 0 */ - -/* eslint-disable @typescript-eslint/no-unused-vars */ - -/* eslint-disable no-case-declarations */ - -export class SlackIntegrationService extends IntegrationServiceBase { - static maxRetrospect: number = SLACK_CONFIG.maxRetrospectInSeconds || 3600 - - constructor() { - super(IntegrationType.SLACK, 20) - - this.globalLimit = SLACK_CONFIG.globalLimit || 0 - } - - async preprocess(context: IStepContext): Promise { - const redis = await getRedisClient(REDIS_CONFIG, true) - const membersCache = new RedisCache('slack-members', redis, context.logger) - - let channelsFromSlackAPI = await getChannels( - { token: context.integration.token }, - context.logger, - ) - - const channels = context.integration.settings.channels - ? context.integration.settings.channels - : [] - - channelsFromSlackAPI = channelsFromSlackAPI.map((c) => { - if (channels.filter((a) => a.id === c.id).length <= 0) { - return { ...c, new: true } - } - return c - }) - - const team = await getTeam({ token: context.integration.token }, context.logger) - const teamUrl = team.url - - context.pipelineData = { - membersCache, - channels: channelsFromSlackAPI, - team, - teamUrl, - channelsInfo: channelsFromSlackAPI.reduce((acc, channel) => { - acc[channel.id] = { - name: channel.name, - new: !!(channel as any).new, - } - return acc - }, {}), - } - } - - async createMemberAttributes(context: IStepContext): Promise { - const service = new MemberAttributeSettingsService(context.repoContext) - await service.createPredefined(SLACK_MEMBER_ATTRIBUTES) - } - - async getStreams(context: IStepContext): Promise { - const streams = [] - - if (context.onboarding) { - streams.push({ - value: 'members', - metadata: { page: '' }, - }) - } - - const channelStreams = context.pipelineData.channels.map((c) => ({ - value: 'channel', - metadata: { channelId: c.id, page: '', general: c.general }, - })) - if (channelStreams.length > 0) { - streams.push(...channelStreams) - } - - return streams - } - - async processStream( - stream: IIntegrationStream, - context: IStepContext, - ): Promise { - await timeout(1000) - - const operations: IStreamResultOperation[] = [] - let nextPage: string - let newStreams: IPendingStream[] - let lastRecord - - switch (stream.value) { - case 'channel': { - const result = await getMessages( - { - channelId: stream.metadata.channelId, - page: stream.metadata.page, - perPage: 200, - token: context.integration.token, - }, - context.logger, - ) - - nextPage = result.nextPage - - if (result.records.length > 0) { - const { activities, additionalStreams } = await this.parseActivities( - result.records, - stream, - context, - ) - - operations.push({ - type: Operations.UPSERT_ACTIVITIES_WITH_MEMBERS, - records: activities, - }) - newStreams = additionalStreams - lastRecord = activities.length > 0 ? activities[activities.length - 1] : undefined - } - - break - } - case 'threads': { - const result = await getMessagesThreads( - { - token: context.integration.token, - channelId: stream.metadata.channelId, - page: stream.metadata.page, - perPage: 200, - threadId: stream.metadata.threadId, - }, - context.logger, - ) - - nextPage = result.nextPage - - if (result.records.length > 0) { - const { activities, additionalStreams } = await this.parseActivities( - result.records, - stream, - context, - ) - - operations.push({ - type: Operations.UPSERT_ACTIVITIES_WITH_MEMBERS, - records: activities, - }) - newStreams = additionalStreams - lastRecord = activities.length > 0 ? activities[activities.length - 1] : undefined - } - break - } - case 'members': { - const result = await getMembers( - { - token: context.integration.token, - page: stream.metadata.page, - perPage: 200, - teamId: context.pipelineData.team.id, - }, - context.logger, - ) - - nextPage = result.nextPage - if (result.records.length > 0) { - const { activities } = await this.parseActivities(result.records, stream, context) - - operations.push({ - type: Operations.UPSERT_ACTIVITIES_WITH_MEMBERS, - records: activities, - }) - } - break - } - - default: - throw new Error(`Unknown stream value ${stream.value}!`) - } - - const nextPageStream: IPendingStream = nextPage - ? { value: stream.value, metadata: { ...(stream.metadata || {}), page: nextPage } } - : undefined - - return { - operations, - lastRecord, - lastRecordTimestamp: lastRecord ? lastRecord.timestamp.getTime() : undefined, - newStreams, - nextPageStream, - } - } - - async postprocess(context: IStepContext): Promise { - // Strip the new property from channels - context.integration.settings.channels = context.pipelineData.channels.map((ch) => { - const { new: _, ...raw } = ch - return raw - }) - } - - private async parseActivities( - records: any[], - stream: IIntegrationStream, - context: IStepContext, - ): Promise<{ activities: AddActivitiesSingle[]; additionalStreams: IPendingStream[] }> { - switch (stream.value) { - case 'members': { - const members = await this.parseMembers(records, context) - return { - activities: members, - additionalStreams: [], - } - } - case 'threads': - const parseMessagesInThreadsResult = await this.parseMessagesInThreads( - records, - stream, - context, - ) - return { - activities: parseMessagesInThreadsResult.activities, - additionalStreams: parseMessagesInThreadsResult.additionalStreams, - } - default: - const parseMessagesResult = await this.parseMessages(records, stream, context) - return { - activities: parseMessagesResult.activities, - additionalStreams: parseMessagesResult.additionalStreams, - } - } - } - - /** - * Get the URL for a Slack message - * @param stream Stream we are parsing - * @param pipelineData Pipeline data - * @param record Message record - * @returns Return the url: workspaceUrl + channelUrl + messageUrl - */ - private static getUrl(stream, pipelineData, record) { - const channelId = stream.metadata.channelId - return `${pipelineData.teamUrl}archives/${channelId}/p${record.ts.replace('.', '')}` - } - - private static parseMember(record: any, context: IStepContext): Member { - const member: Member = { - displayName: record.profile.real_name, - username: { - [PlatformType.SLACK]: { - username: record.name, - integrationId: context.integration.id, - sourceId: record.id, - }, - } as PlatformIdentities, - emails: record.profile.email ? [record.profile.email] : [], - attributes: { - [MemberAttributeName.SOURCE_ID]: { - [PlatformType.SLACK]: record.id, - }, - ...(record.profile.image_72 && { - [MemberAttributeName.AVATAR_URL]: { - [PlatformType.SLACK]: record.profile.image_72, - }, - }), - ...(record.tz_label && { - [MemberAttributeName.TIMEZONE]: { - [PlatformType.SLACK]: record.tz_label, - }, - }), - ...(record.profile.title && { - [MemberAttributeName.JOB_TITLE]: { - [PlatformType.SLACK]: record.profile.title, - }, - }), - }, - } - - ;(member as any).platform = PlatformType.SLACK - - return member - } - - private static async getMember(userId: string, context: IStepContext): Promise { - const membersCache: RedisCache = context.pipelineData.membersCache - - const cached = await membersCache.get(userId) - if (cached) { - if (cached === 'null') { - return undefined - } - - return JSON.parse(cached) - } - const result = await getMember({ token: context.integration.token, userId }, context.logger) - - const member = result.records - - if (member) { - await membersCache.set(userId, JSON.stringify(member), 24 * 60 * 60) - - return member - } - - await membersCache.set(userId, 'null', 24 * 60 * 60) - return undefined - } - - private async fetchAndParseMember(context: IStepContext, userId: string): Promise { - try { - if (userId === undefined) { - return undefined - } - - const record = await SlackIntegrationService.getMember(userId, context) - - if (!record || record.is_bot) { - return undefined - } - - return SlackIntegrationService.parseMember(record, context) - } catch (e) { - context.logger.error('Error getting member in Slack', { userId }) - throw e - } - } - - /** - * Map the messages coming from Slack to activities and members - * @param records List of records coming from the API - * @param stream - * @param context - * @returns List of activities and members - */ - private async parseMessages( - records: SlackMessages, - stream: IIntegrationStream, - context: IStepContext, - ): Promise<{ activities: AddActivitiesSingle[]; additionalStreams: IPendingStream[] }> { - const newStreams: IPendingStream[] = [] - const activities: AddActivitiesSingle[] = [] - - for (const record of records) { - const member = await this.fetchAndParseMember(context, record.user) - - if (member !== undefined) { - let body = record.text - ? await SlackIntegrationService.removeMentions(record.text, context) - : '' - - let activityType - let score - let isContribution - let sourceId - if (record.subtype === 'channel_join') { - activityType = 'channel_joined' - score = SLACK_GRID[SlackActivityType.JOINED_CHANNEL].score - isContribution = SLACK_GRID[SlackActivityType.JOINED_CHANNEL].isContribution - body = undefined - sourceId = record.user - } else { - activityType = 'message' - score = SLACK_GRID[SlackActivityType.MESSAGE].score - isContribution = SLACK_GRID[SlackActivityType.MESSAGE].isContribution - sourceId = record.ts - } - activities.push({ - username: member.username[PlatformType.SLACK].username, - tenant: context.integration.tenantId, - platform: PlatformType.SLACK, - type: activityType, - sourceId, - sourceParentId: '', - timestamp: moment(parseInt(record.ts, 10) * 1000) - .utc() - .toDate(), - body, - url: SlackIntegrationService.getUrl(stream, context.pipelineData, record), - channel: context.pipelineData.channelsInfo[stream.metadata.channelId].name, - attributes: { - thread: false, - reactions: record.reactions ? record.reactions : [], - attachments: record.attachments ? record.attachments : [], - }, - score, - isContribution, - member, - }) - if (record.thread_ts) { - newStreams.push({ - value: 'threads', - metadata: { - page: '', - threadId: record.thread_ts, - channel: context.pipelineData.channelsInfo[stream.metadata.channelId].name, - channelId: stream.metadata.channelId, - placeholder: body, - new: context.pipelineData.channelsInfo[stream.metadata.channelId].new, - }, - }) - } - } - } - - return { - activities, - additionalStreams: newStreams, - } - } - - private async parseMembers( - records: any[], - context: IStepContext, - ): Promise { - const activities: AddActivitiesSingle[] = [] - for (const record of records) { - if (record.is_bot) { - // eslint-disable-next-line no-continue - continue - } - - const member = SlackIntegrationService.parseMember(record, context) - - activities.push({ - username: member.username[PlatformType.SLACK].username, - tenant: context.integration.tenantId, - platform: PlatformType.SLACK, - type: 'channel_joined', - sourceId: record.id, - timestamp: moment('1970-01-01T00:00:00+00:00').utc().toDate(), - body: undefined, - attributes: { - thread: false, - reactions: record.reactions ? record.reactions : [], - attachments: record.attachments ? record.attachments : [], - }, - score: SLACK_GRID[SlackActivityType.JOINED_CHANNEL].score, - isContribution: SLACK_GRID[SlackActivityType.JOINED_CHANNEL].isContribution, - member, - }) - } - return activities - } - - /** - * Map the messages coming from Slack to activities and members records to the format of the message to add activities and members - * @param records List of records coming from the API - * @param stream - * @param context - * @returns List of activities and members - */ - private async parseMessagesInThreads( - records: SlackMessages, - stream: IIntegrationStream, - context: IStepContext, - ): Promise<{ activities: AddActivitiesSingle[]; additionalStreams: IIntegrationStream[] }> { - const threadInfo = stream.metadata - const activities: AddActivitiesSingle[] = [] - for (const record of records) { - const member = await this.fetchAndParseMember(context, record.user) - if (member !== undefined) { - const body = record.text - ? await SlackIntegrationService.removeMentions(record.text, context) - : '' - activities.push({ - username: member.username[PlatformType.SLACK].username, - tenant: context.integration.tenantId, - platform: PlatformType.SLACK, - type: 'message', - sourceId: record.ts, - sourceParentId: threadInfo.threadId, - timestamp: moment(parseInt(record.ts, 10) * 1000) - .utc() - .toDate(), - body, - url: SlackIntegrationService.getUrl(stream, context.pipelineData, record), - channel: threadInfo.channel, - attributes: { - thread: { - body: sanitizeHtml(threadInfo.placeholder), - id: threadInfo.threadId, - }, - reactions: record.reactions ? record.reactions : [], - attachments: record.attachments ? record.attachments : [], - }, - member, - score: SLACK_GRID[SlackActivityType.MESSAGE].score, - isContribution: SLACK_GRID[SlackActivityType.MESSAGE].isContribution, - }) - } - } - - return { - activities, - additionalStreams: [], - } - } - - /** - * Parse mentions - * @param text Message text - * @param context - * @returns Message text, swapping mention IDs by mentions - */ - private static async removeMentions(text: string, context: IStepContext): Promise { - const regex = /<@!?[^>]*>/ - const globalRegex = /<@!?[^>]*>/g - const matches = text.match(globalRegex) - if (matches) { - for (let match of matches) { - match = match.replace('<', '').replace('>', '').replace('@', '').replace('!', '') - - const user = await SlackIntegrationService.getMember(match, context) - const username = user ? user.name : 'mention' - text = text.replace(regex, `@${username}`) - } - } - - return text - } - - async isProcessingFinished( - context: IStepContext, - currentStream: IIntegrationStream, - lastOperations: IStreamResultOperation[], - lastRecord?: any, - lastRecordTimestamp?: number, - ): Promise { - switch (currentStream.value) { - case 'members': - if (lastRecord === undefined) return true - - return lastRecord.sourceId in context.pipelineData.members - case 'threads': - if ((currentStream.metadata as Thread).new) { - return false - } - - if (lastRecordTimestamp === undefined) return true - - return IntegrationServiceBase.isRetrospectOver( - lastRecordTimestamp, - context.startTimestamp, - SlackIntegrationService.maxRetrospect, - ) - - default: - if (context.pipelineData.channelsInfo[currentStream.metadata.channelId].new) { - return false - } - - if (lastRecordTimestamp === undefined) return true - - return IntegrationServiceBase.isRetrospectOver( - lastRecordTimestamp, - context.startTimestamp, - SlackIntegrationService.maxRetrospect, - ) - } - } -} diff --git a/backend/src/serverless/integrations/types/slackTypes.ts b/backend/src/serverless/integrations/types/slackTypes.ts deleted file mode 100644 index 327cb8a45d..0000000000 --- a/backend/src/serverless/integrations/types/slackTypes.ts +++ /dev/null @@ -1,87 +0,0 @@ -export interface SlackGetChannelsInput { - token: string -} - -export interface SlackGetMessagesInput { - channelId: string - token: string - page: string | undefined - perPage: number | 100 -} - -export interface SlackGetMessagesInThreadsInput { - channelId: string - threadId: string - token: string - page: string | undefined - perPage: number | 100 -} - -export interface SlackGetMembersInput { - token: string - teamId: string - page: string | undefined - perPage: number | 100 -} - -export interface SlackGetMemberInput { - token: string - userId: string -} - -export interface SlackChannel { - id: string - name: string - is_member?: boolean - is_general: boolean -} - -export interface SlackTeam { - id: string - name: string - url: string - domain: string -} - -export type SlackChannels = SlackChannel[] - -export interface SlackMessage { - ts: string - type: string - text: string - subtype?: string - reactions?: any - attachments?: any - user: string - thread_ts?: string -} - -export type SlackMessages = SlackMessage[] - -export interface SlackMember { - id: string - name: string - tz_label: string - is_bot: boolean - profile: { - title: string - real_name: string - image_72: string - email: string - } -} - -export type SlackMembers = SlackMember[] - -export interface SlackParsedResponse { - records: any - nextPage: string -} - -export interface SlackGetMembersOutput extends SlackParsedResponse { - records: SlackMembers | [] -} - -export interface SlackGetMemberOutput extends SlackParsedResponse { - records: SlackMember -} diff --git a/backend/src/services/__tests__/activityService.test.ts b/backend/src/services/__tests__/activityService.test.ts index c113466622..2e9deda019 100644 --- a/backend/src/services/__tests__/activityService.test.ts +++ b/backend/src/services/__tests__/activityService.test.ts @@ -1461,7 +1461,7 @@ describe('ActivityService tests', () => { mockIRepositoryOptions, ) - expect(activity.organization.name).toEqual(org1.name) + expect(activity.organization.name).toEqual(org2.name) }) it(`Shouldn't update existing activity's organization, when a different organization comes with the member`, async () => { diff --git a/backend/src/services/__tests__/memberService.test.ts b/backend/src/services/__tests__/memberService.test.ts index d231552f9f..8782f1cfe7 100644 --- a/backend/src/services/__tests__/memberService.test.ts +++ b/backend/src/services/__tests__/memberService.test.ts @@ -461,7 +461,7 @@ describe('MemberService tests', () => { const foundMember = await MemberRepository.findById(memberCreated.id, mockIServiceOptions) - const o1 = foundMember.organizations[0].dataValues + const o1 = foundMember.organizations[0].get({ plain: true }) delete o1.createdAt delete o1.updatedAt @@ -478,6 +478,11 @@ describe('MemberService tests', () => { emails: null, phoneNumbers: null, logo: null, + memberOrganizations: { + dateEnd: null, + dateStart: null, + title: null, + }, tags: null, twitter: null, linkedin: null, @@ -531,7 +536,7 @@ describe('MemberService tests', () => { const foundMember = await MemberRepository.findById(memberCreated.id, mockIServiceOptions) - const o1 = foundMember.organizations[0].dataValues + const o1 = foundMember.organizations[0].get({ plain: true }) delete o1.createdAt delete o1.updatedAt @@ -548,6 +553,11 @@ describe('MemberService tests', () => { emails: null, phoneNumbers: null, logo: null, + memberOrganizations: { + dateEnd: null, + dateStart: null, + title: null, + }, tags: null, twitter: null, linkedin: null, @@ -605,7 +615,7 @@ describe('MemberService tests', () => { const foundMember = await MemberRepository.findById(memberCreated.id, mockIServiceOptions) - const o1 = foundMember.organizations[0].dataValues + const o1 = foundMember.organizations[0].get({ plain: true }) delete o1.createdAt delete o1.updatedAt @@ -622,6 +632,11 @@ describe('MemberService tests', () => { emails: null, phoneNumbers: null, logo: null, + memberOrganizations: { + dateEnd: null, + dateStart: null, + title: null, + }, tags: null, twitter: null, linkedin: null, @@ -678,7 +693,7 @@ describe('MemberService tests', () => { const foundMember = await MemberRepository.findById(memberCreated.id, mockIServiceOptions) - const o1 = foundMember.organizations[0].dataValues + const o1 = foundMember.organizations[0].get({ plain: true }) delete o1.createdAt delete o1.updatedAt @@ -696,6 +711,11 @@ describe('MemberService tests', () => { emails: ['hello@crowd.dev', 'jonathan@crowd.dev', 'careers@crowd.dev'], phoneNumbers: ['+42 424242'], logo: 'https://logo.clearbit.com/crowd.dev', + memberOrganizations: { + dateEnd: null, + dateStart: null, + title: null, + }, tags: [], twitter: { id: '1362101830923259908', @@ -1723,7 +1743,9 @@ describe('MemberService tests', () => { // Sequelize returns associations as array of models, we need to get plain objects mergedMember.activities = mergedMember.activities.map((i) => i.get({ plain: true })) mergedMember.tags = mergedMember.tags.map((i) => i.get({ plain: true })) - mergedMember.organizations = mergedMember.organizations.map((i) => i.get({ plain: true })) + mergedMember.organizations = mergedMember.organizations.map((i) => + SequelizeTestUtils.objectWithoutKey(i.get({ plain: true }), ['memberOrganizations']), + ) mergedMember.tasks = mergedMember.tasks.map((i) => i.get({ plain: true })) mergedMember.notes = mergedMember.notes.map((i) => i.get({ plain: true })) @@ -1769,9 +1791,24 @@ describe('MemberService tests', () => { t3 = SequelizeTestUtils.objectWithoutKey(t3, 'members') // remove organizations->member relations as well (we should be only checking 1-deep relations) - o1 = SequelizeTestUtils.objectWithoutKey(o1, ['memberCount', 'joinedAt', 'activityCount']) - o2 = SequelizeTestUtils.objectWithoutKey(o2, ['memberCount', 'joinedAt', 'activityCount']) - o3 = SequelizeTestUtils.objectWithoutKey(o3, ['memberCount', 'joinedAt', 'activityCount']) + o1 = SequelizeTestUtils.objectWithoutKey(o1, [ + 'memberCount', + 'joinedAt', + 'activityCount', + 'memberOrganizations', + ]) + o2 = SequelizeTestUtils.objectWithoutKey(o2, [ + 'memberCount', + 'joinedAt', + 'activityCount', + 'memberOrganizations', + ]) + o3 = SequelizeTestUtils.objectWithoutKey(o3, [ + 'memberCount', + 'joinedAt', + 'activityCount', + 'memberOrganizations', + ]) // remove tasks->member and tasks->activity tasks->assignees relations as well (we should be only checking 1-deep relations) task1 = SequelizeTestUtils.objectWithoutKey(task1, ['members', 'activities', 'assignees']) diff --git a/backend/src/services/auth/authService.ts b/backend/src/services/auth/authService.ts index cbd7367d43..20e142ccc0 100644 --- a/backend/src/services/auth/authService.ts +++ b/backend/src/services/auth/authService.ts @@ -474,7 +474,7 @@ class AuthService { track( 'Signed in', { - google: true, + [provider]: true, email: user.email, }, options, @@ -486,6 +486,8 @@ class AuthService { await UserRepository.update( user.id, { + firstName, + lastName, provider, providerId, emailVerified, @@ -512,7 +514,7 @@ class AuthService { track( 'Signed up', { - google: true, + [provider]: true, email: user.email, }, options, @@ -559,7 +561,7 @@ class AuthService { track( 'Signed in', { - google: providerId.includes('google'), + [provider]: true, email: user.email, }, options, @@ -568,12 +570,22 @@ class AuthService { } // If there was no provider, we can link it to the provider - if (user && (!user.provider || !user.providerId || !user.emailVerified)) { + if (user && (!user.provider || !user.providerId)) { await UserRepository.update( user.id, { provider, providerId, + emailVerified: true, + }, + options, + ) + log.debug({ user }, 'User') + } + if (user && !user.emailVerified && emailVerified) { + await UserRepository.update( + user.id, + { emailVerified, }, options, @@ -596,7 +608,7 @@ class AuthService { track( 'Signed up', { - google: true, + [provider]: true, email: user.email, }, options, diff --git a/backend/src/services/auth/passportStrategies/githubStrategy.ts b/backend/src/services/auth/passportStrategies/githubStrategy.ts new file mode 100644 index 0000000000..bfb2576c00 --- /dev/null +++ b/backend/src/services/auth/passportStrategies/githubStrategy.ts @@ -0,0 +1,50 @@ +import { get } from 'lodash' +import GithubStrategy from 'passport-github2' +import { getServiceChildLogger } from '@crowd/logging' +import { GITHUB_CONFIG } from '../../../conf' +import { databaseInit } from '../../../database/databaseConnection' +import AuthService from '../authService' +import { splitFullName } from '../../../utils/splitName' +import { AuthProvider } from '../../../types/common' + +const log = getServiceChildLogger('AuthSocial') + +export function getGithubStrategy(): GithubStrategy { + return new GithubStrategy( + { + clientID: GITHUB_CONFIG.clientId, + clientSecret: GITHUB_CONFIG.clientSecret, + callbackURL: GITHUB_CONFIG.callbackUrl, + scope: ['user:email'], // Request email scope + }, + (accessToken, refreshToken, profile, done) => { + databaseInit() + .then((database) => { + const email = get(profile, 'emails[0].value') + // GitHub user's profile doesn't include 'verified' field + // However, GitHub accounts require email verification for activation + const emailVerified = !!email + const displayName = get(profile, 'displayName') + const { firstName, lastName } = splitFullName(displayName) + + return AuthService.signinFromSocial( + AuthProvider.GITHUB, + profile.id, + email, + emailVerified, + firstName, + lastName, + displayName, + { database }, + ) + }) + .then((jwtToken) => { + done(null, jwtToken) + }) + .catch((error) => { + log.error(error, 'Error while handling github auth!') + done(error, null) + }) + }, + ) +} diff --git a/backend/src/services/auth/passportStrategies/googleStrategy.ts b/backend/src/services/auth/passportStrategies/googleStrategy.ts index 38ac46b6b7..1b9e6636f6 100644 --- a/backend/src/services/auth/passportStrategies/googleStrategy.ts +++ b/backend/src/services/auth/passportStrategies/googleStrategy.ts @@ -4,6 +4,8 @@ import { getServiceChildLogger } from '@crowd/logging' import { GOOGLE_CONFIG } from '../../../conf' import { databaseInit } from '../../../database/databaseConnection' import AuthService from '../authService' +import { splitFullName } from '../../../utils/splitName' +import { AuthProvider } from '../../../types/common' const log = getServiceChildLogger('AuthSocial') @@ -23,7 +25,7 @@ export function getGoogleStrategy(): GoogleStrategy { const { firstName, lastName } = splitFullName(displayName) return AuthService.signinFromSocial( - 'google', + AuthProvider.GOOGLE, profile.id, email, emailVerified, @@ -43,19 +45,3 @@ export function getGoogleStrategy(): GoogleStrategy { }, ) } - -function splitFullName(fullName) { - let firstName - let lastName - - if (fullName && fullName.split(' ').length > 1) { - const [firstNameArray, ...lastNameArray] = fullName.split(' ') - firstName = firstNameArray - lastName = lastNameArray.join(' ') - } else { - firstName = fullName || null - lastName = null - } - - return { firstName, lastName } -} diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 3730470df2..afba2b93ea 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -703,9 +703,9 @@ export default class IntegrationService { const transaction = await SequelizeRepository.createTransaction(this.options) let integration - let run try { + this.options.log.info('Creating Slack integration!') integration = await this.createOrUpdate( { platform: PlatformType.SLACK, @@ -715,23 +715,24 @@ export default class IntegrationService { transaction, ) - const isOnboarding: boolean = !('channels' in integration.settings) - - run = await new IntegrationRunRepository({ ...this.options, transaction }).create({ - integrationId: integration.id, - tenantId: integration.tenantId, - onboarding: isOnboarding, - state: IntegrationRunState.PENDING, - }) await SequelizeRepository.commitTransaction(transaction) } catch (err) { await SequelizeRepository.rollbackTransaction(transaction) throw err } - await sendNodeWorkerMessage( + this.options.log.info( + { tenantId: integration.tenantId }, + 'Sending Slack message to int-run-worker!', + ) + + const isOnboarding: boolean = !('channels' in integration.settings) + const emitter = await getIntegrationRunWorkerEmitter() + await emitter.triggerIntegrationRun( integration.tenantId, - new NodeWorkerIntegrationProcessMessage(run.id), + integration.platform, + integration.id, + isOnboarding, ) return integration diff --git a/backend/src/services/memberService.ts b/backend/src/services/memberService.ts index d23348d6a4..f1888b9b3f 100644 --- a/backend/src/services/memberService.ts +++ b/backend/src/services/memberService.ts @@ -306,11 +306,13 @@ export default class MemberService extends LoggerBase { // If organizations are sent if (data.organizations) { // Collect IDs for relation - const organizationsIds = [] + const organizations = [] for (const organization of data.organizations) { if (typeof organization === 'string' && validator.isUUID(organization)) { // If an ID was already sent, we simply push it to the list - organizationsIds.push(organization) + organizations.push(organization) + } else if (typeof organization === 'object' && organization.id) { + organizations.push(organization) } else { // Otherwise, either another string or an object was sent const organizationService = new OrganizationService(this.options) @@ -324,11 +326,32 @@ export default class MemberService extends LoggerBase { } // We findOrCreate the organization and add it to the list of IDs const organizationRecord = await organizationService.findOrCreate(data) - organizationsIds.push(organizationRecord.id) + organizations.push(organizationRecord.id) } } + + // Auto assign member to organization if email domain matches + if (data.emails) { + const emailDomains = new Set() + + // Collect unique domains + for (const email of data.emails) { + const domain = email.split('@')[1] + emailDomains.add(domain) + } + + // Fetch organization ids for these domains + const organizationService = new OrganizationService(this.options) + for (const domain of emailDomains) { + const organizationRecord = await organizationService.findByUrl(domain) + if (organizationRecord) { + organizations.push(organizationRecord.id) + } + } + } + // Remove dups - data.organizations = [...new Set(organizationsIds)] + data.organizations = [...new Set(organizations)] } const fillRelations = false @@ -669,25 +692,30 @@ export default class MemberService extends LoggerBase { return toKeep }, organizations: (oldOrganizations, newOrganizations) => { - oldOrganizations = oldOrganizations - ? oldOrganizations.map((o) => { - if (o.id) { - return o.id - } - return o - }) - : [] - - newOrganizations = newOrganizations - ? newOrganizations.map((o) => { - if (o.id) { - return o.id - } - return o - }) - : [] - - return Array.from(new Set([...oldOrganizations, ...newOrganizations])) + const convertOrgs = (orgs) => + orgs + ? orgs + .map((o) => (o.dataValues ? o.get({ plain: true }) : o)) + .map((o) => { + if (typeof o === 'string') { + return { + id: o, + } + } + const memberOrg = o.memberOrganizations + return { + id: o.id, + title: memberOrg?.title, + startDate: memberOrg?.dateStart, + endDate: memberOrg?.dateEnd, + } + }) + : [] + + oldOrganizations = convertOrgs(oldOrganizations) + newOrganizations = convertOrgs(newOrganizations) + + return lodash.uniqWith([...oldOrganizations, ...newOrganizations], lodash.isEqual) }, }) } diff --git a/backend/src/services/organizationService.ts b/backend/src/services/organizationService.ts index 05864a88c3..ade9b550c2 100644 --- a/backend/src/services/organizationService.ts +++ b/backend/src/services/organizationService.ts @@ -192,6 +192,10 @@ export default class OrganizationService extends LoggerBase { return OrganizationRepository.findAndCountAll(args, this.options) } + async findByUrl(url) { + return OrganizationRepository.findByUrl(url, this.options) + } + async query(data) { const advancedFilter = data.filter const orderBy = data.orderBy diff --git a/backend/src/services/premium/enrichment/memberEnrichmentService.ts b/backend/src/services/premium/enrichment/memberEnrichmentService.ts index 1a41792cc2..8ca1bf2a7b 100644 --- a/backend/src/services/premium/enrichment/memberEnrichmentService.ts +++ b/backend/src/services/premium/enrichment/memberEnrichmentService.ts @@ -2,6 +2,7 @@ import { LoggerBase } from '@crowd/logging' import { RedisPubSubEmitter, getRedisClient } from '@crowd/redis' import axios from 'axios' import lodash from 'lodash' +import moment from 'moment' import { ApiWebsocketMessage, MemberAttributeName, @@ -29,6 +30,9 @@ import { EnrichmentAPISkills, EnrichmentAPIWorkExperience, } from './types/memberEnrichmentTypes' +import OrganizationService from '../../organizationService' +import MemberRepository from '../../../database/repositories/memberRepository' +import OrganizationRepository from '../../../database/repositories/organizationRepository' export default class MemberEnrichmentService extends LoggerBase { options: IServiceOptions @@ -267,7 +271,38 @@ export default class MemberEnrichmentService extends LoggerBase { this.options, ) - return memberService.upsert({ ...normalized, platform: Object.keys(member.username)[0] }) + const result = await memberService.upsert({ + ...normalized, + platform: Object.keys(member.username)[0], + }) + + // for every work experience in `enrichmentData` + // - upsert organization + // - upsert `memberOrganization` relation + const organizationService = new OrganizationService(this.options) + if (enrichmentData.work_experiences) { + for (const workExperience of enrichmentData.work_experiences) { + const org = await organizationService.findOrCreate({ + name: workExperience.company, + }) + + const dateEnd = workExperience.endDate + ? moment.utc(workExperience.endDate).toISOString() + : null + + const data = { + memberId: result.id, + organizationId: org.id, + title: workExperience.title, + dateStart: workExperience.startDate, + dateEnd, + } + await MemberRepository.createOrUpdateWorkExperience(data, this.options) + await OrganizationRepository.includeOrganizationToSegments(org.id, this.options) + } + } + + return result } return null } @@ -300,6 +335,12 @@ export default class MemberEnrichmentService extends LoggerBase { organization.location = organizationsByWorkExperience[0].location organization.linkedin = organizationsByWorkExperience[0].companyLinkedInUrl organization.url = organizationsByWorkExperience[0].companyUrl + + // fetch jobTitle from most recent work experience + member.attributes.jobTitle = { + custom: organizationsByWorkExperience[0].title, + default: organizationsByWorkExperience[0].title, + } } } diff --git a/backend/src/types/common.ts b/backend/src/types/common.ts index c78f57c281..a88a4c5a57 100644 --- a/backend/src/types/common.ts +++ b/backend/src/types/common.ts @@ -33,3 +33,8 @@ export enum FeatureFlagRedisKey { MEMBER_ENRICHMENT_COUNT = 'memberEnrichmentCount', ORGANIZATION_ENRICHMENT_COUNT = 'organizationEnrichmentCount', } + +export enum AuthProvider { + GOOGLE = 'google', + GITHUB = 'github', +} diff --git a/backend/src/utils/splitName.ts b/backend/src/utils/splitName.ts new file mode 100644 index 0000000000..809ac1a13c --- /dev/null +++ b/backend/src/utils/splitName.ts @@ -0,0 +1,15 @@ +export function splitFullName(fullName) { + let firstName + let lastName + + if (fullName && fullName.split(' ').length > 1) { + const [firstNameArray, ...lastNameArray] = fullName.split(' ') + firstName = firstNameArray + lastName = lastNameArray.join(' ') + } else { + firstName = fullName || null + lastName = null + } + + return { firstName, lastName } +} diff --git a/frontend/src/modules/auth/pages/signin-page.vue b/frontend/src/modules/auth/pages/signin-page.vue index 2bc966eb72..bb4bfac6c9 100644 --- a/frontend/src/modules/auth/pages/signin-page.vue +++ b/frontend/src/modules/auth/pages/signin-page.vue @@ -117,7 +117,7 @@
-
+
Sign in with Google + + + Sign in with GitHub +

diff --git a/frontend/src/modules/auth/pages/signup-page.vue b/frontend/src/modules/auth/pages/signup-page.vue index 9047018930..c02da342ef 100644 --- a/frontend/src/modules/auth/pages/signup-page.vue +++ b/frontend/src/modules/auth/pages/signup-page.vue @@ -194,7 +194,7 @@

-
+
Sign up with Google + + + Sign up with GitHub +

diff --git a/frontend/src/modules/dashboard/components/organization/dashboard-organization-item.vue b/frontend/src/modules/dashboard/components/organization/dashboard-organization-item.vue index 334a055913..624eec8e4c 100644 --- a/frontend/src/modules/dashboard/components/organization/dashboard-organization-item.vue +++ b/frontend/src/modules/dashboard/components/organization/dashboard-organization-item.vue @@ -28,7 +28,7 @@

- {{ organization.displayName }} + {{ organization.displayName || organization.name }}
diff --git a/frontend/src/modules/layout/components/layout.vue b/frontend/src/modules/layout/components/layout.vue index 3e3dec8d7f..21c5d8a631 100644 --- a/frontend/src/modules/layout/components/layout.vue +++ b/frontend/src/modules/layout/components/layout.vue @@ -1,7 +1,7 @@