diff --git a/config/custom-environment-variables.yml b/config/custom-environment-variables.yml index 0f35b0284e058..18011390b7a47 100644 --- a/config/custom-environment-variables.yml +++ b/config/custom-environment-variables.yml @@ -95,6 +95,7 @@ private: obs_user: 'OBS_USER' obs_pass: 'OBS_PASS' redis_url: 'REDIS_URL' + opencollective_token: 'OPENCOLLECTIVE_TOKEN' postgres_url: 'POSTGRES_URL' sentry_dsn: 'SENTRY_DSN' sl_insight_userUuid: 'SL_INSIGHT_USER_UUID' diff --git a/core/server/server.js b/core/server/server.js index 81eaf8e6d8098..1b2870f50462b 100644 --- a/core/server/server.js +++ b/core/server/server.js @@ -184,6 +184,7 @@ const privateConfigSchema = Joi.object({ obs_user: Joi.string(), obs_pass: Joi.string(), redis_url: Joi.string().uri({ scheme: ['redis', 'rediss'] }), + opencollective_token: Joi.string(), postgres_url: Joi.string().uri({ scheme: 'postgresql' }), sentry_dsn: Joi.string(), sl_insight_userUuid: Joi.string(), diff --git a/doc/server-secrets.md b/doc/server-secrets.md index 5e0e9927923fe..10c88cd12dd5f 100644 --- a/doc/server-secrets.md +++ b/doc/server-secrets.md @@ -231,7 +231,7 @@ installation access to private npm packages [npm token]: https://docs.npmjs.com/getting-started/working_with_tokens -## Open Build Service +### Open Build Service - `OBS_USER` (yml: `private.obs_user`) - `OBS_PASS` (yml: `private.obs_user`) @@ -246,6 +246,14 @@ they can only be scoped to execute specific actions on a POST request. This means however, that an actual account is required to read the build status of a package. +### OpenCollective + +- `OPENCOLLECTIVE_TOKEN` (yml: `opencollective_token`) + +OpenCollective's GraphQL API only allows 10 reqs/minute for anonymous users. +An [API token](https://graphql-docs-v2.opencollective.com/access) +can be provided to access a higher rate limit of 100 reqs/minute. + ### SymfonyInsight (formerly Sensiolabs) - `SL_INSIGHT_USER_UUID` (yml: `private.sl_insight_userUuid`) diff --git a/services/opencollective/opencollective-all.service.js b/services/opencollective/opencollective-all.service.js index 38e6ecb7edf8b..066f065931df7 100644 --- a/services/opencollective/opencollective-all.service.js +++ b/services/opencollective/opencollective-all.service.js @@ -16,12 +16,18 @@ export default class OpencollectiveAll extends OpencollectiveBase { }, } + static _cacheLength = 900 + static defaultBadgeData = { label: 'backers and sponsors', } async handle({ collective }) { - const { backersCount } = await this.fetchCollectiveInfo(collective) + const data = await this.fetchCollectiveInfo({ + collective, + accountType: [], + }) + const backersCount = this.getCount(data) return this.constructor.render(backersCount) } } diff --git a/services/opencollective/opencollective-all.tester.js b/services/opencollective/opencollective-all.tester.js index b31ac58ec00ce..61f2ef3e2cabb 100644 --- a/services/opencollective/opencollective-all.tester.js +++ b/services/opencollective/opencollective-all.tester.js @@ -2,25 +2,6 @@ import { nonNegativeInteger } from '../validators.js' import { createServiceTester } from '../tester.js' export const t = await createServiceTester() -t.create('renders correctly') - .get('/shields.json') - .intercept(nock => - nock('https://opencollective.com/').get('/shields.json').reply(200, { - slug: 'shields', - currency: 'USD', - image: - 'https://opencollective-production.s3-us-west-1.amazonaws.com/44dcbb90-1ee9-11e8-a4c3-7bb1885c0b6e.png', - balance: 105494, - yearlyIncome: 157371, - backersCount: 35, - contributorsCount: 276, - }), - ) - .expectBadge({ - label: 'backers and sponsors', - message: '35', - color: 'brightgreen', - }) t.create('gets amount of backers and sponsors') .get('/shields.json') .expectBadge({ @@ -28,23 +9,10 @@ t.create('gets amount of backers and sponsors') message: nonNegativeInteger, }) -t.create('renders not found correctly') - .get('/nonexistent-collective.json') - .intercept(nock => - nock('https://opencollective.com/') - .get('/nonexistent-collective.json') - .reply(404, 'Not found'), - ) - .expectBadge({ - label: 'backers and sponsors', - message: 'collective not found', - color: 'red', - }) - t.create('handles not found correctly') .get('/nonexistent-collective.json') .expectBadge({ label: 'backers and sponsors', - message: 'collective not found', - color: 'red', + message: 'No collective found with slug nonexistent-collective', + color: 'lightgrey', }) diff --git a/services/opencollective/opencollective-backers.service.js b/services/opencollective/opencollective-backers.service.js index 6e5204a42ae01..c07efd5b1fa33 100644 --- a/services/opencollective/opencollective-backers.service.js +++ b/services/opencollective/opencollective-backers.service.js @@ -16,15 +16,19 @@ export default class OpencollectiveBackers extends OpencollectiveBase { }, } + static _cacheLength = 900 + static defaultBadgeData = { label: 'backers', } async handle({ collective }) { - const { backersCount } = await this.fetchCollectiveBackersCount( + const data = await this.fetchCollectiveInfo({ collective, - { userType: 'users' }, - ) + accountType: ['INDIVIDUAL'], + }) + const backersCount = this.getCount(data) + return this.constructor.render(backersCount) } } diff --git a/services/opencollective/opencollective-backers.tester.js b/services/opencollective/opencollective-backers.tester.js index b69901cb1821d..29855ae57d07c 100644 --- a/services/opencollective/opencollective-backers.tester.js +++ b/services/opencollective/opencollective-backers.tester.js @@ -2,80 +2,6 @@ import { nonNegativeInteger } from '../validators.js' import { createServiceTester } from '../tester.js' export const t = await createServiceTester() -t.create('renders correctly') - .get('/shields.json') - .intercept(nock => - nock('https://opencollective.com/') - .get('/shields/members/users.json') - .reply(200, [ - { MemberId: 8685, type: 'USER', role: 'ADMIN' }, - { MemberId: 8686, type: 'USER', role: 'ADMIN' }, - { MemberId: 8682, type: 'USER', role: 'ADMIN' }, - { MemberId: 10305, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 10396, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 10733, type: 'USER', role: 'BACKER' }, - { MemberId: 8684, type: 'USER', role: 'ADMIN' }, - { MemberId: 10741, type: 'USER', role: 'BACKER' }, - { - MemberId: 10756, - type: 'USER', - role: 'BACKER', - tier: 'monthly backer', - }, - { MemberId: 11578, type: 'USER', role: 'CONTRIBUTOR' }, - { MemberId: 13459, type: 'USER', role: 'CONTRIBUTOR' }, - { - MemberId: 13507, - type: 'USER', - role: 'BACKER', - tier: 'monthly backer', - }, - { MemberId: 13512, type: 'USER', role: 'BACKER' }, - { MemberId: 13513, type: 'USER', role: 'FUNDRAISER' }, - { MemberId: 13984, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 14916, type: 'USER', role: 'BACKER' }, - { - MemberId: 16326, - type: 'USER', - role: 'BACKER', - tier: 'monthly backer', - }, - { MemberId: 18252, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 17631, type: 'USER', role: 'BACKER', tier: 'backer' }, - { - MemberId: 16420, - type: 'USER', - role: 'BACKER', - tier: 'monthly backer', - }, - { MemberId: 17186, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 18791, type: 'USER', role: 'BACKER', tier: 'backer' }, - { - MemberId: 19279, - type: 'USER', - role: 'BACKER', - tier: 'monthly backer', - }, - { MemberId: 19863, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 21451, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 22718, type: 'USER', role: 'BACKER' }, - { MemberId: 23561, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 25092, type: 'USER', role: 'CONTRIBUTOR' }, - { MemberId: 24473, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 25439, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 24483, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 25090, type: 'USER', role: 'CONTRIBUTOR' }, - { MemberId: 26404, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 27026, type: 'USER', role: 'BACKER', tier: 'backer' }, - { MemberId: 27132, type: 'USER', role: 'CONTRIBUTOR' }, - ]), - ) - .expectBadge({ - label: 'backers', - message: '25', - color: 'brightgreen', - }) - t.create('gets amount of backers').get('/shields.json').expectBadge({ label: 'backers', message: nonNegativeInteger, @@ -85,6 +11,6 @@ t.create('handles not found correctly') .get('/nonexistent-collective.json') .expectBadge({ label: 'backers', - message: 'collective not found', - color: 'red', + message: 'No collective found with slug nonexistent-collective', + color: 'lightgrey', }) diff --git a/services/opencollective/opencollective-base.js b/services/opencollective/opencollective-base.js index 203cfabf03274..97e42da30d58a 100644 --- a/services/opencollective/opencollective-base.js +++ b/services/opencollective/opencollective-base.js @@ -1,28 +1,38 @@ +import gql from 'graphql-tag' import Joi from 'joi' +import { BaseGraphqlService } from '../index.js' import { nonNegativeInteger } from '../validators.js' -import { BaseJsonService } from '../index.js' import { metric } from '../text-formatters.js' -// https://developer.opencollective.com/#/api/collectives?id=get-info -const collectiveDetailsSchema = Joi.object().keys({ - slug: Joi.string().required(), - backersCount: nonNegativeInteger, -}) +const schema = Joi.object({ + data: Joi.object({ + account: Joi.object({ + name: Joi.string(), + slug: Joi.string(), + members: Joi.object({ + totalCount: nonNegativeInteger, + nodes: Joi.array().items( + Joi.object({ + tier: Joi.object({ + legacyId: Joi.number(), + name: Joi.string(), + }).allow(null), + }), + ), + }).required(), + }).required(), + }).required(), +}).required() -// https://developer.opencollective.com/#/api/collectives?id=get-members -function buildMembersArraySchema({ userType, tierRequired }) { - const keys = { - MemberId: Joi.number().required(), - type: userType || Joi.string().required(), - role: Joi.string().required(), - } - if (tierRequired) keys.tier = Joi.string().required() - return Joi.array().items(Joi.object().keys(keys)) -} - -export default class OpencollectiveBase extends BaseJsonService { +export default class OpencollectiveBase extends BaseGraphqlService { static category = 'funding' + static auth = { + passKey: 'opencollective_token', + authorizedOrigins: ['https://api.opencollective.com'], + isRequired: false, + } + static buildRoute(base, withTierId) { return { base: `opencollective${base ? `/${base}` : ''}`, @@ -38,45 +48,51 @@ export default class OpencollectiveBase extends BaseJsonService { } } - async fetchCollectiveInfo(collective) { - return this._requestJson({ - schema: collectiveDetailsSchema, - // https://developer.opencollective.com/#/api/collectives?id=get-info - url: `https://opencollective.com/${collective}.json`, - httpErrors: { - 404: 'collective not found', - }, - }) + async fetchCollectiveInfo({ collective, accountType }) { + return this._requestGraphql( + this.authHelper.withQueryStringAuth( + { passKey: 'personalToken' }, + { + schema, + url: 'https://api.opencollective.com/graphql/v2', + query: gql` + query account($slug: String, $accountType: [AccountType]) { + account(slug: $slug) { + name + slug + members(accountType: $accountType, role: BACKER) { + totalCount + nodes { + tier { + legacyId + name + } + } + } + } + } + `, + variables: { + slug: collective, + accountType, + }, + options: { + headers: { 'content-type': 'application/json' }, + }, + }, + ), + ) } - async fetchCollectiveBackersCount(collective, { userType, tierId }) { - const schema = buildMembersArraySchema({ - userType: - userType === 'users' - ? 'USER' - : userType === 'organizations' - ? 'ORGANIZATION' - : undefined, - tierRequired: tierId, - }) - const members = await this._requestJson({ - schema, - // https://developer.opencollective.com/#/api/collectives?id=get-members - // https://developer.opencollective.com/#/api/collectives?id=get-members-per-tier - url: `https://opencollective.com/${collective}/members/${ - userType || 'all' - }.json${tierId ? `?TierId=${tierId}` : ''}`, - httpErrors: { - 404: 'collective not found', + getCount(data) { + const { + data: { + account: { + members: { totalCount }, + }, }, - }) + } = data - const result = { - backersCount: members.filter(member => member.role === 'BACKER').length, - } - // Find the title of the tier - if (tierId && members.length > 0) - result.tier = members.map(member => member.tier)[0] - return result + return totalCount } } diff --git a/services/opencollective/opencollective-base.spec.js b/services/opencollective/opencollective-base.spec.js new file mode 100644 index 0000000000000..87a6646f2cd31 --- /dev/null +++ b/services/opencollective/opencollective-base.spec.js @@ -0,0 +1,37 @@ +import { expect } from 'chai' +import nock from 'nock' +import { cleanUpNockAfterEach, defaultContext } from '../test-helpers.js' +import OpencollectiveBase from './opencollective-base.js' + +class DummyOpencollectiveService extends OpencollectiveBase { + static route = this.buildRoute('dummy') + + async handle({ collective }) { + const data = await this.fetchCollectiveInfo({ + collective, + accountType: [], + }) + return this.constructor.render(this.getCount(data)) + } +} + +describe('OpencollectiveBase', function () { + describe('auth', function () { + cleanUpNockAfterEach() + + const config = { private: { opencollective_token: 'fake-token' } } + + it('sends the auth information as configured', async function () { + const scope = nock('https://api.opencollective.com') + .post('/graphql/v2') + .query({ personalToken: 'fake-token' }) + .reply(200, { data: { account: { members: { totalCount: 1 } } } }) + + expect( + await DummyOpencollectiveService.invoke(defaultContext, config, {}), + ).to.deep.equal({ color: 'brightgreen', label: undefined, message: '1' }) + + scope.done() + }) + }) +}) diff --git a/services/opencollective/opencollective-by-tier.service.js b/services/opencollective/opencollective-by-tier.service.js index 98acc383f509d..673597df41d2f 100644 --- a/services/opencollective/opencollective-by-tier.service.js +++ b/services/opencollective/opencollective-by-tier.service.js @@ -1,9 +1,91 @@ -import OpencollectiveBase from './opencollective-base.js' +import Joi from 'joi' +import { nonNegativeInteger } from '../validators.js' +import { BaseJsonService } from '../index.js' +import { metric } from '../text-formatters.js' const documentation = `

How to get the tierId

According to open collectives documentation, you can find the tierId by looking at the URL after clicking on a Tier Card on the collective page. (e.g. tierId for https://opencollective.com/shields/order/2988 is 2988)

` -export default class OpencollectiveByTier extends OpencollectiveBase { +// https://developer.opencollective.com/#/api/collectives?id=get-info +const collectiveDetailsSchema = Joi.object().keys({ + slug: Joi.string().required(), + backersCount: nonNegativeInteger, +}) + +// https://developer.opencollective.com/#/api/collectives?id=get-members +function buildMembersArraySchema({ userType, tierRequired }) { + const keys = { + MemberId: Joi.number().required(), + type: userType || Joi.string().required(), + role: Joi.string().required(), + } + if (tierRequired) keys.tier = Joi.string().required() + return Joi.array().items(Joi.object().keys(keys)) +} + +class OpencollectiveBaseJson extends BaseJsonService { + static category = 'funding' + + static buildRoute(base, withTierId) { + return { + base: `opencollective${base ? `/${base}` : ''}`, + pattern: `:collective${withTierId ? '/:tierId' : ''}`, + } + } + + static render(backersCount, label) { + return { + label, + message: metric(backersCount), + color: backersCount > 0 ? 'brightgreen' : 'lightgrey', + } + } + + async fetchCollectiveInfo(collective) { + return this._requestJson({ + schema: collectiveDetailsSchema, + // https://developer.opencollective.com/#/api/collectives?id=get-info + url: `https://opencollective.com/${collective}.json`, + httpErrors: { + 404: 'collective not found', + }, + }) + } + + async fetchCollectiveBackersCount(collective, { userType, tierId }) { + const schema = buildMembersArraySchema({ + userType: + userType === 'users' + ? 'USER' + : userType === 'organizations' + ? 'ORGANIZATION' + : undefined, + tierRequired: tierId, + }) + const members = await this._requestJson({ + schema, + // https://developer.opencollective.com/#/api/collectives?id=get-members + // https://developer.opencollective.com/#/api/collectives?id=get-members-per-tier + url: `https://opencollective.com/${collective}/members/${ + userType || 'all' + }.json${tierId ? `?TierId=${tierId}` : ''}`, + httpErrors: { + 404: 'collective not found', + }, + }) + + const result = { + backersCount: members.filter(member => member.role === 'BACKER').length, + } + // Find the title of the tier + if (tierId && members.length > 0) + result.tier = members.map(member => member.tier)[0] + return result + } +} + +// TODO: 1. pagination is needed. 2. use new graphql api instead of legacy rest api +export default class OpencollectiveByTier extends OpencollectiveBaseJson { static route = this.buildRoute('tier', true) static examples = [ diff --git a/services/opencollective/opencollective-sponsors.service.js b/services/opencollective/opencollective-sponsors.service.js index c8246370a85e7..b88e6ee2c6bd3 100644 --- a/services/opencollective/opencollective-sponsors.service.js +++ b/services/opencollective/opencollective-sponsors.service.js @@ -16,15 +16,18 @@ export default class OpencollectiveSponsors extends OpencollectiveBase { }, } + static _cacheLength = 900 + static defaultBadgeData = { label: 'sponsors', } async handle({ collective }) { - const { backersCount } = await this.fetchCollectiveBackersCount( + const data = await this.fetchCollectiveInfo({ collective, - { userType: 'organizations' }, - ) + accountType: ['ORGANIZATION'], + }) + const backersCount = this.getCount(data) return this.constructor.render(backersCount) } } diff --git a/services/opencollective/opencollective-sponsors.tester.js b/services/opencollective/opencollective-sponsors.tester.js index db6c97d4d323f..d563744a49ae3 100644 --- a/services/opencollective/opencollective-sponsors.tester.js +++ b/services/opencollective/opencollective-sponsors.tester.js @@ -2,80 +2,16 @@ import { nonNegativeInteger } from '../validators.js' import { createServiceTester } from '../tester.js' export const t = await createServiceTester() -t.create('renders correctly') - .get('/shields.json') - .intercept(nock => - nock('https://opencollective.com/') - .get('/shields/members/organizations.json') - .reply(200, [ - { MemberId: 8683, type: 'ORGANIZATION', role: 'HOST' }, - { - MemberId: 13484, - type: 'ORGANIZATION', - role: 'BACKER', - tier: 'backer', - }, - { MemberId: 13508, type: 'ORGANIZATION', role: 'FUNDRAISER' }, - { MemberId: 15987, type: 'ORGANIZATION', role: 'BACKER' }, - { - MemberId: 16561, - type: 'ORGANIZATION', - role: 'BACKER', - tier: 'sponsor', - }, - { - MemberId: 16469, - type: 'ORGANIZATION', - role: 'BACKER', - tier: 'sponsor', - }, - { - MemberId: 18162, - type: 'ORGANIZATION', - role: 'BACKER', - tier: 'sponsor', - }, - { - MemberId: 21023, - type: 'ORGANIZATION', - role: 'BACKER', - tier: 'sponsor', - }, - { - MemberId: 21482, - type: 'ORGANIZATION', - role: 'BACKER', - tier: 'monthly backer', - }, - { - MemberId: 26367, - type: 'ORGANIZATION', - role: 'BACKER', - tier: 'monthly backer', - }, - { MemberId: 27531, type: 'ORGANIZATION', role: 'BACKER' }, - { - MemberId: 29443, - type: 'ORGANIZATION', - role: 'BACKER', - tier: 'monthly backer', - }, - ]), - ) - .expectBadge({ - label: 'sponsors', - message: '10', - color: 'brightgreen', - }) t.create('gets amount of sponsors').get('/shields.json').expectBadge({ label: 'sponsors', message: nonNegativeInteger, + color: 'brightgreen', }) t.create('handles not found correctly') .get('/nonexistent-collective.json') .expectBadge({ label: 'sponsors', - message: 'collective not found', - color: 'red', + message: 'No collective found with slug nonexistent-collective', + color: 'lightgrey', })