diff --git a/core/base-service/base-graphql.js b/core/base-service/base-graphql.js index d70dbe96ca319..650e6fb549a82 100644 --- a/core/base-service/base-graphql.js +++ b/core/base-service/base-graphql.js @@ -50,6 +50,8 @@ class BaseGraphqlService extends BaseService { * See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes} * for allowed keys * and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values + * @param {number[]} [attrs.logErrors=[429]] An array of http error codes + * that will be logged (to sentry, if configured). * @param {Function} [attrs.transformJson=data => data] Function which takes the raw json and transforms it before * further processing. In case of multiple query in a single graphql call and few of them * throw error, partial data might be used ignoring the error. @@ -69,6 +71,7 @@ class BaseGraphqlService extends BaseService { options = {}, httpErrorMessages = {}, systemErrors = {}, + logErrors = [429], transformJson = data => data, transformErrors = defaultTransformErrors, }) { @@ -83,6 +86,7 @@ class BaseGraphqlService extends BaseService { options: mergedOptions, httpErrors: httpErrorMessages, systemErrors, + logErrors, }) const json = transformJson(this._parseJson(buffer)) if (json.errors) { diff --git a/core/base-service/base-json.js b/core/base-service/base-json.js index 075ea561b44af..281eebe6b4778 100644 --- a/core/base-service/base-json.js +++ b/core/base-service/base-json.js @@ -40,6 +40,8 @@ class BaseJsonService extends BaseService { * See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes} * for allowed keys * and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values + * @param {number[]} [attrs.logErrors=[429]] An array of http error codes + * that will be logged (to sentry, if configured). * @returns {object} Parsed response * @see https://github.com/sindresorhus/got/blob/main/documentation/2-options.md */ @@ -49,6 +51,7 @@ class BaseJsonService extends BaseService { options = {}, httpErrors = {}, systemErrors = {}, + logErrors = [429], }) { const mergedOptions = { ...{ headers: { Accept: 'application/json' } }, @@ -59,6 +62,7 @@ class BaseJsonService extends BaseService { options: mergedOptions, httpErrors, systemErrors, + logErrors, }) const json = this._parseJson(buffer) return this.constructor._validate(json, schema) diff --git a/core/base-service/base-svg-scraping.js b/core/base-service/base-svg-scraping.js index 95bc3be8ac61f..7eacea6f7dab6 100644 --- a/core/base-service/base-svg-scraping.js +++ b/core/base-service/base-svg-scraping.js @@ -63,6 +63,8 @@ class BaseSvgScrapingService extends BaseService { * See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes} * for allowed keys * and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values + * @param {number[]} [attrs.logErrors=[429]] An array of http error codes + * that will be logged (to sentry, if configured). * @returns {object} Parsed response * @see https://github.com/sindresorhus/got/blob/main/documentation/2-options.md */ @@ -73,6 +75,7 @@ class BaseSvgScrapingService extends BaseService { options = {}, httpErrors = {}, systemErrors = {}, + logErrors = [429], }) { const logTrace = (...args) => trace.logTrace('fetch', ...args) const mergedOptions = { @@ -84,6 +87,7 @@ class BaseSvgScrapingService extends BaseService { options: mergedOptions, httpErrors, systemErrors, + logErrors, }) logTrace(emojic.dart, 'Response SVG', buffer) const data = { diff --git a/core/base-service/base-toml.js b/core/base-service/base-toml.js index 47bd5d66231e2..435f6bafbefea 100644 --- a/core/base-service/base-toml.js +++ b/core/base-service/base-toml.js @@ -33,6 +33,8 @@ class BaseTomlService extends BaseService { * See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes} * for allowed keys * and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values + * @param {number[]} [attrs.logErrors=[429]] An array of http error codes + * that will be logged (to sentry, if configured). * @returns {object} Parsed response * @see https://github.com/sindresorhus/got/blob/main/documentation/2-options.md */ @@ -42,6 +44,7 @@ class BaseTomlService extends BaseService { options = {}, httpErrors = {}, systemErrors = {}, + logErrors = [429], }) { const logTrace = (...args) => trace.logTrace('fetch', ...args) const mergedOptions = { @@ -61,6 +64,7 @@ class BaseTomlService extends BaseService { options: mergedOptions, httpErrors, systemErrors, + logErrors, }) let parsed try { diff --git a/core/base-service/base-xml.js b/core/base-service/base-xml.js index 7395c45ea64bd..4424abf608a7f 100644 --- a/core/base-service/base-xml.js +++ b/core/base-service/base-xml.js @@ -34,6 +34,8 @@ class BaseXmlService extends BaseService { * See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes} * for allowed keys * and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values + * @param {number[]} [attrs.logErrors=[429]] An array of http error codes + * that will be logged (to sentry, if configured). * @param {object} [attrs.parserOptions={}] Options to pass to fast-xml-parser. See * [documentation](https://github.com/NaturalIntelligence/fast-xml-parser#xml-to-json) * @returns {object} Parsed response @@ -46,6 +48,7 @@ class BaseXmlService extends BaseService { options = {}, httpErrors = {}, systemErrors = {}, + logErrors = [429], parserOptions = {}, }) { const logTrace = (...args) => trace.logTrace('fetch', ...args) @@ -58,6 +61,7 @@ class BaseXmlService extends BaseService { options: mergedOptions, httpErrors, systemErrors, + logErrors, }) const validateResult = XMLValidator.validate(buffer) if (validateResult !== true) { diff --git a/core/base-service/base-yaml.js b/core/base-service/base-yaml.js index bae76068f213f..9cb700a18442b 100644 --- a/core/base-service/base-yaml.js +++ b/core/base-service/base-yaml.js @@ -33,6 +33,8 @@ class BaseYamlService extends BaseService { * See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes} * for allowed keys * and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values + * @param {number[]} [attrs.logErrors=[429]] An array of http error codes + * that will be logged (to sentry, if configured). * @param {object} [attrs.encoding='utf8'] Character encoding * @returns {object} Parsed response * @see https://github.com/sindresorhus/got/blob/main/documentation/2-options.md @@ -43,6 +45,7 @@ class BaseYamlService extends BaseService { options = {}, httpErrors = {}, systemErrors = {}, + logErrors = [429], encoding = 'utf8', }) { const logTrace = (...args) => trace.logTrace('fetch', ...args) @@ -60,6 +63,7 @@ class BaseYamlService extends BaseService { options: mergedOptions, httpErrors, systemErrors, + logErrors, }) let parsed try { diff --git a/core/base-service/base.js b/core/base-service/base.js index 4f6d3da09e746..02eefc59b18f8 100644 --- a/core/base-service/base.js +++ b/core/base-service/base.js @@ -263,7 +263,13 @@ class BaseService { this._metricHelper = metricHelper } - async _request({ url, options = {}, httpErrors = {}, systemErrors = {} }) { + async _request({ + url, + options = {}, + httpErrors = {}, + systemErrors = {}, + logErrors = [429], + }) { const logTrace = (...args) => trace.logTrace('fetch', ...args) let logUrl = url const logOptions = Object.assign({}, options) @@ -290,7 +296,7 @@ class BaseService { ) await this._meterResponse(res, buffer) logTrace(emojic.dart, 'Response status code', res.statusCode) - return checkErrorResponse(httpErrors)({ buffer, res }) + return checkErrorResponse(httpErrors, logErrors)({ buffer, res }) } static enabledMetrics = [] diff --git a/core/base-service/check-error-response.js b/core/base-service/check-error-response.js index 62b6b695775da..bc0d0a1ae1161 100644 --- a/core/base-service/check-error-response.js +++ b/core/base-service/check-error-response.js @@ -1,3 +1,4 @@ +import log from '../server/log.js' import { NotFound, InvalidResponse, Inaccessible } from './errors.js' const defaultErrorMessages = { @@ -5,7 +6,7 @@ const defaultErrorMessages = { 429: 'rate limited by upstream service', } -export default function checkErrorResponse(httpErrors = {}) { +export default function checkErrorResponse(httpErrors = {}, logErrors = [429]) { return async function ({ buffer, res }) { let error httpErrors = { ...defaultErrorMessages, ...httpErrors } @@ -25,6 +26,11 @@ export default function checkErrorResponse(httpErrors = {}) { error = new InvalidResponse(props) } } + + if (logErrors.includes(res.statusCode)) { + log.error(new Error(`${res.statusCode} calling ${res.requestUrl.origin}`)) + } + if (error) { error.response = res error.buffer = buffer diff --git a/core/base-service/check-error-response.spec.js b/core/base-service/check-error-response.spec.js index ba5d9a33e9b43..411008dfd6fd0 100644 --- a/core/base-service/check-error-response.spec.js +++ b/core/base-service/check-error-response.spec.js @@ -47,7 +47,7 @@ describe('async error handler', function () { context('when status is 429', function () { const buffer = Buffer.from('some stuff') - const res = { statusCode: 429 } + const res = { statusCode: 429, requestUrl: new URL('https://example.com/') } it('throws InvalidResponse', async function () { try { diff --git a/services/discord/discord.service.js b/services/discord/discord.service.js index 4df26b1ef64ba..b5c4809f89d18 100644 --- a/services/discord/discord.service.js +++ b/services/discord/discord.service.js @@ -49,7 +49,7 @@ export default class Discord extends BaseJsonService { }, } - static _cacheLength = 30 + static _cacheLength = 60 static defaultBadgeData = { label: 'chat' } diff --git a/services/dynamic/dynamic-json.service.js b/services/dynamic/dynamic-json.service.js index f730395073471..dac28d664813c 100644 --- a/services/dynamic/dynamic-json.service.js +++ b/services/dynamic/dynamic-json.service.js @@ -49,6 +49,7 @@ export default class DynamicJson extends jsonPath(BaseJsonService) { schema, url, httpErrors, + logErrors: [], }) } } diff --git a/services/dynamic/dynamic-toml.service.js b/services/dynamic/dynamic-toml.service.js index 950d9ab085d93..bd30652614095 100644 --- a/services/dynamic/dynamic-toml.service.js +++ b/services/dynamic/dynamic-toml.service.js @@ -49,6 +49,7 @@ export default class DynamicToml extends jsonPath(BaseTomlService) { schema, url, httpErrors, + logErrors: [], }) } } diff --git a/services/dynamic/dynamic-xml.service.js b/services/dynamic/dynamic-xml.service.js index cd84d2f547868..9a8eeffb812d5 100644 --- a/services/dynamic/dynamic-xml.service.js +++ b/services/dynamic/dynamic-xml.service.js @@ -109,6 +109,7 @@ export default class DynamicXml extends BaseService { url, options: { headers: { Accept: 'application/xml, text/xml' } }, httpErrors, + logErrors: [], }) const { values: value } = this.transform({ diff --git a/services/dynamic/dynamic-yaml.service.js b/services/dynamic/dynamic-yaml.service.js index 0857c8ef144b4..12f8647c3468f 100644 --- a/services/dynamic/dynamic-yaml.service.js +++ b/services/dynamic/dynamic-yaml.service.js @@ -49,6 +49,7 @@ export default class DynamicYaml extends jsonPath(BaseYamlService) { schema, url, httpErrors, + logErrors: [], }) } } diff --git a/services/endpoint-common.js b/services/endpoint-common.js index fa51d390a1621..fabf483d057fb 100644 --- a/services/endpoint-common.js +++ b/services/endpoint-common.js @@ -95,6 +95,7 @@ async function fetchEndpointData( schema: anySchema, url, httpErrors, + logErrors: [], options: { decompress: true }, }) return validateEndpointData(json, { diff --git a/services/github/github-actions-workflow-status.service.js b/services/github/github-actions-workflow-status.service.js index 44dfccf9b43b7..cd0d2cb872ece 100644 --- a/services/github/github-actions-workflow-status.service.js +++ b/services/github/github-actions-workflow-status.service.js @@ -73,6 +73,8 @@ export default class GithubActionsWorkflowStatus extends BaseSvgScrapingService }, ] + static _cacheLength = 60 + static defaultBadgeData = { label: 'build', } diff --git a/services/opencollective/opencollective-all.service.js b/services/opencollective/opencollective-all.service.js index 066f065931df7..d9e406a7da592 100644 --- a/services/opencollective/opencollective-all.service.js +++ b/services/opencollective/opencollective-all.service.js @@ -16,7 +16,7 @@ export default class OpencollectiveAll extends OpencollectiveBase { }, } - static _cacheLength = 900 + static _cacheLength = 1800 static defaultBadgeData = { label: 'backers and sponsors', diff --git a/services/opencollective/opencollective-backers.service.js b/services/opencollective/opencollective-backers.service.js index c07efd5b1fa33..e54094e151961 100644 --- a/services/opencollective/opencollective-backers.service.js +++ b/services/opencollective/opencollective-backers.service.js @@ -16,7 +16,7 @@ export default class OpencollectiveBackers extends OpencollectiveBase { }, } - static _cacheLength = 900 + static _cacheLength = 1800 static defaultBadgeData = { label: 'backers', diff --git a/services/opencollective/opencollective-sponsors.service.js b/services/opencollective/opencollective-sponsors.service.js index b88e6ee2c6bd3..f8faf78e87a2c 100644 --- a/services/opencollective/opencollective-sponsors.service.js +++ b/services/opencollective/opencollective-sponsors.service.js @@ -16,7 +16,7 @@ export default class OpencollectiveSponsors extends OpencollectiveBase { }, } - static _cacheLength = 900 + static _cacheLength = 1800 static defaultBadgeData = { label: 'sponsors', diff --git a/services/uptimerobot/uptimerobot-base.js b/services/uptimerobot/uptimerobot-base.js index fbdae811d388e..0d5b1506986b0 100644 --- a/services/uptimerobot/uptimerobot-base.js +++ b/services/uptimerobot/uptimerobot-base.js @@ -74,6 +74,7 @@ export default class UptimeRobotBase extends BaseJsonService { ...opts, }, }, + logErrors: [], }) if (stat === 'fail') { diff --git a/services/weblate/weblate-base.js b/services/weblate/weblate-base.js index 9128e753fa995..c046ed6a38ab0 100644 --- a/services/weblate/weblate-base.js +++ b/services/weblate/weblate-base.js @@ -2,6 +2,8 @@ import Joi from 'joi' import { BaseJsonService } from '../index.js' import { optionalUrl } from '../validators.js' +export const defaultServer = 'https://hosted.weblate.org' + export default class WeblateBase extends BaseJsonService { static queryParamSchema = Joi.object({ server: optionalUrl, diff --git a/services/weblate/weblate-component-license.service.js b/services/weblate/weblate-component-license.service.js index 2a31fa8b5c1d3..8e710fc5887ed 100644 --- a/services/weblate/weblate-component-license.service.js +++ b/services/weblate/weblate-component-license.service.js @@ -1,5 +1,5 @@ import Joi from 'joi' -import WeblateBase from './weblate-base.js' +import WeblateBase, { defaultServer } from './weblate-base.js' const schema = Joi.object({ license: Joi.string().required(), @@ -21,7 +21,7 @@ export default class WeblateComponentLicense extends WeblateBase { { title: 'Weblate component license', namedParams: { project: 'godot-engine', component: 'godot' }, - queryParams: { server: 'https://hosted.weblate.org' }, + queryParams: { server: defaultServer }, staticPreview: this.render({ license: 'MIT' }), keywords: ['i18n', 'translation', 'internationalization'], }, @@ -33,7 +33,7 @@ export default class WeblateComponentLicense extends WeblateBase { return { message: `${license}` } } - async fetch({ project, component, server = 'https://hosted.weblate.org' }) { + async fetch({ project, component, server = defaultServer }) { return super.fetch({ schema, url: `${server}/api/components/${project}/${component}/`, @@ -41,6 +41,7 @@ export default class WeblateComponentLicense extends WeblateBase { 403: 'access denied by remote server', 404: 'component not found', }, + logErrors: server === defaultServer ? [429] : [], }) } diff --git a/services/weblate/weblate-entities.service.js b/services/weblate/weblate-entities.service.js index 26f59b4774b26..01935be405f80 100644 --- a/services/weblate/weblate-entities.service.js +++ b/services/weblate/weblate-entities.service.js @@ -1,7 +1,7 @@ import Joi from 'joi' import { nonNegativeInteger } from '../validators.js' import { metric } from '../text-formatters.js' -import WeblateBase from './weblate-base.js' +import WeblateBase, { defaultServer } from './weblate-base.js' const schema = Joi.object({ count: nonNegativeInteger, @@ -20,7 +20,7 @@ export default class WeblateEntities extends WeblateBase { { title: 'Weblate entities', namedParams: { type: 'projects' }, - queryParams: { server: 'https://hosted.weblate.org' }, + queryParams: { server: defaultServer }, staticPreview: this.render({ type: 'projects', count: 533 }), keywords: ['i18n', 'internationalization'], }, @@ -32,13 +32,14 @@ export default class WeblateEntities extends WeblateBase { return { label: type, message: metric(count) } } - async fetch({ type, server = 'https://hosted.weblate.org' }) { + async fetch({ type, server = defaultServer }) { return super.fetch({ schema, url: `${server}/api/${type}/`, httpErrors: { 403: 'access denied by remote server', }, + logErrors: server === defaultServer ? [429] : [], }) } diff --git a/services/weblate/weblate-project-translated-percentage.service.js b/services/weblate/weblate-project-translated-percentage.service.js index d9743b03c34b9..80cc4d429b4a5 100644 --- a/services/weblate/weblate-project-translated-percentage.service.js +++ b/services/weblate/weblate-project-translated-percentage.service.js @@ -1,6 +1,6 @@ import Joi from 'joi' import { colorScale } from '../color-formatters.js' -import WeblateBase from './weblate-base.js' +import WeblateBase, { defaultServer } from './weblate-base.js' const schema = Joi.object({ translated_percent: Joi.number().required(), @@ -23,7 +23,7 @@ export default class WeblateProjectTranslatedPercentage extends WeblateBase { { title: 'Weblate project translated', namedParams: { project: 'godot-engine' }, - queryParams: { server: 'https://hosted.weblate.org' }, + queryParams: { server: defaultServer }, staticPreview: this.render({ translatedPercent: 20.5 }), keywords: ['i18n', 'translation', 'internationalization'], }, @@ -45,7 +45,7 @@ export default class WeblateProjectTranslatedPercentage extends WeblateBase { return { message: `${translatedPercent.toFixed(0)}%`, color } } - async fetch({ project, server = 'https://hosted.weblate.org' }) { + async fetch({ project, server = defaultServer }) { return super.fetch({ schema, url: `${server}/api/projects/${project}/statistics/`, @@ -53,6 +53,7 @@ export default class WeblateProjectTranslatedPercentage extends WeblateBase { 403: 'access denied by remote server', 404: 'project not found', }, + logErrors: server === defaultServer ? [429] : [], }) } diff --git a/services/weblate/weblate-user-statistic.service.js b/services/weblate/weblate-user-statistic.service.js index 6a4c3a89dcf71..5a8be882f21f4 100644 --- a/services/weblate/weblate-user-statistic.service.js +++ b/services/weblate/weblate-user-statistic.service.js @@ -1,7 +1,7 @@ import Joi from 'joi' import { nonNegativeInteger } from '../validators.js' import { metric } from '../text-formatters.js' -import WeblateBase from './weblate-base.js' +import WeblateBase, { defaultServer } from './weblate-base.js' const schema = Joi.object({ translated: nonNegativeInteger, @@ -33,7 +33,7 @@ export default class WeblateUserStatistic extends WeblateBase { { title: 'Weblate user statistic', namedParams: { statistic: 'translations', user: 'nijel' }, - queryParams: { server: 'https://hosted.weblate.org' }, + queryParams: { server: defaultServer }, staticPreview: this.render({ statistic: 'translations', count: 30585 }), keywords: ['i18n', 'internationalization'], }, @@ -45,7 +45,7 @@ export default class WeblateUserStatistic extends WeblateBase { return { label: statistic, message: metric(count) } } - async fetch({ user, server = 'https://hosted.weblate.org' }) { + async fetch({ user, server = defaultServer }) { return super.fetch({ schema, url: `${server}/api/users/${user}/statistics/`, @@ -53,6 +53,7 @@ export default class WeblateUserStatistic extends WeblateBase { 403: 'access denied by remote server', 404: 'user not found', }, + logErrors: server === defaultServer ? [429] : [], }) }