From 6e26a526aa1023f801442b984e78d33ee110f350 Mon Sep 17 00:00:00 2001 From: jNullj <15849761+jNullj@users.noreply.github.com> Date: Fri, 4 Aug 2023 13:38:28 +0300 Subject: [PATCH 1/3] Add TOML support with [BaseTomlService] Add base toml service to enable fetch of toml files Add spec file for the new toml service for automated testing This was added to allow a new way to retrive python version from pyproject.toml as described in issue #9410 --- core/base-service/base-toml.js | 82 +++++++++++++++ core/base-service/base-toml.spec.js | 150 ++++++++++++++++++++++++++++ package-lock.json | 10 ++ package.json | 1 + 4 files changed, 243 insertions(+) create mode 100644 core/base-service/base-toml.js create mode 100644 core/base-service/base-toml.spec.js diff --git a/core/base-service/base-toml.js b/core/base-service/base-toml.js new file mode 100644 index 0000000000000..5b4ca779c9e37 --- /dev/null +++ b/core/base-service/base-toml.js @@ -0,0 +1,82 @@ +/** + * @module + */ + +import emojic from 'emojic' +import { parse } from 'smol-toml' +import BaseService from './base.js' +import { InvalidResponse } from './errors.js' +import trace from './trace.js' + +/** + * Services which query a TOML endpoint should extend BaseTomlService + * + * @abstract + */ +class BaseTomlService extends BaseService { + /** + * Request data from an upstream API serving TOML, + * parse it and validate against a schema + * + * @param {object} attrs Refer to individual attrs + * @param {Joi} attrs.schema Joi schema to validate the response against + * @param {string} attrs.url URL to request + * @param {object} [attrs.options={}] Options to pass to got. See + * [documentation](https://github.com/sindresorhus/got/blob/main/documentation/2-options.md) + * @param {object} [attrs.httpErrors={}] Key-value map of status codes + * and custom error messages e.g: `{ 404: 'package not found' }`. + * This can be used to extend or override the + * [default](https://github.com/badges/shields/blob/master/core/base-service/check-error-response.js#L5) + * @param {object} [attrs.systemErrors={}] Key-value map of got network exception codes + * and an object of params to pass when we construct an Inaccessible exception object + * e.g: `{ ECONNRESET: { prettyMessage: 'connection reset' } }`. + * See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes} + * for allowed keys + * and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values + * @returns {object} Parsed response + * @see https://github.com/sindresorhus/got/blob/main/documentation/2-options.md + */ + async _requestToml({ + schema, + url, + options = {}, + httpErrors = {}, + systemErrors = {}, + }) { + const logTrace = (...args) => trace.logTrace('fetch', ...args) + const mergedOptions = { + ...{ + headers: { + Accept: + // the officeal header should be application/toml - see https://toml.io/en/v1.0.0#mime-type + // but as this is not registered here https://www.iana.org/assignments/media-types/media-types.xhtml + // some apps use other mime-type like application/x-toml, text/plain etc.... + 'text/x-toml, text/toml, application/x-toml, application/toml, text/plain', + }, + }, + ...options, + } + const { buffer } = await this._request({ + url, + options: mergedOptions, + httpErrors, + systemErrors, + }) + let parsed + try { + parsed = parse(buffer.toString()) + } catch (err) { + logTrace(emojic.dart, 'Response TOML (unparseable)', buffer) + throw new InvalidResponse({ + prettyMessage: 'unparseable toml response', + underlyingError: err, + }) + } + logTrace(emojic.dart, 'Response TOML (before validation)', parsed, { + deep: true, + }) + return this.constructor._validate(parsed, schema) + } +} + +export default BaseTomlService diff --git a/core/base-service/base-toml.spec.js b/core/base-service/base-toml.spec.js new file mode 100644 index 0000000000000..6d1ba583b02e5 --- /dev/null +++ b/core/base-service/base-toml.spec.js @@ -0,0 +1,150 @@ +import Joi from 'joi' +import { expect } from 'chai' +import sinon from 'sinon' +import BaseTomlService from './base-toml.js' + +const dummySchema = Joi.object({ + requiredString: Joi.string().required(), +}).required() + +class DummyTomlService extends BaseTomlService { + static category = 'cat' + static route = { base: 'foo' } + + async handle() { + const { requiredString } = await this._requestToml({ + schema: dummySchema, + url: 'http://example.com/foo.toml', + }) + return { message: requiredString } + } +} + +const expectedToml = ` +# example toml +requiredString = "some-string" +` + +const unexpectedToml = ` +# example toml - unexpected val +unexpectedKey = "some-string" +` + +const invalidToml = ` +# example invalid toml +missing= "space" +colonsCantBeUsed: 42 +missing "assignment" +` + +describe('BaseTomlService', function () { + describe('Making requests', function () { + let requestFetcher + beforeEach(function () { + requestFetcher = sinon.stub().returns( + Promise.resolve({ + buffer: expectedToml, + res: { statusCode: 200 }, + }), + ) + }) + + it('invokes _requestFetcher', async function () { + await DummyTomlService.invoke( + { requestFetcher }, + { handleInternalErrors: false }, + ) + + expect(requestFetcher).to.have.been.calledOnceWith( + 'http://example.com/foo.toml', + { + headers: { + Accept: + 'text/x-toml, text/toml, application/x-toml, application/toml, text/plain', + }, + }, + ) + }) + + it('forwards options to _requestFetcher', async function () { + class WithOptions extends DummyTomlService { + async handle() { + const { requiredString } = await this._requestToml({ + schema: dummySchema, + url: 'http://example.com/foo.toml', + options: { method: 'POST', searchParams: { queryParam: 123 } }, + }) + return { message: requiredString } + } + } + + await WithOptions.invoke( + { requestFetcher }, + { handleInternalErrors: false }, + ) + + expect(requestFetcher).to.have.been.calledOnceWith( + 'http://example.com/foo.toml', + { + headers: { + Accept: + 'text/x-toml, text/toml, application/x-toml, application/toml, text/plain', + }, + method: 'POST', + searchParams: { queryParam: 123 }, + }, + ) + }) + }) + + describe('Making badges', function () { + it('handles valid toml responses', async function () { + const requestFetcher = async () => ({ + buffer: expectedToml, + res: { statusCode: 200 }, + }) + expect( + await DummyTomlService.invoke( + { requestFetcher }, + { handleInternalErrors: false }, + ), + ).to.deep.equal({ + message: 'some-string', + }) + }) + + it('handles toml responses which do not match the schema', async function () { + const requestFetcher = async () => ({ + buffer: unexpectedToml, + res: { statusCode: 200 }, + }) + expect( + await DummyTomlService.invoke( + { requestFetcher }, + { handleInternalErrors: false }, + ), + ).to.deep.equal({ + isError: true, + color: 'lightgray', + message: 'invalid response data', + }) + }) + + it('handles unparseable toml responses', async function () { + const requestFetcher = async () => ({ + buffer: invalidToml, + res: { statusCode: 200 }, + }) + expect( + await DummyTomlService.invoke( + { requestFetcher }, + { handleInternalErrors: false }, + ), + ).to.deep.equal({ + isError: true, + color: 'lightgray', + message: 'unparseable toml response', + }) + }) + }) +}) diff --git a/package-lock.json b/package-lock.json index 352c98726dfb1..7378703278c4d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,6 +52,7 @@ "query-string": "^8.1.0", "semver": "~7.5.4", "simple-icons": "9.8.0", + "smol-toml": "^1.1.1", "webextension-store-meta": "^1.0.5", "xpath": "~0.0.33" }, @@ -24203,6 +24204,15 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/smol-toml": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.1.1.tgz", + "integrity": "sha512-qyYMygHyDKiy82iiKTH/zXr0DZmEpsou0AMZnkXdYhA/0LhPLoZ/xHaOBrbecLbAJ/Gd5KhMWWH8TXtgv1g+DQ==", + "engines": { + "node": ">= 18", + "pnpm": ">= 8" + } + }, "node_modules/snap-shot-compare": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/snap-shot-compare/-/snap-shot-compare-3.0.0.tgz", diff --git a/package.json b/package.json index 0ba0df0bdcc47..ccfa1c78b500f 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "query-string": "^8.1.0", "semver": "~7.5.4", "simple-icons": "9.8.0", + "smol-toml": "1.1.1", "webextension-store-meta": "^1.0.5", "xpath": "~0.0.33" }, From 9fb28ee09792a364cd67309e8da80d0cd408faf5 Mon Sep 17 00:00:00 2001 From: jNullj <15849761+jNullj@users.noreply.github.com> Date: Mon, 14 Aug 2023 01:29:28 +0300 Subject: [PATCH 2/3] Fix typo Co-authored-by: chris48s --- core/base-service/base-toml.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base-service/base-toml.js b/core/base-service/base-toml.js index 5b4ca779c9e37..47bd5d66231e2 100644 --- a/core/base-service/base-toml.js +++ b/core/base-service/base-toml.js @@ -48,7 +48,7 @@ class BaseTomlService extends BaseService { ...{ headers: { Accept: - // the officeal header should be application/toml - see https://toml.io/en/v1.0.0#mime-type + // the official header should be application/toml - see https://toml.io/en/v1.0.0#mime-type // but as this is not registered here https://www.iana.org/assignments/media-types/media-types.xhtml // some apps use other mime-type like application/x-toml, text/plain etc.... 'text/x-toml, text/toml, application/x-toml, application/toml, text/plain', From f762a3d22600870bcd592bf2007456ec4219d038 Mon Sep 17 00:00:00 2001 From: jNullj <15849761+jNullj@users.noreply.github.com> Date: Mon, 14 Aug 2023 01:41:08 +0300 Subject: [PATCH 3/3] refactor: improve code readability solve code review https://github.com/badges/shields/pull/9438#discussion_r1291503340 --- core/base-service/base-toml.spec.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/base-service/base-toml.spec.js b/core/base-service/base-toml.spec.js index 6d1ba583b02e5..becf3063e4fc9 100644 --- a/core/base-service/base-toml.spec.js +++ b/core/base-service/base-toml.spec.js @@ -25,13 +25,13 @@ const expectedToml = ` requiredString = "some-string" ` -const unexpectedToml = ` -# example toml - unexpected val +const invalidSchemaToml = ` +# example toml - legal toml syntax but invalid schema unexpectedKey = "some-string" ` -const invalidToml = ` -# example invalid toml +const invalidTomlSyntax = ` +# example illegal toml syntax that can't be parsed missing= "space" colonsCantBeUsed: 42 missing "assignment" @@ -115,7 +115,7 @@ describe('BaseTomlService', function () { it('handles toml responses which do not match the schema', async function () { const requestFetcher = async () => ({ - buffer: unexpectedToml, + buffer: invalidSchemaToml, res: { statusCode: 200 }, }) expect( @@ -132,7 +132,7 @@ describe('BaseTomlService', function () { it('handles unparseable toml responses', async function () { const requestFetcher = async () => ({ - buffer: invalidToml, + buffer: invalidTomlSyntax, res: { statusCode: 200 }, }) expect(