From 94d9a9562058fe31ab6b81e4dd7d835ef1bd1552 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 8 Oct 2024 17:32:13 +0200 Subject: [PATCH 01/30] Eslint tweaks --- .eslintrc.js | 6 ++++++ packages/core/src/log.ts | 4 ---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 4395372d..08971781 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -384,5 +384,11 @@ module.exports = { 'no-console': ['error', { allow: ['time', 'timeEnd'] }], }, }, + { + files: ['packages/core/src/log.ts'], + rules: { + 'no-console': 'off', + }, + }, ], }; diff --git a/packages/core/src/log.ts b/packages/core/src/log.ts index 49675f63..69b89ae1 100644 --- a/packages/core/src/log.ts +++ b/packages/core/src/log.ts @@ -11,20 +11,16 @@ export type Logger = (text: any, type?: LogLevel) => void; const log = (text: any, level: LogLevel, type: LogLevel = 'debug', name?: string) => { // By default (debug) we print dimmed. let color = c.dim; - // eslint-disable-next-line no-console let logFn = console.log; if (type === 'error') { color = c.red; - // eslint-disable-next-line no-console logFn = console.error; } else if (type === 'warn') { color = c.yellow; - // eslint-disable-next-line no-console logFn = console.warn; } else if (type === 'info') { color = c.cyan; - // eslint-disable-next-line no-console logFn = console.log; } From 0bfbd18ea3adc1fe165ba858dc3c019a9ef797f6 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 8 Oct 2024 17:36:04 +0200 Subject: [PATCH 02/30] Re-use existing loop script --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5a87c9c6..f34287a1 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "yarn": "1.22.19" }, "scripts": { - "build:all": "yarn workspaces foreach -Apti --exclude \"@dd/*\" run build", + "build:all": "yarn loop run build", "cli": "yarn workspace @dd/tools cli", "format": "yarn lint --fix", "lint": "eslint ./packages/**/*.{ts,js} --quiet", From ebe254f7a39e4b05f9ea59aad091a8ad8984815b Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 8 Oct 2024 17:36:20 +0200 Subject: [PATCH 03/30] Strongify injection plugin detection --- packages/core/src/plugins/injection/index.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/core/src/plugins/injection/index.ts b/packages/core/src/plugins/injection/index.ts index 80df01f8..fba16045 100644 --- a/packages/core/src/plugins/injection/index.ts +++ b/packages/core/src/plugins/injection/index.ts @@ -48,6 +48,10 @@ export const getInjectionPlugins = ( }, }; + const isInjectedFile = (id: string) => { + return id.includes(INJECTED_FILE); + }; + // This plugin happens in 3 steps in order to cover all bundlers: // 1. Prepare the content to inject, fetching distant/local files and anything necessary. // 2. Inject a virtual file into the bundling, this file will be home of all injected content. @@ -135,17 +139,19 @@ export const getInjectionPlugins = ( // Resolve the injected file. { name: RESOLUTION_PLUGIN_NAME, - enforce: 'post', + enforce: 'pre', resolveId(id) { - if (id === INJECTED_FILE) { + if (isInjectedFile(id)) { return { id, moduleSideEffects: true }; } }, loadInclude(id) { - return id === INJECTED_FILE; + if (isInjectedFile(id)) { + return true; + } }, load(id) { - if (id === INJECTED_FILE) { + if (isInjectedFile(id)) { return getContentToInject(); } }, From 4fe9556abb05d4577fe6e6da707dab150b6ab280 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 8 Oct 2024 17:37:55 +0200 Subject: [PATCH 04/30] Re-use doRequest to submit metrics --- .../plugins/telemetry/src/common/sender.ts | 38 +++---------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/packages/plugins/telemetry/src/common/sender.ts b/packages/plugins/telemetry/src/common/sender.ts index dd5736b7..173e49fc 100644 --- a/packages/plugins/telemetry/src/common/sender.ts +++ b/packages/plugins/telemetry/src/common/sender.ts @@ -2,10 +2,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { formatDuration } from '@dd/core/helpers'; +import { doRequest, formatDuration } from '@dd/core/helpers'; import type { Logger } from '@dd/core/log'; -import { request } from 'https'; -import type { ServerResponse } from 'http'; import type { MetricToSend } from '../types'; @@ -28,41 +26,15 @@ export const sendMetrics = ( .sort() .map((name) => `${name} - ${metrics.filter((m) => m.metric === name).length}`); - // eslint-disable-next-line no-console log(` Sending ${metrics.length} metrics. Metrics: - ${metricsNames.join('\n - ')}`); - return new Promise((resolve, reject) => { - const req = request({ - method: 'POST', - hostname: auth.endPoint, - path: `/api/v1/series?api_key=${auth.apiKey}`, - }); - - req.write( - JSON.stringify({ - series: metrics, - }), - ); - - req.on('response', (res: ServerResponse) => { - if (!(res.statusCode >= 200 && res.statusCode < 300)) { - // Untyped method https://nodejs.org/api/http.html#http_http_get_url_options_callback - // Consume response data to free up memory - // @ts-ignore - res.resume(); - reject(`Request Failed.\nStatus Code: ${res.statusCode}`); - return; - } - // Empty event required, otherwise the 'end' event is never emitted - res.on('data', () => {}); - res.on('end', resolve); - }); - - req.on('error', reject); - req.end(); + return doRequest({ + method: 'POST', + url: `${auth.endPoint}/api/v1/series?api_key=${auth.apiKey}`, + getData: () => ({ data: JSON.stringify({ series: metrics }) }), }) .then(() => { log(`Sent metrics in ${formatDuration(Date.now() - startSending)}.`); From 2f006876d68b14b83871a0ed67e67119be9d0944 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 8 Oct 2024 17:39:16 +0200 Subject: [PATCH 05/30] Add new mocks --- packages/tests/src/helpers/mocks.ts | 13 +++++++++++++ packages/tests/src/plugins/telemetry/testHelpers.ts | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/packages/tests/src/helpers/mocks.ts b/packages/tests/src/helpers/mocks.ts index 8726b1e2..0cddd634 100644 --- a/packages/tests/src/helpers/mocks.ts +++ b/packages/tests/src/helpers/mocks.ts @@ -4,6 +4,8 @@ import { getResolvedPath } from '@dd/core/helpers'; import type { GlobalContext, Options } from '@dd/core/types'; +import { getSourcemapsConfiguration } from '@dd/tests/plugins/rum/testHelpers'; +import { getTelemetryConfiguration } from '@dd/tests/plugins/telemetry/testHelpers'; import path from 'path'; if (!process.env.PROJECT_CWD) { @@ -84,6 +86,17 @@ export const getComplexBuildOverrides = ( return bundlerOverrides; }; +export const getFullPluginConfig = (overrides: Partial = {}): Options => { + return { + ...defaultPluginOptions, + rum: { + sourcemaps: getSourcemapsConfiguration(), + }, + telemetry: getTelemetryConfiguration(), + ...overrides, + }; +}; + // Returns a JSON of files with their content. // To be used with memfs' vol.fromJSON. export const getMirroredFixtures = (paths: string[], cwd: string) => { diff --git a/packages/tests/src/plugins/telemetry/testHelpers.ts b/packages/tests/src/plugins/telemetry/testHelpers.ts index aef19b6b..3ed9b552 100644 --- a/packages/tests/src/plugins/telemetry/testHelpers.ts +++ b/packages/tests/src/plugins/telemetry/testHelpers.ts @@ -13,6 +13,7 @@ import type { TelemetryOptions, Module, } from '@dd/telemetry-plugins/types'; +import { FAKE_URL } from '@dd/tests/helpers/mocks'; import type { PluginBuild, Metafile } from 'esbuild'; import esbuild from 'esbuild'; @@ -130,3 +131,15 @@ export const mockOptionsWithTelemetry: OptionsWithTelemetry = { ...mockOptions, telemetry: mockTelemetryOptions, }; + +export const getTelemetryConfiguration = ( + overrides: Partial = {}, +): TelemetryOptions => ({ + enableTracing: true, + endPoint: FAKE_URL, + output: true, + prefix: 'prefix', + tags: ['tag'], + timestamp: new Date().getTime(), + ...overrides, +}); From 1ee1d528eabf3e2f52f7427c9e3235925a99c929 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 8 Oct 2024 17:40:54 +0200 Subject: [PATCH 06/30] Incorporate unplugin loaders in bundles --- packages/tools/src/rollupConfig.mjs | 31 ++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/tools/src/rollupConfig.mjs b/packages/tools/src/rollupConfig.mjs index 60668e62..c1f051de 100644 --- a/packages/tools/src/rollupConfig.mjs +++ b/packages/tools/src/rollupConfig.mjs @@ -6,7 +6,10 @@ import babel from '@rollup/plugin-babel'; import commonjs from '@rollup/plugin-commonjs'; import json from '@rollup/plugin-json'; import { nodeResolve } from '@rollup/plugin-node-resolve'; +import fs from 'fs'; import modulePackage from 'module'; +import { createRequire } from 'node:module'; +import path from 'path'; import dts from 'rollup-plugin-dts'; import esbuild from 'rollup-plugin-esbuild'; @@ -21,6 +24,8 @@ export const bundle = (config) => ({ // These are peer dependencies 'webpack', 'esbuild', + 'vite', + 'rollup', // These should be internal only and never be anywhere published. '@dd/core', '@dd/tools', @@ -62,7 +67,31 @@ if (typeof module !== 'undefined') { */ export const getDefaultBuildConfigs = (packageJson) => [ bundle({ - plugins: [esbuild()], + plugins: [ + esbuild(), + { + name: 'copy-unplugin-loaders', + writeBundle(options) { + // Unplugins comes with loaders that need to be copied in place + // to be usable. + const outputDir = options.dir || path.dirname(options.file); + const require = createRequire(import.meta.url); + const unpluginDir = path.dirname(require.resolve('unplugin')); + fs.cpSync( + path.resolve(unpluginDir, 'webpack'), + path.resolve(outputDir, 'webpack'), + { recursive: true }, + ); + fs.cpSync( + path.resolve(unpluginDir, 'rspack'), + path.resolve(outputDir, 'rspack'), + { + recursive: true, + }, + ); + }, + }, + ], output: { file: packageJson.module, format: 'esm', From 5bce0e0910408db04c62d603fd70262167abd320 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 8 Oct 2024 17:41:27 +0200 Subject: [PATCH 07/30] Better cleanup and error handling in runBundlers --- packages/tests/src/helpers/runBundlers.ts | 63 ++++++++++++++++------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/packages/tests/src/helpers/runBundlers.ts b/packages/tests/src/helpers/runBundlers.ts index 69a6b7c7..0147187b 100644 --- a/packages/tests/src/helpers/runBundlers.ts +++ b/packages/tests/src/helpers/runBundlers.ts @@ -60,7 +60,7 @@ type BundlerRunFunction = ( seed: string, pluginOverrides: Options, bundlerOverrides: any, -) => Promise; +) => Promise<{ cleanup: CleanupFn; errors: string[] }>; const getCleanupFunction = (bundlerName: string, outdirs: (string | undefined)[]): CleanupFn => @@ -85,6 +85,7 @@ export const runWebpack: BundlerRunFunction = async ( ) => { const bundlerConfigs = getWebpack5Options(seed, pluginOverrides, bundlerOverrides); const { webpack } = await import('webpack'); + const errors = []; try { await new Promise((resolve, reject) => { @@ -94,9 +95,10 @@ export const runWebpack: BundlerRunFunction = async ( }); } catch (e: any) { console.error(`Build failed for Webpack 5`, e); + errors.push(e.message); } - return getCleanupFunction('Webpack 5', [bundlerConfigs.output?.path]); + return { cleanup: getCleanupFunction('Webpack 5', [bundlerConfigs.output?.path]), errors }; }; export const runWebpack4: BundlerRunFunction = async ( @@ -106,6 +108,8 @@ export const runWebpack4: BundlerRunFunction = async ( ) => { const bundlerConfigs = getWebpack4Options(seed, pluginOverrides, bundlerOverrides); const webpack = (await import('webpack4')).default; + const errors = []; + try { await new Promise((resolve, reject) => { webpack(bundlerConfigs, (err, stats) => { @@ -113,10 +117,11 @@ export const runWebpack4: BundlerRunFunction = async ( }); }); } catch (e: any) { - console.error(`Build failed for Webpack 5`, e); + console.error(`Build failed for Webpack 4`, e); + errors.push(e.message); } - return getCleanupFunction('Webpack 4', [bundlerConfigs.output?.path]); + return { cleanup: getCleanupFunction('Webpack 4', [bundlerConfigs.output?.path]), errors }; }; export const runEsbuild: BundlerRunFunction = async ( @@ -126,14 +131,16 @@ export const runEsbuild: BundlerRunFunction = async ( ) => { const bundlerConfigs = getEsbuildOptions(seed, pluginOverrides, bundlerOverrides); const { build } = await import('esbuild'); + const errors = []; try { await build(bundlerConfigs); } catch (e: any) { console.error(`Build failed for ESBuild`, e); + errors.push(e.message); } - return getCleanupFunction('ESBuild', [bundlerConfigs.outdir]); + return { cleanup: getCleanupFunction('ESBuild', [bundlerConfigs.outdir]), errors }; }; export const runVite: BundlerRunFunction = async ( @@ -143,10 +150,12 @@ export const runVite: BundlerRunFunction = async ( ) => { const bundlerConfigs = getViteOptions(seed, pluginOverrides, bundlerOverrides); const vite = await import('vite'); + const errors = []; try { await vite.build(bundlerConfigs); - } catch (e) { + } catch (e: any) { console.error(`Build failed for Vite`, e); + errors.push(e.message); } const outdirs: (string | undefined)[] = []; @@ -156,7 +165,7 @@ export const runVite: BundlerRunFunction = async ( outdirs.push(bundlerConfigs.build.rollupOptions.output.dir); } - return getCleanupFunction('Vite', outdirs); + return { cleanup: getCleanupFunction('Vite', outdirs), errors }; }; export const runRollup: BundlerRunFunction = async ( @@ -166,6 +175,7 @@ export const runRollup: BundlerRunFunction = async ( ) => { const bundlerConfigs = getRollupOptions(seed, pluginOverrides, bundlerOverrides); const { rollup } = await import('rollup'); + const errors = []; try { const result = await rollup(bundlerConfigs); @@ -182,8 +192,9 @@ export const runRollup: BundlerRunFunction = async ( await Promise.all(outputProms); } - } catch (e) { + } catch (e: any) { console.error(`Build failed for Rollup`, e); + errors.push(e.message); } const outdirs: (string | undefined)[] = []; @@ -192,7 +203,8 @@ export const runRollup: BundlerRunFunction = async ( } else if (bundlerConfigs.output?.dir) { outdirs.push(bundlerConfigs.output.dir); } - return getCleanupFunction('Rollup', outdirs); + + return { cleanup: getCleanupFunction('Rollup', outdirs), errors }; }; export type Bundler = { @@ -248,6 +260,7 @@ export const runBundlers = async ( bundlers?: string[], ): Promise => { const cleanups: CleanupFn[] = []; + const errors: string[] = []; // Generate a seed to avoid collision of builds. const seed: string = `${Date.now()}-${jest.getSeed()}`; @@ -278,20 +291,34 @@ export const runBundlers = async ( if (webpackBundlers.length) { const webpackProms = webpackBundlers.map(runBundlerFunction); - const cleanupFns = await Promise.all(webpackProms); - cleanups.push(...cleanupFns); + const results = await Promise.all(webpackProms); + cleanups.push(...results.map((result) => result.cleanup)); + errors.push(...results.map((result) => result.errors).flat()); } if (otherBundlers.length) { const otherProms = otherBundlers.map(runBundlerFunction); - const cleanupFns = await Promise.all(otherProms); - cleanups.push(...cleanupFns); + const results = await Promise.all(otherProms); + cleanups.push(...results.map((result) => result.cleanup)); + errors.push(...results.map((result) => result.errors).flat()); } - // Return a cleanUp function. - return async () => { - await Promise.all(cleanups.map((cleanup) => cleanup())); - // Remove the seeded directory. - await remove(path.resolve(defaultDestination, seed)); + const cleanupEverything = async () => { + try { + await Promise.all(cleanups.map((cleanup) => cleanup())); + // Remove the seeded directory. + await remove(path.resolve(defaultDestination, seed)); + } catch (e) { + console.error('Error during cleanup', e); + } }; + + if (errors.length) { + // We'll throw, so clean everything first. + await cleanupEverything(); + throw new Error(errors.join('\n')); + } + + // Return a cleanUp function. + return cleanupEverything; }; From 8abaf1406b03076fd426ecf653a88b742287a0c6 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 8 Oct 2024 17:41:49 +0200 Subject: [PATCH 08/30] Add tests for tool helpers --- packages/tests/src/tools/src/helpers.test.ts | 209 +++++++++++++++++++ packages/tools/src/helpers.ts | 12 +- 2 files changed, 215 insertions(+), 6 deletions(-) create mode 100644 packages/tests/src/tools/src/helpers.test.ts diff --git a/packages/tests/src/tools/src/helpers.test.ts b/packages/tests/src/tools/src/helpers.test.ts new file mode 100644 index 00000000..f337fc01 --- /dev/null +++ b/packages/tests/src/tools/src/helpers.test.ts @@ -0,0 +1,209 @@ +import { + getCamelCase, + getPascalCase, + getTitle, + replaceInBetween, + slugify, +} from '@dd/tools/helpers'; + +describe('Tools helpers', () => { + describe('slugify', () => { + const cases = [ + { + description: 'convert a string to lowercase', + input: 'Hello World', + expectation: 'hello-world', + }, + { + description: 'remove accents from characters', + input: 'Çäfé', + expectation: 'cafe', + }, + { + description: 'replace spaces with hyphens', + input: 'hello world', + expectation: 'hello-world', + }, + { + description: 'remove special characters', + input: 'hello@world!', + expectation: 'helloworld', + }, + { + description: 'trim leading and trailing spaces', + input: ' hello world ', + expectation: 'hello-world', + }, + { + description: 'handle empty strings', + input: '', + expectation: '', + }, + { + description: 'handle strings with only special characters', + input: '!@#$%^&*()', + expectation: '', + }, + { + description: 'handle strings with multiple spaces', + input: 'hello world', + expectation: 'hello-world', + }, + { + description: 'handle strings with mixed case and special characters', + input: 'Hello@World! 123', + expectation: 'helloworld-123', + }, + ]; + test.each(cases)('should $description', ({ input, expectation }) => { + expect(slugify(input)).toBe(expectation); + }); + }); + + describe('replaceInBetween', () => { + const cases = [ + { + description: 'should replace content between two markers', + content: '\n\n', + mark: '/* MARK */', + injection: 'Hello World', + expectation: '/* MARK */\nHello World\n/* MARK */', + }, + { + description: 'should handle multiple lines between markers', + content: 'Line 1\nLine 2\n', + mark: '/* MARK */', + injection: 'New Line 1\nNew Line 2', + expectation: '/* MARK */\nNew Line 1\nNew Line 2\n/* MARK */', + }, + { + description: 'should handle special characters in markers', + content: '\n\n', + mark: '', + injection: 'Hello World', + expectation: '\nHello World\n', + }, + { + description: 'should handle no content between markers', + content: '', + mark: '{{MARK}}', + injection: 'Hello World', + expectation: '{{MARK}}\nHello World\n{{MARK}}', + }, + { + description: 'should handle markers with regex special characters', + content: '\n\n', + mark: '/* M*\\/.K */', + injection: 'Hel$lo $$World $100', + expectation: '/* M*\\/.K */\nHel$lo $$World $100\n/* M*\\/.K */', + }, + ]; + + test.each(cases)('should $description', ({ content, mark, injection, expectation }) => { + const fullContent = `${mark}${content}${mark}`; + expect(replaceInBetween(fullContent, mark, injection)).toBe(expectation); + }); + }); + + describe('getTitle', () => { + const cases = [ + { + description: 'convert a string to title case', + input: 'hello-world', + expectation: 'Hello World', + }, + { + description: 'handle strings with multiple hyphens', + input: 'hello-world-123', + expectation: 'Hello World 123', + }, + { + description: 'handle strings with special characters', + input: 'hello-world@123', + expectation: 'Hello World@123', + }, + { + description: 'handle strings with mixed case', + input: 'HelLo---WORLD', + expectation: 'Hello World', + }, + { + description: 'handle empty strings', + input: '', + expectation: '', + }, + ]; + + test.each(cases)('should $description', ({ input, expectation }) => { + expect(getTitle(input)).toBe(expectation); + }); + }); + + describe('getPascaleCase', () => { + const cases = [ + { + description: 'convert a string to PascalCase', + input: 'hello-world', + expectation: 'HelloWorld', + }, + { + description: 'handle strings with multiple hyphens', + input: 'hello-world-123', + expectation: 'HelloWorld123', + }, + { + description: 'handle strings with special characters', + input: 'hello-world@123', + expectation: 'HelloWorld@123', + }, + { + description: 'handle strings with mixed case', + input: 'HelLo---WORLD', + expectation: 'HelloWorld', + }, + { + description: 'handle empty strings', + input: '', + expectation: '', + }, + ]; + + test.each(cases)('should $description', ({ input, expectation }) => { + expect(getPascalCase(input)).toBe(expectation); + }); + }); + + describe('getCamelCase', () => { + const cases = [ + { + description: 'convert a string to camelCase', + input: 'hello-world', + expectation: 'helloWorld', + }, + { + description: 'handle strings with multiple hyphens', + input: 'hello-world-123', + expectation: 'helloWorld123', + }, + { + description: 'handle strings with special characters', + input: 'hello-world@123', + expectation: 'helloWorld@123', + }, + { + description: 'handle strings with mixed case', + input: 'HelLo---WORLD', + expectation: 'helloWorld', + }, + { + description: 'handle empty strings', + input: '', + expectation: '', + }, + ]; + + test.each(cases)('should $description', ({ input, expectation }) => { + expect(getCamelCase(input)).toBe(expectation); + }); + }); +}); diff --git a/packages/tools/src/helpers.ts b/packages/tools/src/helpers.ts index 05bb8aa2..bb873535 100644 --- a/packages/tools/src/helpers.ts +++ b/packages/tools/src/helpers.ts @@ -38,19 +38,19 @@ export const slugify = (string: string) => { // Inject some text in between two markers. export const replaceInBetween = (content: string, mark: string, injection: string) => { - const rx = new RegExp(`${mark}[\\S\\s]*${mark}`, 'gm'); - return content.replace(rx, `${mark}\n${injection}\n${mark}`); + const escapedMark = mark.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const escapedInjection = injection.replace(/\$/g, '$$$$'); + const rx = new RegExp(`${escapedMark}[\\S\\s]*${escapedMark}`, 'gm'); + return content.replace(rx, `${mark}\n${escapedInjection}\n${mark}`); }; export const getTitle = (name: string): string => name - .split('-') + .toLowerCase() + .split(/-+/g) .map((s) => s.charAt(0).toUpperCase() + s.slice(1)) .join(' '); -export const getUpperCase = (name: string): string => - getTitle(name).toUpperCase().replace(/ /g, '_'); - export const getPascalCase = (name: string): string => getTitle(name).replace(/ /g, ''); export const getCamelCase = (name: string): string => { From 8bd0302604e5da61d1ae49b96c3748abb0babae3 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 8 Oct 2024 17:45:50 +0200 Subject: [PATCH 09/30] Add tests for bundles --- .../tests/src/tools/src/rollupConfig.test.ts | 96 +++++++++++++++++++ packages/tests/tsconfig.json | 8 ++ 2 files changed, 104 insertions(+) create mode 100644 packages/tests/src/tools/src/rollupConfig.test.ts create mode 100644 packages/tests/tsconfig.json diff --git a/packages/tests/src/tools/src/rollupConfig.test.ts b/packages/tests/src/tools/src/rollupConfig.test.ts new file mode 100644 index 00000000..4b22ec1d --- /dev/null +++ b/packages/tests/src/tools/src/rollupConfig.test.ts @@ -0,0 +1,96 @@ +import { datadogEsbuildPlugin } from '@datadog/esbuild-plugin'; +import { datadogRollupPlugin } from '@datadog/rollup-plugin'; +import { datadogVitePlugin } from '@datadog/vite-plugin'; +import { datadogWebpackPlugin } from '@datadog/webpack-plugin'; +import { + API_PATH, + FAKE_URL, + getComplexBuildOverrides, + getFullPluginConfig, +} from '@dd/tests/helpers/mocks'; +import { BUNDLERS } from '@dd/tests/helpers/runBundlers'; +import { ROOT } from '@dd/tools/constants'; +import { execute } from '@dd/tools/helpers'; +import { removeSync } from 'fs-extra'; +import nock from 'nock'; +import path from 'path'; + +// Mock all the published packages so we can replace them with the built ones. +jest.mock('@datadog/esbuild-plugin', () => ({ + datadogEsbuildPlugin: jest.fn(), +})); +jest.mock('@datadog/rollup-plugin', () => ({ + datadogRollupPlugin: jest.fn(), +})); +jest.mock('@datadog/vite-plugin', () => ({ + datadogVitePlugin: jest.fn(), +})); +jest.mock('@datadog/webpack-plugin', () => ({ + datadogWebpackPlugin: jest.fn(), +})); + +const datadogWebpackPluginMock = jest.mocked(datadogWebpackPlugin); +const datadogEsbuildPluginMock = jest.mocked(datadogEsbuildPlugin); +const datadogRollupPluginMock = jest.mocked(datadogRollupPlugin); +const datadogVitePluginMock = jest.mocked(datadogVitePlugin); + +describe('Bundling', () => { + const complexProjectOverrides = getComplexBuildOverrides(); + const pluginConfig = getFullPluginConfig(); + beforeAll(async () => { + // First, bundle the plugins. + // FIXME: This is slow because of the dts() build. + await execute('yarn', ['build:all']); + + // Make the mocks target the built packages. + const getPackageDestination = (bundlerName: string) => { + return path.resolve(ROOT, `packages/${bundlerName}-plugin/dist/src`); + }; + + datadogWebpackPluginMock.mockImplementation( + jest.requireActual(getPackageDestination('webpack')).datadogWebpackPlugin, + ); + datadogEsbuildPluginMock.mockImplementation( + jest.requireActual(getPackageDestination('esbuild')).datadogEsbuildPlugin, + ); + datadogRollupPluginMock.mockImplementation( + jest.requireActual(getPackageDestination('rollup')).datadogRollupPlugin, + ); + datadogVitePluginMock.mockImplementation( + jest.requireActual(getPackageDestination('vite')).datadogVitePlugin, + ); + + // Mock network requests. + nock(FAKE_URL) + .persist() + // For sourcemaps submissions. + .post(API_PATH) + .reply(200, {}) + // For metrics submissions. + .post('/api/v1/series?api_key=123') + .reply(200, {}); + }, 30000); + + afterAll(async () => { + nock.cleanAll(); + removeSync(path.resolve(ROOT, 'packages/tests/src/fixtures/dist')); + }); + + describe.each(BUNDLERS)('Bundler: $name', (bundler) => { + test('Should not throw on a simple project.', async () => { + const SEED = `${Date.now()}-${jest.getSeed()}`; + const { errors } = await bundler.run(SEED, pluginConfig, {}); + expect(errors).toHaveLength(0); + }); + + test('Should not throw on a complex project.', async () => { + const SEED = `${Date.now()}-${jest.getSeed()}`; + const { errors } = await bundler.run( + SEED, + pluginConfig, + complexProjectOverrides[bundler.name], + ); + expect(errors).toHaveLength(0); + }); + }); +}); diff --git a/packages/tests/tsconfig.json b/packages/tests/tsconfig.json new file mode 100644 index 00000000..766073b8 --- /dev/null +++ b/packages/tests/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "allowJs": true + }, + "include": ["**/*"], + "exclude": ["dist", "node_modules"] +} From 4d439188d454815a8a502e6d95235716add0bd79 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Tue, 8 Oct 2024 17:56:22 +0200 Subject: [PATCH 10/30] Factorise injection detection --- packages/core/src/helpers.ts | 4 ++++ packages/core/src/plugins/build-report/esbuild.ts | 4 ++-- packages/core/src/plugins/build-report/helpers.ts | 4 +--- packages/core/src/plugins/build-report/webpack.ts | 3 ++- packages/core/src/plugins/injection/index.ts | 11 ++++------- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts index 303e0a4a..3b835290 100644 --- a/packages/core/src/helpers.ts +++ b/packages/core/src/helpers.ts @@ -5,6 +5,7 @@ import retry from 'async-retry'; import type { RequestInit } from 'undici-types'; +import { INJECTED_FILE } from './plugins/injection/constants'; import type { RequestOpts } from './types'; // Format a duration 0h 0m 0s 0ms @@ -117,3 +118,6 @@ export const truncateString = ( return `${str.slice(0, leftStop)}${placeholder}${str.slice(-rightStop)}`; }; + +// Is the file coming from the injection plugin? +export const isInjection = (filename: string) => filename.includes(INJECTED_FILE); diff --git a/packages/core/src/plugins/build-report/esbuild.ts b/packages/core/src/plugins/build-report/esbuild.ts index 8c7562f9..82d2a2eb 100644 --- a/packages/core/src/plugins/build-report/esbuild.ts +++ b/packages/core/src/plugins/build-report/esbuild.ts @@ -2,12 +2,12 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { getResolvedPath } from '@dd/core/helpers'; +import { getResolvedPath, isInjection } from '@dd/core/helpers'; import type { Logger } from '@dd/core/log'; import type { Entry, GlobalContext, Input, Output, PluginOptions } from '@dd/core/types'; import { glob } from 'glob'; -import { cleanName, getAbsolutePath, getType, isInjection } from './helpers'; +import { cleanName, getAbsolutePath, getType } from './helpers'; // Re-index metafile data for easier access. const reIndexMeta = (obj: Record, cwd: string) => diff --git a/packages/core/src/plugins/build-report/helpers.ts b/packages/core/src/plugins/build-report/helpers.ts index a2647579..6827b5f6 100644 --- a/packages/core/src/plugins/build-report/helpers.ts +++ b/packages/core/src/plugins/build-report/helpers.ts @@ -2,6 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { isInjection } from '@dd/core/helpers'; import { INJECTED_FILE } from '@dd/core/plugins/injection/constants'; import type { BuildReport, @@ -189,9 +190,6 @@ export const unserializeBuildReport = (report: SerializedBuildReport): BuildRepo }; }; -// Is the file coming from the injection plugin? -export const isInjection = (filename: string) => filename.includes(INJECTED_FILE); - const BUNDLER_SPECIFICS = ['unknown', 'commonjsHelpers.js', 'vite/preload-helper.js']; // Make list of paths unique, remove the current file and particularities. export const cleanReport = ( diff --git a/packages/core/src/plugins/build-report/webpack.ts b/packages/core/src/plugins/build-report/webpack.ts index b3c4c7b9..8d7dc673 100644 --- a/packages/core/src/plugins/build-report/webpack.ts +++ b/packages/core/src/plugins/build-report/webpack.ts @@ -2,10 +2,11 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { isInjection } from '@dd/core/helpers'; import type { Logger } from '@dd/core/log'; import type { Entry, GlobalContext, Input, Output, PluginOptions } from '@dd/core/types'; -import { cleanName, cleanReport, getAbsolutePath, getType, isInjection } from './helpers'; +import { cleanName, cleanReport, getAbsolutePath, getType } from './helpers'; export const getWebpackPlugin = (context: GlobalContext, PLUGIN_NAME: string, log: Logger): PluginOptions['webpack'] => diff --git a/packages/core/src/plugins/injection/index.ts b/packages/core/src/plugins/injection/index.ts index fba16045..62a30fc6 100644 --- a/packages/core/src/plugins/injection/index.ts +++ b/packages/core/src/plugins/injection/index.ts @@ -2,6 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { isInjection } from '@dd/core/helpers'; import { getLogger } from '@dd/core/log'; import type { GlobalContext, Options, PluginOptions, ToInjectItem } from '@dd/core/types'; @@ -48,10 +49,6 @@ export const getInjectionPlugins = ( }, }; - const isInjectedFile = (id: string) => { - return id.includes(INJECTED_FILE); - }; - // This plugin happens in 3 steps in order to cover all bundlers: // 1. Prepare the content to inject, fetching distant/local files and anything necessary. // 2. Inject a virtual file into the bundling, this file will be home of all injected content. @@ -141,17 +138,17 @@ export const getInjectionPlugins = ( name: RESOLUTION_PLUGIN_NAME, enforce: 'pre', resolveId(id) { - if (isInjectedFile(id)) { + if (isInjection(id)) { return { id, moduleSideEffects: true }; } }, loadInclude(id) { - if (isInjectedFile(id)) { + if (isInjection(id)) { return true; } }, load(id) { - if (isInjectedFile(id)) { + if (isInjection(id)) { return getContentToInject(); } }, From b11061c834c1bff1b197d547c46dfaa006c7925f Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 9 Oct 2024 11:21:11 +0200 Subject: [PATCH 11/30] Add back skipped test --- packages/tests/src/core/helpers.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tests/src/core/helpers.test.ts b/packages/tests/src/core/helpers.test.ts index 8bbf3afd..0b15b82c 100644 --- a/packages/tests/src/core/helpers.test.ts +++ b/packages/tests/src/core/helpers.test.ts @@ -124,7 +124,7 @@ describe('Core Helpers', () => { }); }); - describe.only('truncateString', () => { + describe('truncateString', () => { test.each([ // No truncation needed. ['Short string', 20, '[...]', 'Short string'], From 0714640f6af4c7c3406d23d64aeaed6eed846090 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 9 Oct 2024 11:21:47 +0200 Subject: [PATCH 12/30] Factories webpack entries --- packages/tests/src/helpers/configBundlers.ts | 21 +++++++++----- packages/tests/src/helpers/mocks.ts | 29 ++++++++------------ 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/tests/src/helpers/configBundlers.ts b/packages/tests/src/helpers/configBundlers.ts index 46b5afb2..6f12399a 100644 --- a/packages/tests/src/helpers/configBundlers.ts +++ b/packages/tests/src/helpers/configBundlers.ts @@ -62,12 +62,20 @@ export const getWebpack5Options = ( }; // Webpack 4 doesn't support pnp resolution OOTB. -export const getWebpack4Entries = (entries: Record) => { +export const getWebpack4Entries = ( + entries: string | Record, + cwd: string = process.cwd(), +) => { + const getTrueRelativePath = (filepath: string) => { + return `./${path.relative(cwd, getResolvedPath(filepath))}`; + }; + + if (typeof entries === 'string') { + return getTrueRelativePath(entries); + } + return Object.fromEntries( - Object.entries(entries).map(([name, filepath]) => [ - name, - `./${path.relative(process.cwd(), getResolvedPath(filepath))}`, - ]), + Object.entries(entries).map(([name, filepath]) => [name, getTrueRelativePath(filepath)]), ); }; @@ -85,8 +93,7 @@ export const getWebpack4Options = ( return { ...(getBaseWebpackConfig(seed, 'webpack4') as Configuration4), - // Webpack4 doesn't support pnp resolution. - entry: `./${path.relative(process.cwd(), getResolvedPath(defaultEntry))}`, + entry: getWebpack4Entries(defaultEntry), plugins: [plugin as unknown as Plugin], node: false, ...bundlerOverrides, diff --git a/packages/tests/src/helpers/mocks.ts b/packages/tests/src/helpers/mocks.ts index 0cddd634..95c20cbc 100644 --- a/packages/tests/src/helpers/mocks.ts +++ b/packages/tests/src/helpers/mocks.ts @@ -2,12 +2,13 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { getResolvedPath } from '@dd/core/helpers'; import type { GlobalContext, Options } from '@dd/core/types'; import { getSourcemapsConfiguration } from '@dd/tests/plugins/rum/testHelpers'; import { getTelemetryConfiguration } from '@dd/tests/plugins/telemetry/testHelpers'; import path from 'path'; +import { getWebpack4Entries } from './configBundlers'; + if (!process.env.PROJECT_CWD) { throw new Error('Please update the usage of `process.env.PROJECT_CWD`.'); } @@ -19,6 +20,10 @@ export const API_PATH = '/v2/srcmap'; export const INTAKE_URL = `${FAKE_URL}${API_PATH}`; export const defaultEntry = '@dd/tests/fixtures/main.js'; +export const defaultEntries = { + app1: '@dd/tests/fixtures/project/main1.js', + app2: '@dd/tests/fixtures/project/main2.js', +}; export const defaultDestination = path.resolve(PROJECT_ROOT, '../dist'); export const defaultPluginOptions: Options = { @@ -51,34 +56,22 @@ export const getContextMock = (options: Partial = {}): GlobalCont export const getComplexBuildOverrides = ( overrides: Record = {}, ): Record => { - // Add more entries with more dependencies. - const entries = { - app1: '@dd/tests/fixtures/project/main1.js', - app2: '@dd/tests/fixtures/project/main2.js', - }; - const bundlerOverrides = { rollup: { - input: entries, + input: defaultEntries, ...overrides.rollup, }, vite: { - input: entries, + input: defaultEntries, ...overrides.vite, }, esbuild: { - entryPoints: entries, + entryPoints: defaultEntries, ...overrides.esbuild, }, - webpack5: { entry: entries, ...overrides.webpack5 }, + webpack5: { entry: defaultEntries, ...overrides.webpack5 }, webpack4: { - // Webpack 4 doesn't support pnp. - entry: Object.fromEntries( - Object.entries(entries).map(([name, filepath]) => [ - name, - `./${path.relative(process.cwd(), getResolvedPath(filepath))}`, - ]), - ), + entry: getWebpack4Entries(defaultEntries), ...overrides.webpack4, }, }; From 96fe4c0cc3c9e0e65c2ba88cb4923406e130be89 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 9 Oct 2024 11:30:17 +0200 Subject: [PATCH 13/30] Add missing webpack-sources dependency --- packages/webpack-plugin/package.json | 13 +++++++++++-- yarn.lock | 19 ++++++++++++------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/packages/webpack-plugin/package.json b/packages/webpack-plugin/package.json index 537c05a0..3a2a0e0a 100644 --- a/packages/webpack-plugin/package.json +++ b/packages/webpack-plugin/package.json @@ -45,6 +45,9 @@ "typecheck": "tsc --noEmit", "watch": "yarn clean && rollup --config rollup.config.mjs --watch" }, + "dependencies": { + "webpack-sources": "3.2.3" + }, "devDependencies": { "@babel/core": "7.24.5", "@babel/preset-env": "7.24.5", @@ -64,6 +67,12 @@ "typescript": "5.4.3" }, "peerDependencies": { - "webpack": ">= 4.x < 6.x" - } + "webpack": ">= 4.x < 6.x", + "webpack-sources": "^3" + }, + "peerDependenciesMeta": { + "webpack-sources": { + "optional": true + } + } } diff --git a/yarn.lock b/yarn.lock index 83e10186..43670cea 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1771,8 +1771,13 @@ __metadata: rollup-plugin-dts: "npm:6.1.0" rollup-plugin-esbuild: "npm:6.1.1" typescript: "npm:5.4.3" + webpack-sources: "npm:3.2.3" peerDependencies: webpack: ">= 4.x < 6.x" + webpack-sources: ^3 + peerDependenciesMeta: + webpack-sources: + optional: true languageName: unknown linkType: soft @@ -12133,6 +12138,13 @@ __metadata: languageName: node linkType: hard +"webpack-sources@npm:3.2.3, webpack-sources@npm:^3.2.3": + version: 3.2.3 + resolution: "webpack-sources@npm:3.2.3" + checksum: 10/a661f41795d678b7526ae8a88cd1b3d8ce71a7d19b6503da8149b2e667fc7a12f9b899041c1665d39e38245ed3a59ab68de648ea31040c3829aa695a5a45211d + languageName: node + linkType: hard + "webpack-sources@npm:^1.4.0, webpack-sources@npm:^1.4.1": version: 1.4.3 resolution: "webpack-sources@npm:1.4.3" @@ -12143,13 +12155,6 @@ __metadata: languageName: node linkType: hard -"webpack-sources@npm:^3.2.3": - version: 3.2.3 - resolution: "webpack-sources@npm:3.2.3" - checksum: 10/a661f41795d678b7526ae8a88cd1b3d8ce71a7d19b6503da8149b2e667fc7a12f9b899041c1665d39e38245ed3a59ab68de648ea31040c3829aa695a5a45211d - languageName: node - linkType: hard - "webpack-virtual-modules@npm:^0.6.1": version: 0.6.1 resolution: "webpack-virtual-modules@npm:0.6.1" From 16207ab7cfe921017c5a6df5e6f13a888b7e9e52 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 9 Oct 2024 11:30:31 +0200 Subject: [PATCH 14/30] Re-order injection plugin definition --- packages/core/src/plugins/injection/index.ts | 46 ++++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/packages/core/src/plugins/injection/index.ts b/packages/core/src/plugins/injection/index.ts index 62a30fc6..83e6d32f 100644 --- a/packages/core/src/plugins/injection/index.ts +++ b/packages/core/src/plugins/injection/index.ts @@ -50,10 +50,30 @@ export const getInjectionPlugins = ( }; // This plugin happens in 3 steps in order to cover all bundlers: - // 1. Prepare the content to inject, fetching distant/local files and anything necessary. - // 2. Inject a virtual file into the bundling, this file will be home of all injected content. - // 3. Resolve the virtual file, returning the prepared injected content. + // 1. Setup resolvers for the virtual file, returning the prepared injected content. + // 2. Prepare the content to inject, fetching distant/local files and anything necessary. + // 3. Inject a virtual file into the bundling, this file will be home of all injected content. return [ + // Resolve the injected file. + { + name: RESOLUTION_PLUGIN_NAME, + enforce: 'pre', + resolveId(id) { + if (isInjection(id)) { + return { id, moduleSideEffects: true }; + } + }, + loadInclude(id) { + if (isInjection(id)) { + return true; + } + }, + load(id) { + if (isInjection(id)) { + return getContentToInject(); + } + }, + }, // Prepare and fetch the content to inject. { name: PREPARATION_PLUGIN_NAME, @@ -133,25 +153,5 @@ export const getInjectionPlugins = ( rollup: rollupInjectionPlugin, vite: rollupInjectionPlugin, }, - // Resolve the injected file. - { - name: RESOLUTION_PLUGIN_NAME, - enforce: 'pre', - resolveId(id) { - if (isInjection(id)) { - return { id, moduleSideEffects: true }; - } - }, - loadInclude(id) { - if (isInjection(id)) { - return true; - } - }, - load(id) { - if (isInjection(id)) { - return getContentToInject(); - } - }, - }, ]; }; From 3a6b86ddeaa96e7d5fd61691ea62c1c29707d76c Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 9 Oct 2024 11:30:48 +0200 Subject: [PATCH 15/30] Remove logs from bundling test --- packages/tests/src/tools/src/rollupConfig.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/tests/src/tools/src/rollupConfig.test.ts b/packages/tests/src/tools/src/rollupConfig.test.ts index 4b22ec1d..3fd168f7 100644 --- a/packages/tests/src/tools/src/rollupConfig.test.ts +++ b/packages/tests/src/tools/src/rollupConfig.test.ts @@ -36,7 +36,9 @@ const datadogVitePluginMock = jest.mocked(datadogVitePlugin); describe('Bundling', () => { const complexProjectOverrides = getComplexBuildOverrides(); - const pluginConfig = getFullPluginConfig(); + const pluginConfig = getFullPluginConfig({ + logLevel: 'error', + }); beforeAll(async () => { // First, bundle the plugins. // FIXME: This is slow because of the dts() build. From 122005b725016b4beb8778deb3631ca096eb95fc Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 9 Oct 2024 11:56:31 +0200 Subject: [PATCH 16/30] Fix missing protocol in telemetry endPoint --- .../plugins/telemetry/src/common/helpers.ts | 3 +- .../plugins/telemetry/common/helpers.test.ts | 134 +++++++++++++----- 2 files changed, 98 insertions(+), 39 deletions(-) diff --git a/packages/plugins/telemetry/src/common/helpers.ts b/packages/plugins/telemetry/src/common/helpers.ts index 836c6fc4..6c4d7835 100644 --- a/packages/plugins/telemetry/src/common/helpers.ts +++ b/packages/plugins/telemetry/src/common/helpers.ts @@ -21,15 +21,16 @@ import { defaultFilters } from './filters'; export const validateOptions = (opts: OptionsWithTelemetry): TelemetryOptionsWithDefaults => { const options: TelemetryOptions = opts[CONFIG_KEY] || {}; + const endPoint = options.endPoint || 'https://app.datadoghq.com'; return { disabled: false, enableTracing: false, - endPoint: 'app.datadoghq.com', filters: defaultFilters, output: false, prefix: '', tags: [], ...options, + endPoint: endPoint.startsWith('http') ? endPoint : `https://${endPoint}`, }; }; diff --git a/packages/tests/src/plugins/telemetry/common/helpers.test.ts b/packages/tests/src/plugins/telemetry/common/helpers.test.ts index b98cb179..ccfe0e11 100644 --- a/packages/tests/src/plugins/telemetry/common/helpers.test.ts +++ b/packages/tests/src/plugins/telemetry/common/helpers.test.ts @@ -2,51 +2,109 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { getModuleName, getValueContext } from '@dd/telemetry-plugins/common/helpers'; +import { defaultFilters } from '@dd/telemetry-plugins/common/filters'; +import { + getModuleName, + getValueContext, + validateOptions, +} from '@dd/telemetry-plugins/common/helpers'; +import { CONFIG_KEY } from '@dd/telemetry-plugins'; import { getMockCompilation, getMockModule, mockCompilation } from '../testHelpers'; describe('Telemetry Helpers', () => { - test('It should use the module with webpack4', () => { - const mockModule = getMockModule({ name: 'moduleName' }); - expect(getModuleName(mockModule, mockCompilation)).toBe('moduleName'); + describe('validateOptions', () => { + test('Should return the default options', () => { + const options = { [CONFIG_KEY]: {} }; + expect(validateOptions(options)).toEqual({ + disabled: false, + enableTracing: false, + endPoint: 'https://app.datadoghq.com', + filters: defaultFilters, + output: false, + prefix: '', + tags: [], + }); + }); + + test('Should return the options with the provided values', () => { + const fakeFilter = jest.fn(); + const options = { + [CONFIG_KEY]: { + disabled: true, + enableTracing: true, + endPoint: 'https://app.datadoghq.eu', + filters: [fakeFilter], + output: true, + prefix: 'prefix', + tags: ['tag1'], + }, + }; + expect(validateOptions(options)).toEqual({ + disabled: true, + enableTracing: true, + endPoint: 'https://app.datadoghq.eu', + filters: [fakeFilter], + output: true, + prefix: 'prefix', + tags: ['tag1'], + }); + }); + + test('Should add https:// if the endpoint does not have one', () => { + const options = { + [CONFIG_KEY]: { + endPoint: 'app.datadoghq.eu', + }, + }; + expect(validateOptions(options).endPoint).toBe('https://app.datadoghq.eu'); + }); }); - test('It should use the moduleGraphAPI with webpack5', () => { - const unnamedModule = getMockModule({ name: '' }); - const namedModule = getMockModule({ userRequest: 'moduleName' }); - expect( - getModuleName( - unnamedModule, - getMockCompilation({ - moduleGraph: { - getIssuer: () => namedModule, - getModule: () => namedModule, - issuer: namedModule, - }, - }), - ), - ).toBe('moduleName'); + describe('getModuleName', () => { + test('Should use the module with webpack4', () => { + const mockModule = getMockModule({ name: 'moduleName' }); + expect(getModuleName(mockModule, mockCompilation)).toBe('moduleName'); + }); + + test('Should use the moduleGraphAPI with webpack5', () => { + const unnamedModule = getMockModule({ name: '' }); + const namedModule = getMockModule({ userRequest: 'moduleName' }); + expect( + getModuleName( + unnamedModule, + getMockCompilation({ + moduleGraph: { + getIssuer: () => namedModule, + getModule: () => namedModule, + issuer: namedModule, + }, + }), + ), + ).toBe('moduleName'); + }); }); - test('It should getContext with and without constructor', () => { - const BasicClass: any = function BasicClass() {}; - const instance1 = new BasicClass(); - const instance2 = new BasicClass(); - instance2.constructor = null; - - expect(() => { - getValueContext([instance1, instance2]); - }).not.toThrow(); - - const context = getValueContext([instance1, instance2]); - expect(context).toEqual([ - { - type: 'BasicClass', - }, - { - type: 'object', - }, - ]); + describe('getValueContext', () => { + test('It should getContext with and without constructor', () => { + const BasicClass: any = function BasicClass() {}; + const instance1 = new BasicClass(); + const instance2 = new BasicClass(); + instance2.constructor = null; + + expect(() => { + getValueContext([instance1, instance2]); + }).not.toThrow(); + + const context = getValueContext([instance1, instance2]); + expect(context).toEqual([ + { + type: 'BasicClass', + }, + { + type: 'object', + }, + ]); + }); }); }); From 33c90fba94af6578b6979787fe30cc3c7295a50c Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 9 Oct 2024 15:17:01 +0200 Subject: [PATCH 17/30] Ensure we start heavy tests first --- packages/tests/jest.config.js | 3 +- packages/tests/src/helpers/customSequencer.js | 36 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 packages/tests/src/helpers/customSequencer.js diff --git a/packages/tests/jest.config.js b/packages/tests/jest.config.js index 72f3f5a8..327cddba 100644 --- a/packages/tests/jest.config.js +++ b/packages/tests/jest.config.js @@ -8,9 +8,10 @@ module.exports = { // Without it, vite import is silently crashing the process with code SIGHUP 129 resetModules: true, preset: 'ts-jest/presets/js-with-ts', - reporters: [['default', { summaryThreshold: 0 }]], + reporters: [['default', { summaryThreshold: 2 }]], testEnvironment: 'node', testMatch: ['**/*.test.*'], roots: ['./src/'], setupFilesAfterEnv: ['/src/setupTests.ts'], + testSequencer: '/src/helpers/customSequencer.js', }; diff --git a/packages/tests/src/helpers/customSequencer.js b/packages/tests/src/helpers/customSequencer.js new file mode 100644 index 00000000..a4c6d0ee --- /dev/null +++ b/packages/tests/src/helpers/customSequencer.js @@ -0,0 +1,36 @@ +const Sequencer = require('@jest/test-sequencer').default; + +/** @typedef {Parameters[0]} Tests */ +/** @typedef {Tests[number]} Test */ + +/** + * @param {Test} test + * @returns {boolean} + */ +const isHeavyTest = (test) => { + return test.path.endsWith('src/tools/src/rollupConfig.test.ts'); +}; + +module.exports = class CustomSequencer extends Sequencer { + /** + * @param {Tests} tests + * @returns {Promise} + */ + async sort(tests) { + /** @type {Tests} */ + const sortedTests = []; + + // First, add the heavy tests. + for (const test of tests) { + if (isHeavyTest(test)) { + sortedTests.push(test); + } + } + + // Then add the rest of the tests, using the default sort. + const superSortedTests = await super.sort(tests.filter((test) => !isHeavyTest(test))); + sortedTests.push(...superSortedTests); + + return sortedTests; + } +}; From 9d10e172e40cef3508ccb10840b0df8b5ef30438 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 9 Oct 2024 15:37:13 +0200 Subject: [PATCH 18/30] Integrity --- packages/tests/src/helpers/customSequencer.js | 4 ++++ packages/tests/src/tools/src/helpers.test.ts | 4 ++++ packages/tests/src/tools/src/rollupConfig.test.ts | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/packages/tests/src/helpers/customSequencer.js b/packages/tests/src/helpers/customSequencer.js index a4c6d0ee..9f958bd5 100644 --- a/packages/tests/src/helpers/customSequencer.js +++ b/packages/tests/src/helpers/customSequencer.js @@ -1,3 +1,7 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + const Sequencer = require('@jest/test-sequencer').default; /** @typedef {Parameters[0]} Tests */ diff --git a/packages/tests/src/tools/src/helpers.test.ts b/packages/tests/src/tools/src/helpers.test.ts index f337fc01..d62b4c7b 100644 --- a/packages/tests/src/tools/src/helpers.test.ts +++ b/packages/tests/src/tools/src/helpers.test.ts @@ -1,3 +1,7 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + import { getCamelCase, getPascalCase, diff --git a/packages/tests/src/tools/src/rollupConfig.test.ts b/packages/tests/src/tools/src/rollupConfig.test.ts index 3fd168f7..4b5eaa87 100644 --- a/packages/tests/src/tools/src/rollupConfig.test.ts +++ b/packages/tests/src/tools/src/rollupConfig.test.ts @@ -1,3 +1,7 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + import { datadogEsbuildPlugin } from '@datadog/esbuild-plugin'; import { datadogRollupPlugin } from '@datadog/rollup-plugin'; import { datadogVitePlugin } from '@datadog/vite-plugin'; From e8e76823dcc699b3e97fd98c9163b6c4292d97a2 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Wed, 9 Oct 2024 16:35:44 +0200 Subject: [PATCH 19/30] Update package.json --- packages/webpack-plugin/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/webpack-plugin/package.json b/packages/webpack-plugin/package.json index 3a2a0e0a..c16b2a1c 100644 --- a/packages/webpack-plugin/package.json +++ b/packages/webpack-plugin/package.json @@ -74,5 +74,5 @@ "webpack-sources": { "optional": true } - } + } } From 35aac43fd11e568228882bc20f42f0f11e286cbe Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Thu, 10 Oct 2024 11:57:23 +0200 Subject: [PATCH 20/30] Add plugin names into the global context --- packages/core/src/helpers.ts | 12 +++++++++++- packages/core/src/plugins/bundler-report/index.ts | 2 +- packages/core/src/plugins/index.ts | 1 + packages/core/src/types.ts | 1 + packages/factory/src/index.ts | 10 ++++++++-- packages/tests/src/helpers/mocks.ts | 1 + 6 files changed, 23 insertions(+), 4 deletions(-) diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts index 3b835290..e24739cb 100644 --- a/packages/core/src/helpers.ts +++ b/packages/core/src/helpers.ts @@ -6,7 +6,7 @@ import retry from 'async-retry'; import type { RequestInit } from 'undici-types'; import { INJECTED_FILE } from './plugins/injection/constants'; -import type { RequestOpts } from './types'; +import type { GlobalContext, RequestOpts } from './types'; // Format a duration 0h 0m 0s 0ms export const formatDuration = (duration: number) => { @@ -121,3 +121,13 @@ export const truncateString = ( // Is the file coming from the injection plugin? export const isInjection = (filename: string) => filename.includes(INJECTED_FILE); + +// Is the given plugin name is from our internal plugins? +export const isInternalPlugin = (pluginName: string, context: GlobalContext) => { + for (const internalPluginName of context.pluginNames) { + if (pluginName.includes(internalPluginName)) { + return true; + } + } + return false; +}; diff --git a/packages/core/src/plugins/bundler-report/index.ts b/packages/core/src/plugins/bundler-report/index.ts index 5b8d45fc..91b415cd 100644 --- a/packages/core/src/plugins/bundler-report/index.ts +++ b/packages/core/src/plugins/bundler-report/index.ts @@ -5,7 +5,7 @@ import type { GlobalContext, Options, PluginOptions } from '@dd/core/types'; import path from 'path'; -const PLUGIN_NAME = 'datadog-context-plugin'; +const PLUGIN_NAME = 'datadog-bundler-report-plugin'; const rollupPlugin: (context: GlobalContext) => PluginOptions['rollup'] = (context) => ({ options(options) { diff --git a/packages/core/src/plugins/index.ts b/packages/core/src/plugins/index.ts index 38c9ad51..f2429cf2 100644 --- a/packages/core/src/plugins/index.ts +++ b/packages/core/src/plugins/index.ts @@ -20,6 +20,7 @@ export const getInternalPlugins = ( const toInject: ToInjectItem[] = []; const globalContext: GlobalContext = { auth: options.auth, + pluginNames: [], bundler: { name: meta.framework, fullName: `${meta.framework}${variant}`, diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 98d865cc..8d1af5f2 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -76,6 +76,7 @@ export type GlobalContext = { build: BuildReport; cwd: string; git?: RepositoryData; + pluginNames: string[]; start: number; version: string; }; diff --git a/packages/factory/src/index.ts b/packages/factory/src/index.ts index e3d42fee..4bcf1e49 100644 --- a/packages/factory/src/index.ts +++ b/packages/factory/src/index.ts @@ -45,6 +45,8 @@ const validateOptions = (options: Options = {}): Options => { }; }; +const HOST_NAME = 'datadog-build-plugins'; + export const buildPluginFactory = ({ version, }: { @@ -58,8 +60,8 @@ export const buildPluginFactory = ({ const options = validateOptions(opts); // Set the host name for the esbuild plugin. - if ('esbuildHostName' in unpluginMetaContext) { - unpluginMetaContext.esbuildHostName = 'datadog-plugins'; + if (unpluginMetaContext.framework === 'esbuild') { + unpluginMetaContext.esbuildHostName = HOST_NAME; } // Get the global context and internal plugins. @@ -68,6 +70,8 @@ export const buildPluginFactory = ({ ...unpluginMetaContext, }); + globalContext.pluginNames.push(HOST_NAME); + // List of plugins to be returned. const plugins: (PluginOptions | UnpluginOptions)[] = [...internalPlugins]; @@ -87,6 +91,8 @@ export const buildPluginFactory = ({ } // #configs-injection-marker + globalContext.pluginNames.push(...plugins.map((plugin) => plugin.name)); + return plugins; }); }; diff --git a/packages/tests/src/helpers/mocks.ts b/packages/tests/src/helpers/mocks.ts index 95c20cbc..0a10c96d 100644 --- a/packages/tests/src/helpers/mocks.ts +++ b/packages/tests/src/helpers/mocks.ts @@ -47,6 +47,7 @@ export const getContextMock = (options: Partial = {}): GlobalCont }, cwd: '/cwd/path', inject: jest.fn(), + pluginNames: [], start: Date.now(), version: 'FAKE_VERSION', ...options, From 7ecdcf670684e35932124574de6cf8ab2b799071 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Thu, 10 Oct 2024 12:03:57 +0200 Subject: [PATCH 21/30] Patch other plugins to avoid initialOptions leakage --- packages/core/src/plugins/injection/index.ts | 21 +++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/core/src/plugins/injection/index.ts b/packages/core/src/plugins/injection/index.ts index 83e6d32f..752eff6d 100644 --- a/packages/core/src/plugins/injection/index.ts +++ b/packages/core/src/plugins/injection/index.ts @@ -2,7 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { isInjection } from '@dd/core/helpers'; +import { isInjection, isInternalPlugin } from '@dd/core/helpers'; import { getLogger } from '@dd/core/log'; import type { GlobalContext, Options, PluginOptions, ToInjectItem } from '@dd/core/types'; @@ -90,8 +90,27 @@ export const getInjectionPlugins = ( esbuild: { setup(build) { const { initialOptions } = build; + // Clone the existing inject array to keep it unmutated for other plugins. + const initialInject = initialOptions.inject ? [...initialOptions.inject] : []; + const plugins = initialOptions.plugins || []; + initialOptions.inject = initialOptions.inject || []; initialOptions.inject.push(INJECTED_FILE); + + // Patch all the plugins to remove our injected file from the list. + for (const plugin of plugins) { + const oldSetup = plugin.setup; + + // We don't want to patch our plugins. + if (isInternalPlugin(plugin.name, context)) { + continue; + } + + plugin.setup = async (esbuild) => { + esbuild.initialOptions.inject = initialInject; + await oldSetup(esbuild); + }; + } }, }, webpack: (compiler) => { From 1a11e482cd9ccf40cde0b57339ee13ed303b0bab Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Thu, 10 Oct 2024 12:09:48 +0200 Subject: [PATCH 22/30] Fix helpers --- packages/tools/src/helpers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/tools/src/helpers.ts b/packages/tools/src/helpers.ts index bb873535..33f1f3b4 100644 --- a/packages/tools/src/helpers.ts +++ b/packages/tools/src/helpers.ts @@ -144,6 +144,7 @@ export const getSupportedBundlers = (getPlugins: GetPlugins) => { errors: [], }, inject() {}, + pluginNames: [], }, ); From 3ab48c8a1c9fbcc5b5798b51481fd0943a06287b Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 11 Oct 2024 11:51:16 +0200 Subject: [PATCH 23/30] Fix esbuild's injection plugin --- packages/core/src/plugins/injection/index.ts | 90 ++++++++++++++------ 1 file changed, 62 insertions(+), 28 deletions(-) diff --git a/packages/core/src/plugins/injection/index.ts b/packages/core/src/plugins/injection/index.ts index 752eff6d..96d209b3 100644 --- a/packages/core/src/plugins/injection/index.ts +++ b/packages/core/src/plugins/injection/index.ts @@ -2,9 +2,10 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { isInjection, isInternalPlugin } from '@dd/core/helpers'; +import { isInjection } from '@dd/core/helpers'; import { getLogger } from '@dd/core/log'; import type { GlobalContext, Options, PluginOptions, ToInjectItem } from '@dd/core/types'; +import path from 'path'; import { INJECTED_FILE, @@ -23,18 +24,16 @@ export const getInjectionPlugins = ( const contentToInject: string[] = []; const getContentToInject = () => { - contentToInject.unshift( - // Needs at least one element otherwise ESBuild will throw 'Do not know how to load path'. - // Most likely because it tries to generate an empty file. - ` + // Needs a non empty string otherwise ESBuild will throw 'Do not know how to load path'. + // Most likely because it tries to generate an empty file. + const before = ` /********************************************/ -/* BEGIN INJECTION BY DATADOG BUILD PLUGINS */`, - ); - contentToInject.push(` +/* BEGIN INJECTION BY DATADOG BUILD PLUGINS */`; + const after = ` /* END INJECTION BY DATADOG BUILD PLUGINS */ -/********************************************/`); +/********************************************/`; - return contentToInject.join('\n\n'); + return `${before}\n${contentToInject.join('\n\n')}\n${after}`; }; // Rollup uses its own banner hook @@ -54,11 +53,11 @@ export const getInjectionPlugins = ( // 2. Prepare the content to inject, fetching distant/local files and anything necessary. // 3. Inject a virtual file into the bundling, this file will be home of all injected content. return [ - // Resolve the injected file. + // Resolve the injected file for all bundlers. { name: RESOLUTION_PLUGIN_NAME, enforce: 'pre', - resolveId(id) { + async resolveId(id) { if (isInjection(id)) { return { id, moduleSideEffects: true }; } @@ -74,7 +73,7 @@ export const getInjectionPlugins = ( } }, }, - // Prepare and fetch the content to inject. + // Prepare and fetch the content to inject for all bundlers. { name: PREPARATION_PLUGIN_NAME, enforce: 'pre', @@ -85,32 +84,67 @@ export const getInjectionPlugins = ( }, }, // Inject the virtual file that will be home of all injected content. + // Each bundler has its own way to inject a file. { name: PLUGIN_NAME, esbuild: { setup(build) { const { initialOptions } = build; - // Clone the existing inject array to keep it unmutated for other plugins. - const initialInject = initialOptions.inject ? [...initialOptions.inject] : []; - const plugins = initialOptions.plugins || []; - initialOptions.inject = initialOptions.inject || []; - initialOptions.inject.push(INJECTED_FILE); + build.onResolve({ filter: /.*/ }, async (args) => { + // Only mark the entry point for injection. + if (args.kind !== 'entry-point') { + return null; + } - // Patch all the plugins to remove our injected file from the list. - for (const plugin of plugins) { - const oldSetup = plugin.setup; + // Injected modules via the esbuild `inject` option do also have `kind == "entry-point"`. + if (initialOptions.inject?.includes(args.path)) { + return null; + } - // We don't want to patch our plugins. - if (isInternalPlugin(plugin.name, context)) { - continue; + return { + pluginName: PLUGIN_NAME, + path: path.isAbsolute(args.path) + ? args.path + : path.join(args.resolveDir, args.path), + pluginData: { + isInjectionResolver: true, + originalPath: args.path, + originalResolveDir: args.resolveDir, + }, + // Adding a suffix prevents esbuild from marking the entrypoint as resolved, + // avoiding a dependency loop with the proxy module. + // This ensures esbuild continues to traverse the module tree + // and re-resolves the entrypoint when imported from the proxy module. + suffix: '?datadogInjected=true', + }; + }); + + build.onLoad({ filter: /.*/ }, async (args) => { + // We only want to handle the marked entry point. + if (!args.pluginData?.isInjectionResolver) { + return null; } - plugin.setup = async (esbuild) => { - esbuild.initialOptions.inject = initialInject; - await oldSetup(esbuild); + const originalPath = args.pluginData.originalPath; + const originalResolveDir = args.pluginData.originalResolveDir; + + // Using JSON.stringify to keep escaped backslashes (windows). + // Using ['default'.toString()] to bypass esbuild's import-is-undefined warning. + const contents = ` +import ${JSON.stringify(INJECTED_FILE)}; +import * as OriginalModule from ${JSON.stringify(originalPath)}; +export default OriginalModule['default'.toString()]; +export * from ${JSON.stringify(originalPath)}; +`; + + return { + loader: 'js', + pluginName: PLUGIN_NAME, + contents, + resolveDir: originalResolveDir, }; - } + }); }, }, webpack: (compiler) => { From 053d5290961af00af33ea2b5ed66221c443011a2 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 11 Oct 2024 14:18:29 +0200 Subject: [PATCH 24/30] Run build before tests --- package.json | 2 +- packages/tests/src/tools/src/rollupConfig.test.ts | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index f34287a1..04595475 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "loop": "yarn workspaces foreach -Apti --include \"@datadog/*\" --exclude \"@datadog/build-plugins\"", "oss": "yarn cli oss -d packages -l mit", "publish:all": "yarn loop --no-private npm publish", - "test": "yarn workspace @dd/tests test", + "test": "yarn build:all && yarn workspace @dd/tests test", "test:noisy": "yarn workspace @dd/tests test:noisy", "typecheck:all": "yarn workspaces foreach -Apti run typecheck", "version:all": "yarn loop version --deferred ${0} && yarn version apply --all", diff --git a/packages/tests/src/tools/src/rollupConfig.test.ts b/packages/tests/src/tools/src/rollupConfig.test.ts index 4b5eaa87..dcdf5af6 100644 --- a/packages/tests/src/tools/src/rollupConfig.test.ts +++ b/packages/tests/src/tools/src/rollupConfig.test.ts @@ -14,7 +14,6 @@ import { } from '@dd/tests/helpers/mocks'; import { BUNDLERS } from '@dd/tests/helpers/runBundlers'; import { ROOT } from '@dd/tools/constants'; -import { execute } from '@dd/tools/helpers'; import { removeSync } from 'fs-extra'; import nock from 'nock'; import path from 'path'; @@ -44,10 +43,6 @@ describe('Bundling', () => { logLevel: 'error', }); beforeAll(async () => { - // First, bundle the plugins. - // FIXME: This is slow because of the dts() build. - await execute('yarn', ['build:all']); - // Make the mocks target the built packages. const getPackageDestination = (bundlerName: string) => { return path.resolve(ROOT, `packages/${bundlerName}-plugin/dist/src`); @@ -75,7 +70,7 @@ describe('Bundling', () => { // For metrics submissions. .post('/api/v1/series?api_key=123') .reply(200, {}); - }, 30000); + }); afterAll(async () => { nock.cleanAll(); From 6b3199a706942f0db1160996f0f061992bc8d0aa Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 11 Oct 2024 15:43:09 +0200 Subject: [PATCH 25/30] Fix esbuild build-report after injection update --- packages/core/src/helpers.ts | 7 +++- .../core/src/plugins/build-report/esbuild.ts | 41 ++++++++++++++++--- .../core/src/plugins/build-report/helpers.ts | 8 ++-- .../core/src/plugins/build-report/webpack.ts | 4 +- .../core/src/plugins/injection/constants.ts | 1 + packages/core/src/plugins/injection/index.ts | 11 ++--- 6 files changed, 54 insertions(+), 18 deletions(-) diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts index e24739cb..121b75f5 100644 --- a/packages/core/src/helpers.ts +++ b/packages/core/src/helpers.ts @@ -5,7 +5,7 @@ import retry from 'async-retry'; import type { RequestInit } from 'undici-types'; -import { INJECTED_FILE } from './plugins/injection/constants'; +import { INJECTED_FILE, INJECTION_SUFFIX } from './plugins/injection/constants'; import type { GlobalContext, RequestOpts } from './types'; // Format a duration 0h 0m 0s 0ms @@ -120,7 +120,10 @@ export const truncateString = ( }; // Is the file coming from the injection plugin? -export const isInjection = (filename: string) => filename.includes(INJECTED_FILE); +export const isInjectionFile = (filename: string) => filename.includes(INJECTED_FILE); +export const isInjectionProxy = (filename: string) => filename.endsWith(INJECTION_SUFFIX); +export const isFromInjection = (filename: string) => + isInjectionFile(filename) || isInjectionProxy(filename); // Is the given plugin name is from our internal plugins? export const isInternalPlugin = (pluginName: string, context: GlobalContext) => { diff --git a/packages/core/src/plugins/build-report/esbuild.ts b/packages/core/src/plugins/build-report/esbuild.ts index 82d2a2eb..4057b134 100644 --- a/packages/core/src/plugins/build-report/esbuild.ts +++ b/packages/core/src/plugins/build-report/esbuild.ts @@ -2,7 +2,12 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { getResolvedPath, isInjection } from '@dd/core/helpers'; +import { + getResolvedPath, + isFromInjection, + isInjectionFile, + isInjectionProxy, +} from '@dd/core/helpers'; import type { Logger } from '@dd/core/log'; import type { Entry, GlobalContext, Input, Output, PluginOptions } from '@dd/core/types'; import { glob } from 'glob'; @@ -93,9 +98,31 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt const metaInputsIndexed = reIndexMeta(result.metafile.inputs, cwd); const metaOutputsIndexed = reIndexMeta(result.metafile.outputs, cwd); + // From a proxy entry point, created by our injection plugin, get the real path. + const getRealPathFromInjectionProxy = (entryPoint: string): string => { + if (!isInjectionProxy(entryPoint)) { + return entryPoint; + } + + const metaInput = metaInputsIndexed[getAbsolutePath(cwd, entryPoint)]; + if (!metaInput) { + return entryPoint; + } + + // Get the first non-injection import. + const actualImport = metaInput.imports.find( + (imp) => !isFromInjection(imp.path), + ); + if (!actualImport) { + return entryPoint; + } + + return actualImport.path; + }; + // Loop through inputs. for (const [filename, input] of Object.entries(result.metafile.inputs)) { - if (isInjection(filename)) { + if (isFromInjection(filename)) { continue; } @@ -121,7 +148,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt // Get inputs of this output. const inputFiles: Input[] = []; for (const inputName of Object.keys(output.inputs)) { - if (isInjection(inputName)) { + if (isFromInjection(inputName)) { continue; } @@ -167,7 +194,11 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt continue; } - const inputFile = reportInputsIndexed[getAbsolutePath(cwd, output.entryPoint!)]; + // The entryPoint may have been altered by our injection plugin. + const inputFile = + reportInputsIndexed[ + getAbsolutePath(cwd, getRealPathFromInjectionProxy(output.entryPoint)) + ]; if (inputFile) { // In the case of "splitting: true", all the files are considered entries to esbuild. @@ -216,7 +247,7 @@ export const getEsbuildPlugin = (context: GlobalContext, log: Logger): PluginOpt // There are some exceptions we want to ignore. const FILE_EXCEPTIONS_RX = /(|https:|file:|data:|#)/g; const isFileSupported = (filePath: string) => { - if (isInjection(filePath) || filePath.match(FILE_EXCEPTIONS_RX)) { + if (isInjectionFile(filePath) || filePath.match(FILE_EXCEPTIONS_RX)) { return false; } return true; diff --git a/packages/core/src/plugins/build-report/helpers.ts b/packages/core/src/plugins/build-report/helpers.ts index 6827b5f6..a49eac6c 100644 --- a/packages/core/src/plugins/build-report/helpers.ts +++ b/packages/core/src/plugins/build-report/helpers.ts @@ -2,7 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { isInjection } from '@dd/core/helpers'; +import { isInjectionFile } from '@dd/core/helpers'; import { INJECTED_FILE } from '@dd/core/plugins/injection/constants'; import type { BuildReport, @@ -202,7 +202,7 @@ export const cleanReport = ( const cleanedPath = cleanPath(reportFilepath); if ( // Don't add injections. - isInjection(reportFilepath) || + isInjectionFile(reportFilepath) || // Don't add itself into it. cleanedPath === filepath || // Remove common specific files injected by bundlers. @@ -241,7 +241,7 @@ export const cleanPath = (filepath: string) => { // Will only prepend the cwd if not already there. export const getAbsolutePath = (cwd: string, filepath: string) => { - if (isInjection(filepath)) { + if (isInjectionFile(filepath)) { return INJECTED_FILE; } @@ -253,7 +253,7 @@ export const getAbsolutePath = (cwd: string, filepath: string) => { // Extract a name from a path based on the context (out dir and cwd). export const cleanName = (context: GlobalContext, filepath: string) => { - if (isInjection(filepath)) { + if (isInjectionFile(filepath)) { return INJECTED_FILE; } diff --git a/packages/core/src/plugins/build-report/webpack.ts b/packages/core/src/plugins/build-report/webpack.ts index 8d7dc673..8a9878b6 100644 --- a/packages/core/src/plugins/build-report/webpack.ts +++ b/packages/core/src/plugins/build-report/webpack.ts @@ -2,7 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { isInjection } from '@dd/core/helpers'; +import { isInjectionFile } from '@dd/core/helpers'; import type { Logger } from '@dd/core/log'; import type { Entry, GlobalContext, Input, Output, PluginOptions } from '@dd/core/types'; @@ -139,7 +139,7 @@ export const getWebpackPlugin = const isModuleSupported = (module: (typeof modules)[number]) => { if ( - isInjection(getModulePath(module)) || + isInjectionFile(getModulePath(module)) || // Do not report runtime modules as they are very specific to webpack. module.moduleType === 'runtime' || module.name?.startsWith('(webpack)') || diff --git a/packages/core/src/plugins/injection/constants.ts b/packages/core/src/plugins/injection/constants.ts index 6bce5aba..a9a4baaa 100644 --- a/packages/core/src/plugins/injection/constants.ts +++ b/packages/core/src/plugins/injection/constants.ts @@ -6,4 +6,5 @@ export const PREPARATION_PLUGIN_NAME = 'datadog-injection-preparation-plugin'; export const PLUGIN_NAME = 'datadog-injection-plugin'; export const RESOLUTION_PLUGIN_NAME = 'datadog-injection-resolution-plugin'; export const INJECTED_FILE = '__DATADOG_INJECTION_STUB'; +export const INJECTION_SUFFIX = '?datadogInjected=true'; export const DISTANT_FILE_RX = /^https?:\/\//; diff --git a/packages/core/src/plugins/injection/index.ts b/packages/core/src/plugins/injection/index.ts index 96d209b3..d37af7b6 100644 --- a/packages/core/src/plugins/injection/index.ts +++ b/packages/core/src/plugins/injection/index.ts @@ -2,13 +2,14 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. -import { isInjection } from '@dd/core/helpers'; +import { isInjectionFile } from '@dd/core/helpers'; import { getLogger } from '@dd/core/log'; import type { GlobalContext, Options, PluginOptions, ToInjectItem } from '@dd/core/types'; import path from 'path'; import { INJECTED_FILE, + INJECTION_SUFFIX, PLUGIN_NAME, PREPARATION_PLUGIN_NAME, RESOLUTION_PLUGIN_NAME, @@ -58,17 +59,17 @@ export const getInjectionPlugins = ( name: RESOLUTION_PLUGIN_NAME, enforce: 'pre', async resolveId(id) { - if (isInjection(id)) { + if (isInjectionFile(id)) { return { id, moduleSideEffects: true }; } }, loadInclude(id) { - if (isInjection(id)) { + if (isInjectionFile(id)) { return true; } }, load(id) { - if (isInjection(id)) { + if (isInjectionFile(id)) { return getContentToInject(); } }, @@ -116,7 +117,7 @@ export const getInjectionPlugins = ( // avoiding a dependency loop with the proxy module. // This ensures esbuild continues to traverse the module tree // and re-resolves the entrypoint when imported from the proxy module. - suffix: '?datadogInjected=true', + suffix: INJECTION_SUFFIX, }; }); From 1db7009f71fbd330d93be4b74557938ef3fe9c0d Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 11 Oct 2024 16:29:00 +0200 Subject: [PATCH 26/30] Add clean:all command --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 04595475..9a3e64ce 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ }, "scripts": { "build:all": "yarn loop run build", + "clean:all": "yarn loop run clean", "cli": "yarn workspace @dd/tools cli", "format": "yarn lint --fix", "lint": "eslint ./packages/**/*.{ts,js} --quiet", From bc31c3520c01f760e9bf1448db288fcb5f1b1ecf Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 11 Oct 2024 16:30:53 +0200 Subject: [PATCH 27/30] Remove custom sequencer --- packages/tests/jest.config.js | 1 - packages/tests/src/helpers/customSequencer.js | 40 ------------------- 2 files changed, 41 deletions(-) delete mode 100644 packages/tests/src/helpers/customSequencer.js diff --git a/packages/tests/jest.config.js b/packages/tests/jest.config.js index 327cddba..6bb8ac72 100644 --- a/packages/tests/jest.config.js +++ b/packages/tests/jest.config.js @@ -13,5 +13,4 @@ module.exports = { testMatch: ['**/*.test.*'], roots: ['./src/'], setupFilesAfterEnv: ['/src/setupTests.ts'], - testSequencer: '/src/helpers/customSequencer.js', }; diff --git a/packages/tests/src/helpers/customSequencer.js b/packages/tests/src/helpers/customSequencer.js deleted file mode 100644 index 9f958bd5..00000000 --- a/packages/tests/src/helpers/customSequencer.js +++ /dev/null @@ -1,40 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2019-Present Datadog, Inc. - -const Sequencer = require('@jest/test-sequencer').default; - -/** @typedef {Parameters[0]} Tests */ -/** @typedef {Tests[number]} Test */ - -/** - * @param {Test} test - * @returns {boolean} - */ -const isHeavyTest = (test) => { - return test.path.endsWith('src/tools/src/rollupConfig.test.ts'); -}; - -module.exports = class CustomSequencer extends Sequencer { - /** - * @param {Tests} tests - * @returns {Promise} - */ - async sort(tests) { - /** @type {Tests} */ - const sortedTests = []; - - // First, add the heavy tests. - for (const test of tests) { - if (isHeavyTest(test)) { - sortedTests.push(test); - } - } - - // Then add the rest of the tests, using the default sort. - const superSortedTests = await super.sort(tests.filter((test) => !isHeavyTest(test))); - sortedTests.push(...superSortedTests); - - return sortedTests; - } -}; From fb8520be0739fdd78348db178da39d73808bb636 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 11 Oct 2024 17:03:15 +0200 Subject: [PATCH 28/30] Trim ms from duration helper --- packages/core/src/helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts index 121b75f5..ed0663bd 100644 --- a/packages/core/src/helpers.ts +++ b/packages/core/src/helpers.ts @@ -19,7 +19,7 @@ export const formatDuration = (duration: number) => { const milliseconds = d.getUTCMilliseconds(); return `${days ? `${days}d ` : ''}${hours ? `${hours}h ` : ''}${minutes ? `${minutes}m ` : ''}${ seconds ? `${seconds}s ` : '' - }${milliseconds}ms`.trim(); + }${milliseconds ? `${milliseconds}ms` : ''}`.trim(); }; export const getResolvedPath = (filepath: string) => { From b2402c9aa619a4898b826eb7db7e420093674cae Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 11 Oct 2024 17:34:31 +0200 Subject: [PATCH 29/30] Add a note about sourcemaps --- packages/core/src/plugins/injection/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/src/plugins/injection/index.ts b/packages/core/src/plugins/injection/index.ts index d37af7b6..45dc7ad1 100644 --- a/packages/core/src/plugins/injection/index.ts +++ b/packages/core/src/plugins/injection/index.ts @@ -132,6 +132,7 @@ export const getInjectionPlugins = ( // Using JSON.stringify to keep escaped backslashes (windows). // Using ['default'.toString()] to bypass esbuild's import-is-undefined warning. + // NOTE: Keep an eye on this sourcemaps issue https://github.com/getsentry/sentry-javascript-bundler-plugins/issues/575 const contents = ` import ${JSON.stringify(INJECTED_FILE)}; import * as OriginalModule from ${JSON.stringify(originalPath)}; From 38157b788be53fa669a61d2f7e0f493f09bf0605 Mon Sep 17 00:00:00 2001 From: Yoann Moinet Date: Fri, 11 Oct 2024 17:38:49 +0200 Subject: [PATCH 30/30] Warn about old bundles in tests --- .../tests/src/tools/src/rollupConfig.test.ts | 45 ++++++++++++++++++- packages/tools/src/helpers.ts | 1 + 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/packages/tests/src/tools/src/rollupConfig.test.ts b/packages/tests/src/tools/src/rollupConfig.test.ts index dcdf5af6..8136b9d1 100644 --- a/packages/tests/src/tools/src/rollupConfig.test.ts +++ b/packages/tests/src/tools/src/rollupConfig.test.ts @@ -6,6 +6,7 @@ import { datadogEsbuildPlugin } from '@datadog/esbuild-plugin'; import { datadogRollupPlugin } from '@datadog/rollup-plugin'; import { datadogVitePlugin } from '@datadog/vite-plugin'; import { datadogWebpackPlugin } from '@datadog/webpack-plugin'; +import { formatDuration } from '@dd/core/helpers'; import { API_PATH, FAKE_URL, @@ -14,7 +15,9 @@ import { } from '@dd/tests/helpers/mocks'; import { BUNDLERS } from '@dd/tests/helpers/runBundlers'; import { ROOT } from '@dd/tools/constants'; +import { bgYellow } from '@dd/tools/helpers'; import { removeSync } from 'fs-extra'; +import fs from 'fs'; import nock from 'nock'; import path from 'path'; @@ -45,7 +48,47 @@ describe('Bundling', () => { beforeAll(async () => { // Make the mocks target the built packages. const getPackageDestination = (bundlerName: string) => { - return path.resolve(ROOT, `packages/${bundlerName}-plugin/dist/src`); + const packageDestination = path.resolve( + ROOT, + `packages/${bundlerName}-plugin/dist/src`, + ); + + // If we don't need this bundler, no need to check for its bundle. + if (BUNDLERS.find((bundler) => bundler.name.startsWith(bundlerName)) === undefined) { + return packageDestination; + } + + // Check if the bundle for this bundler is ready and not too old. + try { + const stats = fs.statSync(packageDestination); + const lastUpdateDuration = + Math.ceil((new Date().getTime() - stats.mtimeMs) / 1000) * 1000; + + // If last build was more than 10 minutes ago, warn the user. + if (lastUpdateDuration > 1000 * 60 * 10) { + console.log( + bgYellow(` +${bundlerName}-plugin was last built ${formatDuration(lastUpdateDuration)} ago. +You should run 'yarn build:all' or 'yarn watch:all'. +`), + ); + } + + // If last build was more than 1 day ago, throw an error. + if (lastUpdateDuration > 1000 * 60 * 60 * 24) { + throw new Error( + `The ${bundlerName}-plugin bundle is too old. Please run 'yarn build:all' first.`, + ); + } + } catch (e: any) { + if (e.code === 'ENOENT') { + throw new Error( + `Missing ${bundlerName}-plugin bundle.\nPlease run 'yarn build:all' first.`, + ); + } + } + + return packageDestination; }; datadogWebpackPluginMock.mockImplementation( diff --git a/packages/tools/src/helpers.ts b/packages/tools/src/helpers.ts index 33f1f3b4..9f1d9593 100644 --- a/packages/tools/src/helpers.ts +++ b/packages/tools/src/helpers.ts @@ -14,6 +14,7 @@ import type { SlugLessWorkspace } from './types'; export const green = chalk.bold.green; export const red = chalk.bold.red; +export const bgYellow = chalk.bold.bgYellow.black; export const blue = chalk.bold.cyan; export const bold = chalk.bold; export const dim = chalk.dim;