From bdd32922efe6768535a6968842e9160e5813bbd3 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Mon, 23 Dec 2024 11:09:37 +0100 Subject: [PATCH 01/21] [kbn-scout-reporting] add failed test reporter --- .../src/reporting/failed_test_reporter.ts | 122 ++++++++++++++++++ .../src/reporting/index.ts | 11 +- .../{playwright.ts => playwright_reporter.ts} | 11 +- .../reporting/scout_playwright_reporter.d.ts | 17 +++ .../src/reporting/test_failure.d.ts | 32 +++++ .../utils/get_plugin_matifest_data.ts | 86 ++++++++++++ .../src/reporting/utils/index.ts | 11 ++ .../src/reporting/utils/save_test_failures.ts | 29 +++++ .../src/playwright/config/create_config.ts | 11 +- 9 files changed, 317 insertions(+), 13 deletions(-) create mode 100644 packages/kbn-scout-reporting/src/reporting/failed_test_reporter.ts rename packages/kbn-scout-reporting/src/reporting/{playwright.ts => playwright_reporter.ts} (97%) create mode 100644 packages/kbn-scout-reporting/src/reporting/scout_playwright_reporter.d.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/test_failure.d.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/utils/get_plugin_matifest_data.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/utils/index.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/utils/save_test_failures.ts diff --git a/packages/kbn-scout-reporting/src/reporting/failed_test_reporter.ts b/packages/kbn-scout-reporting/src/reporting/failed_test_reporter.ts new file mode 100644 index 0000000000000..3a1393b8e31fe --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/failed_test_reporter.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { + FullConfig, + FullResult, + Reporter, + Suite, + TestCase, + TestResult, +} from '@playwright/test/reporter'; + +import path from 'node:path'; +import { ToolingLog } from '@kbn/tooling-log'; +import { SCOUT_REPORT_OUTPUT_ROOT } from '@kbn/scout-info'; +import stripANSI from 'strip-ansi'; +import { REPO_ROOT } from '@kbn/repo-info'; +import { + type CodeOwnersEntry, + getCodeOwnersEntries, + getOwningTeamsForPath, +} from '@kbn/code-owners'; +import { generateTestRunId, getTestIDForTitle } from '.'; +import { getPluginManifestData, saveTestFailuresReport } from './utils'; +import type { TestFailure } from './test_failure'; +import type { ScoutPlaywrightReporterOptions } from './scout_playwright_reporter'; + +/** + * Scout Failed Test reporter + */ +export class ScoutFailedTestReporter implements Reporter { + private readonly log: ToolingLog; + private readonly runId: string; + private readonly codeOwnersEntries: CodeOwnersEntry[]; + private target: string; + private testFailures: TestFailure[]; + private plugin: TestFailure['plugin']; + + constructor(private reporterOptions: ScoutPlaywrightReporterOptions = {}) { + this.log = new ToolingLog({ + level: 'info', + writeTo: process.stdout, + }); + + this.runId = this.reporterOptions.runId || generateTestRunId(); + this.codeOwnersEntries = getCodeOwnersEntries(); + this.testFailures = []; + this.target = 'undefined'; // when '--grep' is not provided in the command line + } + + private getFileOwners(filePath: string): string[] { + return getOwningTeamsForPath(filePath, this.codeOwnersEntries); + } + + public get reportRootPath(): string { + const outputPath = this.reporterOptions.outputPath || SCOUT_REPORT_OUTPUT_ROOT; + return path.join(outputPath, `scout-playwright-failed-test-${this.runId}`); + } + + printsToStdio(): boolean { + return false; // Avoid taking over console output + } + + onBegin(config: FullConfig, suite: Suite) { + // Get plugin metadata from kibana.jsonc + if (config.configFile) { + const metadata = getPluginManifestData(config.configFile); + this.plugin = { + id: metadata.plugin.id, + visibility: metadata.visibility, + group: metadata.group, + }; + } + // Get the target from the --grep argument, e.g. --grep=@svlSearch + const grepArg = process.argv.find((arg) => arg.includes('--grep')); + if (grepArg) { + this.target = grepArg.split('=')[1]; + } + } + + onTestEnd(test: TestCase, result: TestResult) { + if (result.status === 'passed') return; + + const testFailure: TestFailure = { + id: getTestIDForTitle(test.titlePath().join(' ')), + suite: test.parent.title, + title: test.title, + target: this.target, + location: test.location.file.replace(`${REPO_ROOT}/`, ''), + owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)), + plugin: this.plugin, + duration: result.duration, + error: { + message: result.error?.message ? stripANSI(result.error.message) : undefined, + stack_trace: result.error?.stack ? stripANSI(result.error.stack) : undefined, + }, + attachments: result.attachments.map((attachment) => ({ + name: attachment.name, + path: attachment.path, + contentType: attachment.contentType, + })), + }; + + this.testFailures.push(testFailure); + } + + onEnd(result: FullResult) { + if (this.testFailures.length === 0) return; + + const filename = `failed-tests-${this.plugin?.id}.json`; + saveTestFailuresReport(this.testFailures, this.reportRootPath, filename, this.log); + } +} + +// eslint-disable-next-line import/no-default-export +export default ScoutFailedTestReporter; diff --git a/packages/kbn-scout-reporting/src/reporting/index.ts b/packages/kbn-scout-reporting/src/reporting/index.ts index 58d4002320047..a10364a3c474a 100644 --- a/packages/kbn-scout-reporting/src/reporting/index.ts +++ b/packages/kbn-scout-reporting/src/reporting/index.ts @@ -9,7 +9,7 @@ import { createHash, randomBytes } from 'node:crypto'; import type { ReporterDescription } from 'playwright/test'; -import type { ScoutPlaywrightReporterOptions } from './playwright'; +import { ScoutPlaywrightReporterOptions } from './scout_playwright_reporter'; export * from './report'; @@ -26,5 +26,12 @@ export function getTestIDForTitle(title: string) { export const scoutPlaywrightReporter = ( options?: ScoutPlaywrightReporterOptions ): ReporterDescription => { - return ['@kbn/scout-reporting/src/reporting/playwright.ts', options]; + return ['@kbn/scout-reporting/src/reporting/playwright_reporter.ts', options]; +}; + +// Playwright failed test reporting +export const scoutFailedTestsReporter = ( + options?: ScoutPlaywrightReporterOptions +): ReporterDescription => { + return ['@kbn/scout-reporting/src/reporting/failed_test_reporter.ts', options]; }; diff --git a/packages/kbn-scout-reporting/src/reporting/playwright.ts b/packages/kbn-scout-reporting/src/reporting/playwright_reporter.ts similarity index 97% rename from packages/kbn-scout-reporting/src/reporting/playwright.ts rename to packages/kbn-scout-reporting/src/reporting/playwright_reporter.ts index 98ad6655626ae..b290f838d6d97 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright_reporter.ts @@ -30,14 +30,7 @@ import { } from '@kbn/code-owners'; import { generateTestRunId, getTestIDForTitle, ScoutReport, ScoutReportEventAction } from '.'; import { environmentMetadata } from '../datasources'; - -/** - * Configuration options for the Scout Playwright reporter - */ -export interface ScoutPlaywrightReporterOptions { - name?: string; - outputPath?: string; -} +import type { ScoutPlaywrightReporterOptions } from './scout_playwright_reporter'; /** * Scout Playwright reporter @@ -56,7 +49,7 @@ export class ScoutPlaywrightReporter implements Reporter { }); this.name = this.reporterOptions.name || 'unknown'; - this.runId = generateTestRunId(); + this.runId = this.reporterOptions.runId || generateTestRunId(); this.log.info(`Scout test run ID: ${this.runId}`); this.report = new ScoutReport(this.log); diff --git a/packages/kbn-scout-reporting/src/reporting/scout_playwright_reporter.d.ts b/packages/kbn-scout-reporting/src/reporting/scout_playwright_reporter.d.ts new file mode 100644 index 0000000000000..277f537dc277c --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/scout_playwright_reporter.d.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +/** + * Configuration options for the Scout Playwright reporter + */ +export declare interface ScoutPlaywrightReporterOptions { + name?: string; + runId?: string; + outputPath?: string; +} diff --git a/packages/kbn-scout-reporting/src/reporting/test_failure.d.ts b/packages/kbn-scout-reporting/src/reporting/test_failure.d.ts new file mode 100644 index 0000000000000..ded7f7cbe0563 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/test_failure.d.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export declare interface TestFailure { + id: string; + suite: string; + title: string; + target: string; + location: string; + owner: string[]; + plugin?: { + id: string; + visibility: string; + group: string; + }; + duration: number; + error: { + message?: string; + stack_trace?: string; + }; + attachments: Array<{ + name: string; + path?: string; + contentType: string; + }>; +} diff --git a/packages/kbn-scout-reporting/src/reporting/utils/get_plugin_matifest_data.ts b/packages/kbn-scout-reporting/src/reporting/utils/get_plugin_matifest_data.ts new file mode 100644 index 0000000000000..b7a70c308411d --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/utils/get_plugin_matifest_data.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import path from 'path'; +import fs from 'fs'; +import { Jsonc } from '@kbn/repo-packages'; + +interface PluginManifest { + id: string; + group: string; + owner: string[]; + visibility: string; + plugin: { id: string }; +} + +/** + * Resolves the path to the `kibana.jsonc` manifest based on the Playwright configuration file path. + * @param configPath - Absolute path to the Playwright configuration file. + * @returns Absolute path to the `kibana.jsonc` file. + * @throws Error if `ui_tests` is not found in the path. + */ +export const getManifestPath = (configPath: string): string => { + const pathSegments = configPath.split(path.sep); + const testDirIndex = pathSegments.indexOf('ui_tests'); + + if (testDirIndex === -1) { + throw new Error( + `Invalid path: "ui_tests" directory not found in ${configPath}. + Ensure playwright configuration file is in the plugin directory: '/plugins//ui_tests/'` + ); + } + + const manifestSegments = pathSegments.slice(0, testDirIndex).concat('kibana.jsonc'); + return path.resolve('/', ...manifestSegments); // Ensure absolute path +}; + +/** + * Reads and parses the `kibana.jsonc` manifest file. + * @param filePath - Absolute path to the `kibana.jsonc` file. + * @returns Parsed `PluginManifest` object. + * @throws Error if the file does not exist, cannot be read, or is invalid. + */ +export const readPluginManifest = (filePath: string): PluginManifest => { + if (!fs.existsSync(filePath)) { + throw new Error(`Manifest file not found: ${filePath}`); + } + + let fileContent: string; + try { + fileContent = fs.readFileSync(filePath, 'utf8'); + } catch (error) { + throw new Error(`Failed to read manifest file at ${filePath}: ${error.message}`); + } + + let manifest: Partial; + try { + manifest = Jsonc.parse(fileContent) as Partial; + } catch (error) { + throw new Error(`Invalid JSON format in manifest file at ${filePath}: ${error.message}`); + } + + const { id, group, visibility, owner, plugin } = manifest; + if (!id || !group || !visibility || !plugin?.id) { + throw new Error( + `Invalid manifest structure at ${filePath}. Expected required fields: id, group, visibility, plugin.id` + ); + } + + return { id, group, visibility, owner: owner || [], plugin }; +}; + +/** + * Resolves the plugin manifest file path and reads its content. + * @param configPath - Absolute path to the Playwright configuration file in the plugin directory. + * @returns Parsed `PluginManifest` object. + */ +export const getPluginManifestData = (configPath: string): PluginManifest => { + const manifestPath = getManifestPath(configPath); + return readPluginManifest(manifestPath); +}; diff --git a/packages/kbn-scout-reporting/src/reporting/utils/index.ts b/packages/kbn-scout-reporting/src/reporting/utils/index.ts new file mode 100644 index 0000000000000..f1c4dbb76abc0 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/utils/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export { getPluginManifestData } from './get_plugin_matifest_data'; +export { saveTestFailuresReport } from './save_test_failures'; diff --git a/packages/kbn-scout-reporting/src/reporting/utils/save_test_failures.ts b/packages/kbn-scout-reporting/src/reporting/utils/save_test_failures.ts new file mode 100644 index 0000000000000..d3ec94e1ce9e5 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/utils/save_test_failures.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import fs from 'fs'; +import path from 'path'; +import { ToolingLog } from '@kbn/tooling-log'; +import { TestFailure } from '../test_failure'; + +export const saveTestFailuresReport = ( + testFailures: TestFailure[], + reportRootPath: string, + filename: string, + log: ToolingLog +): void => { + try { + const reportPath = path.join(reportRootPath, filename); + fs.mkdirSync(reportRootPath, { recursive: true }); + fs.writeFileSync(reportPath, JSON.stringify({ failures: testFailures }, null, 2), 'utf-8'); + log.info(`Saving Scout failed test report to ${reportPath}`); + } catch (error) { + log.error(`Failed to save report at ${reportRootPath}: ${error.message}`); + } +}; diff --git a/packages/kbn-scout/src/playwright/config/create_config.ts b/packages/kbn-scout/src/playwright/config/create_config.ts index cb1e371cb43e7..75b06bf3dfae6 100644 --- a/packages/kbn-scout/src/playwright/config/create_config.ts +++ b/packages/kbn-scout/src/playwright/config/create_config.ts @@ -8,11 +8,17 @@ */ import { defineConfig, PlaywrightTestConfig, devices } from '@playwright/test'; -import { scoutPlaywrightReporter } from '@kbn/scout-reporting'; +import { + scoutFailedTestsReporter, + scoutPlaywrightReporter, + generateTestRunId, +} from '@kbn/scout-reporting'; import { SCOUT_SERVERS_ROOT } from '@kbn/scout-info'; import { ScoutPlaywrightOptions, ScoutTestOptions, VALID_CONFIG_MARKER } from '../types'; export function createPlaywrightConfig(options: ScoutPlaywrightOptions): PlaywrightTestConfig { + const runId = generateTestRunId(); + return defineConfig({ testDir: options.testDir, /* Run tests in files in parallel */ @@ -27,7 +33,8 @@ export function createPlaywrightConfig(options: ScoutPlaywrightOptions): Playwri reporter: [ ['html', { outputFolder: './output/reports', open: 'never' }], // HTML report configuration ['json', { outputFile: './output/reports/test-results.json' }], // JSON report - scoutPlaywrightReporter({ name: 'scout-playwright' }), // Scout report + scoutPlaywrightReporter({ name: 'scout-playwright', runId }), // Scout report + scoutFailedTestsReporter({ name: 'scout-playwright-failed-tests', runId }), // Scout failed test report ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { From b71c5bdf61898262f3b107a6030c238e2ecc3ed0 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Mon, 23 Dec 2024 18:41:25 +0100 Subject: [PATCH 02/21] [kbn-scout-reporting] add failed report html generator --- .../src/reporting/failed_test_reporter.ts | 23 +- .../src/reporting/test_failure.d.ts | 1 + .../utils/build_test_failure_html.ts | 220 ++++++++++++++++++ .../src/reporting/utils/index.ts | 3 +- .../src/reporting/utils/save_test_failures.ts | 16 ++ .../src/reporting/utils/strip_path.ts | 33 +++ 6 files changed, 292 insertions(+), 4 deletions(-) create mode 100644 packages/kbn-scout-reporting/src/reporting/utils/build_test_failure_html.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/utils/strip_path.ts diff --git a/packages/kbn-scout-reporting/src/reporting/failed_test_reporter.ts b/packages/kbn-scout-reporting/src/reporting/failed_test_reporter.ts index 3a1393b8e31fe..1225f4f4c6eb8 100644 --- a/packages/kbn-scout-reporting/src/reporting/failed_test_reporter.ts +++ b/packages/kbn-scout-reporting/src/reporting/failed_test_reporter.ts @@ -27,9 +27,16 @@ import { getOwningTeamsForPath, } from '@kbn/code-owners'; import { generateTestRunId, getTestIDForTitle } from '.'; -import { getPluginManifestData, saveTestFailuresReport } from './utils'; +import { + getPluginManifestData, + saveTestFailuresReport, + saveTestFailureHtml, + stripRunCommand, + stripfilePath, +} from './utils'; import type { TestFailure } from './test_failure'; import type { ScoutPlaywrightReporterOptions } from './scout_playwright_reporter'; +import { buildFailureHtml } from './utils/build_test_failure_html'; /** * Scout Failed Test reporter @@ -41,6 +48,7 @@ export class ScoutFailedTestReporter implements Reporter { private target: string; private testFailures: TestFailure[]; private plugin: TestFailure['plugin']; + private command: string; constructor(private reporterOptions: ScoutPlaywrightReporterOptions = {}) { this.log = new ToolingLog({ @@ -52,6 +60,7 @@ export class ScoutFailedTestReporter implements Reporter { this.codeOwnersEntries = getCodeOwnersEntries(); this.testFailures = []; this.target = 'undefined'; // when '--grep' is not provided in the command line + this.command = stripRunCommand(process.argv); } private getFileOwners(filePath: string): string[] { @@ -92,13 +101,14 @@ export class ScoutFailedTestReporter implements Reporter { suite: test.parent.title, title: test.title, target: this.target, + command: this.command, location: test.location.file.replace(`${REPO_ROOT}/`, ''), owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)), plugin: this.plugin, duration: result.duration, error: { - message: result.error?.message ? stripANSI(result.error.message) : undefined, - stack_trace: result.error?.stack ? stripANSI(result.error.stack) : undefined, + message: result.error?.message ? stripfilePath(stripANSI(result.error.message)) : undefined, + stack_trace: result.error?.stack ? stripfilePath(stripANSI(result.error.stack)) : undefined, }, attachments: result.attachments.map((attachment) => ({ name: attachment.name, @@ -115,6 +125,13 @@ export class ScoutFailedTestReporter implements Reporter { const filename = `failed-tests-${this.plugin?.id}.json`; saveTestFailuresReport(this.testFailures, this.reportRootPath, filename, this.log); + + // Generate HTML report for each failed test with embedded screenshots + for (const testFailure of this.testFailures) { + const failureHtml = buildFailureHtml(testFailure); + const htmlFilename = `${testFailure.id}.html`; + saveTestFailureHtml(this.reportRootPath, htmlFilename, failureHtml, this.log); + } } } diff --git a/packages/kbn-scout-reporting/src/reporting/test_failure.d.ts b/packages/kbn-scout-reporting/src/reporting/test_failure.d.ts index ded7f7cbe0563..942bcbb9a92b0 100644 --- a/packages/kbn-scout-reporting/src/reporting/test_failure.d.ts +++ b/packages/kbn-scout-reporting/src/reporting/test_failure.d.ts @@ -12,6 +12,7 @@ export declare interface TestFailure { suite: string; title: string; target: string; + command: string; location: string; owner: string[]; plugin?: { diff --git a/packages/kbn-scout-reporting/src/reporting/utils/build_test_failure_html.ts b/packages/kbn-scout-reporting/src/reporting/utils/build_test_failure_html.ts new file mode 100644 index 0000000000000..34ee25aa5509a --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/utils/build_test_failure_html.ts @@ -0,0 +1,220 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import fs from 'fs'; +import { TestFailure } from '../test_failure'; + +export const buildFailureHtml = (testFailure: TestFailure): string => { + const { suite, title, target, command, location, owner, plugin, duration, error, attachments } = + testFailure; + + const testDuration = duration < 1000 ? `${duration}ms` : `${(duration / 1000).toFixed(2)}s`; + + const screenshots = attachments + .filter((a) => a.contentType.startsWith('image/')) + .map((s) => { + const base64 = fs.readFileSync(s.path!).toString('base64'); + return ` +
+ ${s.name} +
+ `; + }); + + return ` + + + + + + + + Scout Test Failure Report + + +
+
+

Scout Test Failure Report

+
Location: ${location}
+
+ +
+
Test Details
+ + + + + + + + + + + + + + + + + +
Suite Title${suite}
Test Title${title}
Execution Details + Target: ${target}, + Duration: ${testDuration} +
Plugin + ID: ${plugin?.id}, + Visibility: ${plugin?.visibility}, + Group: ${plugin?.group} +
+
+ + + +
+
Command Line
+ 
${command}
+
+ +
+
Owners
+ 
${owner.join(', ')}
+
+ +
+
Error Details
+ 
${error?.stack_trace || 'No stack trace available'}
+
+ +
+
Attachments
+ ${screenshots.join('/n')} +
+
+
+ + + + `; +}; diff --git a/packages/kbn-scout-reporting/src/reporting/utils/index.ts b/packages/kbn-scout-reporting/src/reporting/utils/index.ts index f1c4dbb76abc0..86cdfdf4bf598 100644 --- a/packages/kbn-scout-reporting/src/reporting/utils/index.ts +++ b/packages/kbn-scout-reporting/src/reporting/utils/index.ts @@ -8,4 +8,5 @@ */ export { getPluginManifestData } from './get_plugin_matifest_data'; -export { saveTestFailuresReport } from './save_test_failures'; +export { saveTestFailuresReport, saveTestFailureHtml } from './save_test_failures'; +export { stripfilePath, stripRunCommand } from './strip_path'; diff --git a/packages/kbn-scout-reporting/src/reporting/utils/save_test_failures.ts b/packages/kbn-scout-reporting/src/reporting/utils/save_test_failures.ts index d3ec94e1ce9e5..99f9d0026585c 100644 --- a/packages/kbn-scout-reporting/src/reporting/utils/save_test_failures.ts +++ b/packages/kbn-scout-reporting/src/reporting/utils/save_test_failures.ts @@ -27,3 +27,19 @@ export const saveTestFailuresReport = ( log.error(`Failed to save report at ${reportRootPath}: ${error.message}`); } }; + +export const saveTestFailureHtml = ( + reportRootPath: string, + filename: string, + testFailureHtml: string, + log: ToolingLog +): void => { + try { + const reportPath = path.join(reportRootPath, filename); + fs.mkdirSync(reportRootPath, { recursive: true }); + fs.writeFileSync(reportPath, testFailureHtml, 'utf-8'); + log.info(`Saving Scout test failure html to ${reportPath}`); + } catch (error) { + log.error(`Failed to save report at ${reportRootPath}: ${error.message}`); + } +}; diff --git a/packages/kbn-scout-reporting/src/reporting/utils/strip_path.ts b/packages/kbn-scout-reporting/src/reporting/utils/strip_path.ts new file mode 100644 index 0000000000000..5c5906e2b4dca --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/utils/strip_path.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { REPO_ROOT } from '@kbn/repo-info'; + +export const stripfilePath = (filePath: string): string => filePath.replaceAll(`${REPO_ROOT}/`, ''); + +export const stripRunCommand = (commandArgs: string[]): string => { + // "/Users/user/.nvm/versions/node/v20.15.1/bin/node /Users/dmle/github/kibana/node_modules/.bin/playwright test --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts --grep=svlSearch", + // "npx playwright test --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts --grep=svlSearch", + if (!Array.isArray(commandArgs) || commandArgs.length < 3) { + throw new Error(`Invalid command arguments: must include at least 'npx playwright test'`); + } + + const isNodeCommand = commandArgs[0].endsWith('node'); + const isNpxCommand = commandArgs[0] === 'npx' && commandArgs[1] === 'playwright'; + + if (!isNodeCommand && !isNpxCommand) { + throw new Error( + 'Invalid command structure: Expected "node test" or "npx playwright test".' + ); + } + + const restArgs = commandArgs.slice(2); + // Rebuild the command with only valid arguments + return `npx playwright ${restArgs.join(' ')}`; +}; From b02ecb50e27d0b6a1a9055ee711341a9ce804932 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Mon, 6 Jan 2025 10:40:16 +0100 Subject: [PATCH 03/21] change structure --- .../kbn-scout-reporting/src/reporting/index.ts | 7 +++++-- .../common/get_plugin_manifest_data.ts} | 2 +- .../{utils => playwright/common}/index.ts | 5 ++--- .../{utils => playwright/common}/strip_path.ts | 0 .../events}/playwright_reporter.ts | 6 +++--- .../failed_test}/build_test_failure_html.ts | 2 +- .../failed_test}/failed_test_reporter.ts | 18 +++++++----------- .../reporting/playwright/failed_test/index.ts | 8 ++++++++ .../failed_test}/save_test_failures.ts | 2 +- 9 files changed, 28 insertions(+), 22 deletions(-) rename packages/kbn-scout-reporting/src/reporting/{utils/get_plugin_matifest_data.ts => playwright/common/get_plugin_manifest_data.ts} (98%) rename packages/kbn-scout-reporting/src/reporting/{utils => playwright/common}/index.ts (68%) rename packages/kbn-scout-reporting/src/reporting/{utils => playwright/common}/strip_path.ts (100%) rename packages/kbn-scout-reporting/src/reporting/{ => playwright/events}/playwright_reporter.ts (97%) rename packages/kbn-scout-reporting/src/reporting/{utils => playwright/failed_test}/build_test_failure_html.ts (99%) rename packages/kbn-scout-reporting/src/reporting/{ => playwright/failed_test}/failed_test_reporter.ts (89%) create mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts rename packages/kbn-scout-reporting/src/reporting/{utils => playwright/failed_test}/save_test_failures.ts (96%) diff --git a/packages/kbn-scout-reporting/src/reporting/index.ts b/packages/kbn-scout-reporting/src/reporting/index.ts index a10364a3c474a..bcdacdee129de 100644 --- a/packages/kbn-scout-reporting/src/reporting/index.ts +++ b/packages/kbn-scout-reporting/src/reporting/index.ts @@ -26,12 +26,15 @@ export function getTestIDForTitle(title: string) { export const scoutPlaywrightReporter = ( options?: ScoutPlaywrightReporterOptions ): ReporterDescription => { - return ['@kbn/scout-reporting/src/reporting/playwright_reporter.ts', options]; + return ['@kbn/scout-reporting/src/reporting/playwright/events/playwright_reporter.ts', options]; }; // Playwright failed test reporting export const scoutFailedTestsReporter = ( options?: ScoutPlaywrightReporterOptions ): ReporterDescription => { - return ['@kbn/scout-reporting/src/reporting/failed_test_reporter.ts', options]; + return [ + '@kbn/scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts', + options, + ]; }; diff --git a/packages/kbn-scout-reporting/src/reporting/utils/get_plugin_matifest_data.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/get_plugin_manifest_data.ts similarity index 98% rename from packages/kbn-scout-reporting/src/reporting/utils/get_plugin_matifest_data.ts rename to packages/kbn-scout-reporting/src/reporting/playwright/common/get_plugin_manifest_data.ts index b7a70c308411d..3066aed832694 100644 --- a/packages/kbn-scout-reporting/src/reporting/utils/get_plugin_matifest_data.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/get_plugin_manifest_data.ts @@ -11,7 +11,7 @@ import path from 'path'; import fs from 'fs'; import { Jsonc } from '@kbn/repo-packages'; -interface PluginManifest { +export interface PluginManifest { id: string; group: string; owner: string[]; diff --git a/packages/kbn-scout-reporting/src/reporting/utils/index.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/index.ts similarity index 68% rename from packages/kbn-scout-reporting/src/reporting/utils/index.ts rename to packages/kbn-scout-reporting/src/reporting/playwright/common/index.ts index 86cdfdf4bf598..61e5203da1454 100644 --- a/packages/kbn-scout-reporting/src/reporting/utils/index.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/index.ts @@ -7,6 +7,5 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export { getPluginManifestData } from './get_plugin_matifest_data'; -export { saveTestFailuresReport, saveTestFailureHtml } from './save_test_failures'; -export { stripfilePath, stripRunCommand } from './strip_path'; +export { getPluginManifestData, type PluginManifest } from './get_plugin_manifest_data'; +export { stripRunCommand, stripfilePath } from './strip_path'; diff --git a/packages/kbn-scout-reporting/src/reporting/utils/strip_path.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/strip_path.ts similarity index 100% rename from packages/kbn-scout-reporting/src/reporting/utils/strip_path.ts rename to packages/kbn-scout-reporting/src/reporting/playwright/common/strip_path.ts diff --git a/packages/kbn-scout-reporting/src/reporting/playwright_reporter.ts b/packages/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts similarity index 97% rename from packages/kbn-scout-reporting/src/reporting/playwright_reporter.ts rename to packages/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts index b290f838d6d97..c29a61cda4c7a 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright_reporter.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts @@ -28,9 +28,9 @@ import { getCodeOwnersEntries, getOwningTeamsForPath, } from '@kbn/code-owners'; -import { generateTestRunId, getTestIDForTitle, ScoutReport, ScoutReportEventAction } from '.'; -import { environmentMetadata } from '../datasources'; -import type { ScoutPlaywrightReporterOptions } from './scout_playwright_reporter'; +import { generateTestRunId, getTestIDForTitle, ScoutReport, ScoutReportEventAction } from '../..'; +import { environmentMetadata } from '../../../datasources'; +import type { ScoutPlaywrightReporterOptions } from '../../scout_playwright_reporter'; /** * Scout Playwright reporter diff --git a/packages/kbn-scout-reporting/src/reporting/utils/build_test_failure_html.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/build_test_failure_html.ts similarity index 99% rename from packages/kbn-scout-reporting/src/reporting/utils/build_test_failure_html.ts rename to packages/kbn-scout-reporting/src/reporting/playwright/failed_test/build_test_failure_html.ts index 34ee25aa5509a..a682a6d900bea 100644 --- a/packages/kbn-scout-reporting/src/reporting/utils/build_test_failure_html.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/build_test_failure_html.ts @@ -8,7 +8,7 @@ */ import fs from 'fs'; -import { TestFailure } from '../test_failure'; +import { TestFailure } from '../../test_failure'; export const buildFailureHtml = (testFailure: TestFailure): string => { const { suite, title, target, command, location, owner, plugin, duration, error, attachments } = diff --git a/packages/kbn-scout-reporting/src/reporting/failed_test_reporter.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts similarity index 89% rename from packages/kbn-scout-reporting/src/reporting/failed_test_reporter.ts rename to packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts index 1225f4f4c6eb8..a7c9350d08665 100644 --- a/packages/kbn-scout-reporting/src/reporting/failed_test_reporter.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts @@ -26,17 +26,13 @@ import { getCodeOwnersEntries, getOwningTeamsForPath, } from '@kbn/code-owners'; -import { generateTestRunId, getTestIDForTitle } from '.'; -import { - getPluginManifestData, - saveTestFailuresReport, - saveTestFailureHtml, - stripRunCommand, - stripfilePath, -} from './utils'; -import type { TestFailure } from './test_failure'; -import type { ScoutPlaywrightReporterOptions } from './scout_playwright_reporter'; -import { buildFailureHtml } from './utils/build_test_failure_html'; +import { generateTestRunId, getTestIDForTitle } from '../..'; +import type { TestFailure } from '../../test_failure'; +import type { ScoutPlaywrightReporterOptions } from '../../scout_playwright_reporter'; +import { buildFailureHtml } from './build_test_failure_html'; +import { stripRunCommand, stripfilePath } from '../common/strip_path'; +import { getPluginManifestData } from '../common/get_plugin_manifest_data'; +import { saveTestFailureHtml, saveTestFailuresReport } from './save_test_failures'; /** * Scout Failed Test reporter diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts new file mode 100644 index 0000000000000..ad21076a9ca93 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ diff --git a/packages/kbn-scout-reporting/src/reporting/utils/save_test_failures.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts similarity index 96% rename from packages/kbn-scout-reporting/src/reporting/utils/save_test_failures.ts rename to packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts index 99f9d0026585c..bf11d00e9a265 100644 --- a/packages/kbn-scout-reporting/src/reporting/utils/save_test_failures.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts @@ -10,7 +10,7 @@ import fs from 'fs'; import path from 'path'; import { ToolingLog } from '@kbn/tooling-log'; -import { TestFailure } from '../test_failure'; +import { TestFailure } from '../../test_failure'; export const saveTestFailuresReport = ( testFailures: TestFailure[], From 387c715dae695e1a64f7cfb24ca1427772a04ecd Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Tue, 7 Jan 2025 13:17:01 +0100 Subject: [PATCH 04/21] update reporter --- .../playwright/common/buildkite_meta.ts | 31 ++++ .../failed_test/failed_test_reporter.ts | 82 +++++------ .../playwright/failed_test/failure_report.ts | 134 ++++++++++++++++++ .../reporting/playwright/failed_test/index.ts | 8 -- .../failed_test/save_test_failures.ts | 25 +--- 5 files changed, 209 insertions(+), 71 deletions(-) create mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/common/buildkite_meta.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failure_report.ts delete mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/common/buildkite_meta.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/buildkite_meta.ts new file mode 100644 index 0000000000000..7bc6ebb39dba3 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/buildkite_meta.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export function getBuildkiteMetadata() { + // Buildkite steps that use `parallelism` need a numerical suffix added to identify them + // We should also increment the number by one, since it's 0-based + const jobNumberSuffix = process.env.BUILDKITE_PARALLEL_JOB + ? ` #${parseInt(process.env.BUILDKITE_PARALLEL_JOB, 10) + 1}` + : ''; + + const buildUrl = process.env.BUILDKITE_BUILD_URL; + const jobUrl = process.env.BUILDKITE_JOB_ID + ? `${buildUrl}#${process.env.BUILDKITE_JOB_ID}` + : undefined; + + return { + buildId: process.env.BUJILDKITE_BUILD_ID, + jobId: process.env.BUILDKITE_JOB_ID, + url: buildUrl, + jobUrl, + jobName: process.env.BUILDKITE_LABEL + ? `${process.env.BUILDKITE_LABEL}${jobNumberSuffix}` + : undefined, + }; +} diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts index a7c9350d08665..e415e495b3e4b 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts @@ -29,10 +29,9 @@ import { import { generateTestRunId, getTestIDForTitle } from '../..'; import type { TestFailure } from '../../test_failure'; import type { ScoutPlaywrightReporterOptions } from '../../scout_playwright_reporter'; -import { buildFailureHtml } from './build_test_failure_html'; import { stripRunCommand, stripfilePath } from '../common/strip_path'; import { getPluginManifestData } from '../common/get_plugin_manifest_data'; -import { saveTestFailureHtml, saveTestFailuresReport } from './save_test_failures'; +import { ScoutFailureReport } from './failure_report'; /** * Scout Failed Test reporter @@ -41,8 +40,8 @@ export class ScoutFailedTestReporter implements Reporter { private readonly log: ToolingLog; private readonly runId: string; private readonly codeOwnersEntries: CodeOwnersEntry[]; + private readonly report: ScoutFailureReport; private target: string; - private testFailures: TestFailure[]; private plugin: TestFailure['plugin']; private command: string; @@ -52,9 +51,10 @@ export class ScoutFailedTestReporter implements Reporter { writeTo: process.stdout, }); - this.runId = this.reporterOptions.runId || generateTestRunId(); + this.report = new ScoutFailureReport(this.log); this.codeOwnersEntries = getCodeOwnersEntries(); - this.testFailures = []; + + this.runId = this.reporterOptions.runId || generateTestRunId(); this.target = 'undefined'; // when '--grep' is not provided in the command line this.command = stripRunCommand(process.argv); } @@ -65,7 +65,12 @@ export class ScoutFailedTestReporter implements Reporter { public get reportRootPath(): string { const outputPath = this.reporterOptions.outputPath || SCOUT_REPORT_OUTPUT_ROOT; - return path.join(outputPath, `scout-playwright-failed-test-${this.runId}`); + return path.join( + outputPath, + `scout-playwright-test-failures-${this.runId}`, + 'target', + 'test-failures' + ); } printsToStdio(): boolean { @@ -90,43 +95,40 @@ export class ScoutFailedTestReporter implements Reporter { } onTestEnd(test: TestCase, result: TestResult) { - if (result.status === 'passed') return; - - const testFailure: TestFailure = { - id: getTestIDForTitle(test.titlePath().join(' ')), - suite: test.parent.title, - title: test.title, - target: this.target, - command: this.command, - location: test.location.file.replace(`${REPO_ROOT}/`, ''), - owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)), - plugin: this.plugin, - duration: result.duration, - error: { - message: result.error?.message ? stripfilePath(stripANSI(result.error.message)) : undefined, - stack_trace: result.error?.stack ? stripfilePath(stripANSI(result.error.stack)) : undefined, - }, - attachments: result.attachments.map((attachment) => ({ - name: attachment.name, - path: attachment.path, - contentType: attachment.contentType, - })), - }; - - this.testFailures.push(testFailure); + if (result.status === 'failed') { + this.report.logFailure({ + id: getTestIDForTitle(test.titlePath().join(' ')), + suite: test.parent.title, + title: test.title, + target: this.target, + command: this.command, + location: test.location.file.replace(`${REPO_ROOT}/`, ''), + owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)), + plugin: this.plugin, + duration: result.duration, + error: { + message: result.error?.message + ? stripfilePath(stripANSI(result.error.message)) + : undefined, + stack_trace: result.error?.stack + ? stripfilePath(stripANSI(result.error.stack)) + : undefined, + }, + attachments: result.attachments.map((attachment) => ({ + name: attachment.name, + path: attachment.path, + contentType: attachment.contentType, + })), + }); + } } onEnd(result: FullResult) { - if (this.testFailures.length === 0) return; - - const filename = `failed-tests-${this.plugin?.id}.json`; - saveTestFailuresReport(this.testFailures, this.reportRootPath, filename, this.log); - - // Generate HTML report for each failed test with embedded screenshots - for (const testFailure of this.testFailures) { - const failureHtml = buildFailureHtml(testFailure); - const htmlFilename = `${testFailure.id}.html`; - saveTestFailureHtml(this.reportRootPath, htmlFilename, failureHtml, this.log); + // Save & conclude the report + try { + this.report.save(this.reportRootPath); + } finally { + this.report.conclude(); } } } diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failure_report.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failure_report.ts new file mode 100644 index 0000000000000..2f014da6a0454 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failure_report.ts @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +// eslint-disable-next-line max-classes-per-file +import os from 'node:os'; +import path from 'node:path'; +import fs from 'node:fs'; +import { ToolingLog } from '@kbn/tooling-log'; +import { TestFailure } from '../../test_failure'; +import { saveTestFailuresReport } from './save_test_failures'; +import { buildFailureHtml } from './build_test_failure_html'; + +/** + * Generic error raised by a Scout report + */ +export class ScoutReportError extends Error {} + +/** + * + */ +export class ScoutFailureReport { + log: ToolingLog; + workDir: string; + concluded = false; + + constructor(log?: ToolingLog) { + this.log = log || new ToolingLog(); + this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-failures-report-')); + } + + public get testFailuresPath(): string { + return path.join(this.workDir, `test-failures.ndjson`); + } + + private raiseIfConcluded(additionalInfo?: string) { + if (this.concluded) { + let message = `Report at ${this.workDir} was concluded`; + + if (additionalInfo) { + message += `: ${additionalInfo}`; + } + + throw new ScoutReportError(message); + } + } + + /** + * Logs a failure to be processed by this reporter + * + * @param failure {TestFailure} - test failure to record + */ + logFailure(failure: TestFailure) { + this.raiseIfConcluded('logging new failures is no longer allowed'); + + fs.appendFileSync(this.testFailuresPath, JSON.stringify(failure) + '\n'); + } + + /** + * Save the report to a non-ephemeral location + * + * @param destination - Full path to the save location. Must not exist. + */ + save(destination: string) { + this.raiseIfConcluded('nothing to save because workdir has been cleared'); + + if (fs.existsSync(destination)) { + throw new ScoutReportError(`Save destination path '${destination}' already exists`); + } + + // Create the destination directory + this.log.info(`Saving Scout failures report to ${destination}`); + fs.mkdirSync(destination, { recursive: true }); + + const testFailures: TestFailure[] = this.readFailuresFromNDJSON(); + + // Generate HTML report for each failed test with embedded screenshots + for (const failure of testFailures) { + const htmlContent = buildFailureHtml(failure); + const htmlReportPath = path.join(destination, `${failure.id}.html`); + saveTestFailuresReport(htmlReportPath, htmlContent, this.log); + } + + const summaryContent = testFailures.map((failure) => { + return { + name: `${failure.target} - ${failure.suite} - ${failure.title}`, + htmlReportFilename: `${failure.id}.html`, + }; + }); + + // Short summary report linking to the detailed HTML reports + const testFailuresSummaryReportPath = path.join(destination, 'test-failures-summary.json'); + saveTestFailuresReport( + testFailuresSummaryReportPath, + JSON.stringify(summaryContent, null, 2), + this.log + ); + } + + /** + * Call this when you're done adding information to this report. + * + * ⚠️**This will delete all the contents of the report's working directory** + */ + conclude() { + // Remove the working directory + this.log.info(`Removing Scout report working directory ${this.workDir}`); + fs.rmSync(this.workDir, { recursive: true, force: true }); + + // Mark this report as concluded + this.concluded = true; + this.log.success('Scout report has concluded.'); + } + + /** + * Reads all failures from the NDJSON file and parses them as TestFailure[]. + */ + readFailuresFromNDJSON(): TestFailure[] { + if (!fs.existsSync(this.testFailuresPath)) { + return []; + } + + const fileContent = fs.readFileSync(this.testFailuresPath, 'utf-8'); + return fileContent + .split('\n') // Split lines + .filter((line) => line.trim() !== '') // Remove empty lines + .map((line) => JSON.parse(line) as TestFailure); // Parse each line into an object + } +} diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts deleted file mode 100644 index ad21076a9ca93..0000000000000 --- a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts index bf11d00e9a265..8b9c19f182acd 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts @@ -8,38 +8,17 @@ */ import fs from 'fs'; -import path from 'path'; import { ToolingLog } from '@kbn/tooling-log'; -import { TestFailure } from '../../test_failure'; export const saveTestFailuresReport = ( - testFailures: TestFailure[], - reportRootPath: string, - filename: string, - log: ToolingLog -): void => { - try { - const reportPath = path.join(reportRootPath, filename); - fs.mkdirSync(reportRootPath, { recursive: true }); - fs.writeFileSync(reportPath, JSON.stringify({ failures: testFailures }, null, 2), 'utf-8'); - log.info(`Saving Scout failed test report to ${reportPath}`); - } catch (error) { - log.error(`Failed to save report at ${reportRootPath}: ${error.message}`); - } -}; - -export const saveTestFailureHtml = ( - reportRootPath: string, - filename: string, + reportPath: string, testFailureHtml: string, log: ToolingLog ): void => { try { - const reportPath = path.join(reportRootPath, filename); - fs.mkdirSync(reportRootPath, { recursive: true }); fs.writeFileSync(reportPath, testFailureHtml, 'utf-8'); log.info(`Saving Scout test failure html to ${reportPath}`); } catch (error) { - log.error(`Failed to save report at ${reportRootPath}: ${error.message}`); + log.error(`Failed to save report at ${reportPath}: ${error.message}`); } }; From 5f71085c48a93432f0dc5c96db504ef16794affa Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Tue, 7 Jan 2025 13:25:18 +0100 Subject: [PATCH 05/21] fail tests --- .../ui_tests/tests/saved_searches.spec.ts | 8 ++++---- .../ui_tests/tests/value_suggestions.spec.ts | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/saved_searches.spec.ts b/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/saved_searches.spec.ts index 94a0d6b7b0638..e1c5644d94a94 100644 --- a/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/saved_searches.spec.ts +++ b/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/saved_searches.spec.ts @@ -71,10 +71,10 @@ test.describe('Discover app - saved searches', { tag: ['@ess', '@svlSearch', '@s state: 'visible', }); - await pageObjects.dashboard.customizePanel({ - name: PANEL_NAME, - customTimeRageCommonlyUsed: { value: 'Last_90 days' }, - }); + // await pageObjects.dashboard.customizePanel({ + // name: PANEL_NAME, + // customTimeRageCommonlyUsed: { value: 'Last_90 days' }, + // }); await expect( page.testSubj.locator('embeddedSavedSearchDocTable').locator('.euiDataGrid__noResults'), 'No results message in Saved Search panel should be visible' diff --git a/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts b/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts index 40b07726b31e4..b4793f5d6a9ef 100644 --- a/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts +++ b/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts @@ -8,7 +8,7 @@ import { expect, tags } from '@kbn/scout'; import { test, testData, assertionMessages } from '../fixtures'; -test.describe( +test.describe.only( 'Discover app - value suggestions: useTimeRange enabled', { tag: tags.DEPLOYMENT_AGNOSTIC }, () => { @@ -34,7 +34,7 @@ test.describe( test('dont show up if outside of range', async ({ page, pageObjects }) => { await pageObjects.datePicker.setAbsoluteRange(testData.LOGSTASH_OUT_OF_RANGE_DATES); await page.testSubj.fill('queryInput', 'extension.raw : '); - await expect(page.testSubj.locator('autoCompleteSuggestionText')).toHaveCount(0); + await expect(page.testSubj.locator('autoCompleteSuggestionText')).toHaveCount(4); }); test('show up if in range', async ({ page, pageObjects }) => { @@ -47,7 +47,7 @@ test.describe( const actualSuggestions = await page.testSubj .locator('autoCompleteSuggestionText') .allTextContents(); - expect(actualSuggestions.join(',')).toContain('jpg'); + expect(actualSuggestions.join(',')).toContain('xyz'); }); test('also displays descriptions for operators', async ({ page, pageObjects }) => { From de7d1ddc37ee065e140848b117e73c4c74363114 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Tue, 7 Jan 2025 13:26:22 +0100 Subject: [PATCH 06/21] run only stateful tests --- .buildkite/scripts/steps/functional/scout_ui_tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/steps/functional/scout_ui_tests.sh b/.buildkite/scripts/steps/functional/scout_ui_tests.sh index b568ed9c80b1a..82260fe528a22 100755 --- a/.buildkite/scripts/steps/functional/scout_ui_tests.sh +++ b/.buildkite/scripts/steps/functional/scout_ui_tests.sh @@ -11,9 +11,9 @@ KIBANA_DIR="$KIBANA_BUILD_LOCATION" declare -A TESTS=( ["Stateful"]="--stateful" - ["Serverless Elasticsearch"]="--serverless=es" - ["Serverless Observability"]="--serverless=oblt" - ["Serverless Security"]="--serverless=security" + # ["Serverless Elasticsearch"]="--serverless=es" + # ["Serverless Observability"]="--serverless=oblt" + # ["Serverless Security"]="--serverless=security" ) ORDER=("Stateful" "Serverless Elasticsearch" "Serverless Observability" "Serverless Security") From fe5d4482a7c14c57fabe1068e52d98eeb96363d5 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:38:09 +0000 Subject: [PATCH 07/21] [CI] Auto-commit changed files from 'node scripts/notice' --- packages/kbn-scout-reporting/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/kbn-scout-reporting/tsconfig.json b/packages/kbn-scout-reporting/tsconfig.json index 30b5e3fca4e6c..7e596426d705b 100644 --- a/packages/kbn-scout-reporting/tsconfig.json +++ b/packages/kbn-scout-reporting/tsconfig.json @@ -20,5 +20,6 @@ "@kbn/scout-info", "@kbn/repo-info", "@kbn/code-owners", + "@kbn/repo-packages", ] } From 94e722ada01384f57d1c5757a7e15c6d2d7ac982 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Tue, 7 Jan 2025 15:51:07 +0100 Subject: [PATCH 08/21] update test --- .../discover_enhanced/ui_tests/tests/value_suggestions.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts b/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts index b4793f5d6a9ef..aeb57eeedbf04 100644 --- a/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts +++ b/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts @@ -8,7 +8,7 @@ import { expect, tags } from '@kbn/scout'; import { test, testData, assertionMessages } from '../fixtures'; -test.describe.only( +test.describe( 'Discover app - value suggestions: useTimeRange enabled', { tag: tags.DEPLOYMENT_AGNOSTIC }, () => { From 6f73332dd54774d1234dd58641cf85c0c71b5c2e Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Tue, 7 Jan 2025 16:03:07 +0100 Subject: [PATCH 09/21] upload scout failure artifacts --- .buildkite/scripts/lifecycle/post_command.sh | 1 + .../playwright/failed_test/failed_test_reporter.ts | 7 +------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/lifecycle/post_command.sh b/.buildkite/scripts/lifecycle/post_command.sh index f90a4b451be1f..01dbfed1dca60 100755 --- a/.buildkite/scripts/lifecycle/post_command.sh +++ b/.buildkite/scripts/lifecycle/post_command.sh @@ -10,6 +10,7 @@ IS_TEST_EXECUTION_STEP="$(buildkite-agent meta-data get "${BUILDKITE_JOB_ID}_is_ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then echo "--- Upload Artifacts" + buildkite-agent artifact upload '.scout/reports/scout-playwright-test-failures-*' buildkite-agent artifact upload 'target/junit/**/*' buildkite-agent artifact upload 'target/kibana-coverage/jest/**/*' buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*' diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts index e415e495b3e4b..06eb100bc3af8 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts @@ -65,12 +65,7 @@ export class ScoutFailedTestReporter implements Reporter { public get reportRootPath(): string { const outputPath = this.reporterOptions.outputPath || SCOUT_REPORT_OUTPUT_ROOT; - return path.join( - outputPath, - `scout-playwright-test-failures-${this.runId}`, - 'target', - 'test-failures' - ); + return path.join(outputPath, `scout-playwright-test-failures-${this.runId}`); } printsToStdio(): boolean { From f933d84ca8ad1ade45c69cffdad75da8faff0fa8 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Tue, 7 Jan 2025 17:30:31 +0100 Subject: [PATCH 10/21] add logging --- .buildkite/scripts/lifecycle/post_command.sh | 2 +- .../failed_tests_reporter_cli.ts | 54 +++++++++++++++++++ .../playwright/failed_test/failure_report.ts | 23 ++++++-- .../failed_test/save_test_failures.ts | 5 +- 4 files changed, 76 insertions(+), 8 deletions(-) diff --git a/.buildkite/scripts/lifecycle/post_command.sh b/.buildkite/scripts/lifecycle/post_command.sh index c4bb111b8cc3e..164004431c840 100755 --- a/.buildkite/scripts/lifecycle/post_command.sh +++ b/.buildkite/scripts/lifecycle/post_command.sh @@ -10,7 +10,7 @@ IS_TEST_EXECUTION_STEP="$(buildkite-agent meta-data get "${BUILDKITE_JOB_ID}_is_ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then echo "--- Upload Artifacts" - buildkite-agent artifact upload '.scout/reports/scout-playwright-test-failures-*' + buildkite-agent artifact upload '.scout/reports/scout-playwright-test-failures-*/**/*' buildkite-agent artifact upload 'target/junit/**/*' buildkite-agent artifact upload 'target/kibana-coverage/jest/**/*' buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*' diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts index 36466c3e3637e..f91f16ff4b80b 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -15,6 +15,8 @@ import { createFailError, createFlagError } from '@kbn/dev-cli-errors'; import { CiStatsReporter } from '@kbn/ci-stats-reporter'; import globby from 'globby'; import normalize from 'normalize-path'; +import fs from 'fs'; +import { createHash } from 'crypto'; import { getFailures } from './get_failures'; import { GithubApi } from './github_api'; @@ -184,6 +186,58 @@ run( await reportFailuresToFile(log, failures, bkMeta, getRootMetadata(report)); } + + // Scout test failures reporting + log.info('Searching for Scout test failure reports'); + const scoutTestFailuresDirPattern = '.scout/reports/scout-playwright-test-failures-*'; + + const dirs = await globby(scoutTestFailuresDirPattern, { + onlyDirectories: true, + }); + + if (dirs.length === 0) { + throw new Error(`No directories found matching pattern: ${scoutTestFailuresDirPattern}`); + } + + const dirPath = dirs[0]; // temp take the last one + const summaryFilePath = Path.join(dirPath, 'test-failures-summary.json'); + + // Check if summary JSON exists + if (!fs.existsSync(summaryFilePath)) { + throw new Error(`Summary file not found in: ${dirPath}`); + } + + const summaryData: Array<{ name: string; htmlReportFilename: string }> = JSON.parse( + fs.readFileSync(summaryFilePath, 'utf-8') + ); + + log.info('Creating failure artifacts for', summaryData.length, 'test failures'); + for (const { name, htmlReportFilename } of summaryData) { + const htmlFilePath = Path.join(dirPath, htmlReportFilename); + const failureHTML = fs.readFileSync(htmlFilePath, 'utf-8'); + + const hash = createHash('md5').update(name).digest('hex'); // eslint-disable-line @kbn/eslint/no_unsafe_hash + const filenameBase = `${ + process.env.BUILDKITE_JOB_ID ? process.env.BUILDKITE_JOB_ID + '_' : '' + }${hash}`; + const dir = Path.join('target', 'test_failures'); + const failureJSON = JSON.stringify( + { + name, + hash, + buildId: bkMeta.buildId, + jobId: bkMeta.jobId, + url: bkMeta.url, + jobUrl: bkMeta.jobUrl, + jobName: bkMeta.jobName, + }, + null, + 2 + ); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(Path.join(dir, `${filenameBase}.html`), failureHTML, 'utf8'); + fs.writeFileSync(Path.join(dir, `${filenameBase}.json`), failureJSON, 'utf8'); + } } finally { await CiStatsReporter.fromEnv(log).metrics([ { diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failure_report.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failure_report.ts index 2f014da6a0454..a0e65acb6370c 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failure_report.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failure_report.ts @@ -73,17 +73,29 @@ export class ScoutFailureReport { throw new ScoutReportError(`Save destination path '${destination}' already exists`); } + const testFailures: TestFailure[] = this.readFailuresFromNDJSON(); + + if (testFailures.length === 0) { + this.log.info('No test failures to report'); + return; + } + // Create the destination directory - this.log.info(`Saving Scout failures report to ${destination}`); + this.log.info( + `Saving Scout failures report to ${destination}: ${testFailures.length} failures reported` + ); fs.mkdirSync(destination, { recursive: true }); - const testFailures: TestFailure[] = this.readFailuresFromNDJSON(); - // Generate HTML report for each failed test with embedded screenshots for (const failure of testFailures) { const htmlContent = buildFailureHtml(failure); const htmlReportPath = path.join(destination, `${failure.id}.html`); - saveTestFailuresReport(htmlReportPath, htmlContent, this.log); + saveTestFailuresReport( + htmlReportPath, + htmlContent, + this.log, + `html report for ${failure.id} is saved at ${htmlReportPath}` + ); } const summaryContent = testFailures.map((failure) => { @@ -98,7 +110,8 @@ export class ScoutFailureReport { saveTestFailuresReport( testFailuresSummaryReportPath, JSON.stringify(summaryContent, null, 2), - this.log + this.log, + `Summary report is saved at ${testFailuresSummaryReportPath}` ); } diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts index 8b9c19f182acd..18a7df3a98137 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts @@ -13,11 +13,12 @@ import { ToolingLog } from '@kbn/tooling-log'; export const saveTestFailuresReport = ( reportPath: string, testFailureHtml: string, - log: ToolingLog + log: ToolingLog, + message: string ): void => { try { fs.writeFileSync(reportPath, testFailureHtml, 'utf-8'); - log.info(`Saving Scout test failure html to ${reportPath}`); + log.info(message); } catch (error) { log.error(`Failed to save report at ${reportPath}: ${error.message}`); } From 9977feb39d1e0f20de1733260c07c0b510ae6494 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Tue, 7 Jan 2025 18:30:32 +0100 Subject: [PATCH 11/21] 0 retries, adjust post build script --- .../failed_tests_reporter_cli.ts | 135 +++++++++--------- .../src/playwright/config/create_config.ts | 2 +- 2 files changed, 68 insertions(+), 69 deletions(-) diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts index f91f16ff4b80b..0d56da397f432 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -103,88 +103,86 @@ run( return; } - if (!reportPaths.length) { - throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`); - } + if (reportPaths.length) { + log.info('found', reportPaths.length, 'junit reports', reportPaths); - log.info('found', reportPaths.length, 'junit reports', reportPaths); + const existingIssues = new ExistingFailedTestIssues(log); + for (const reportPath of reportPaths) { + const report = await readTestReport(reportPath); + const messages = Array.from(getReportMessageIter(report)); + const failures = getFailures(report); - const existingIssues = new ExistingFailedTestIssues(log); - for (const reportPath of reportPaths) { - const report = await readTestReport(reportPath); - const messages = Array.from(getReportMessageIter(report)); - const failures = getFailures(report); + await existingIssues.loadForFailures(failures); - await existingIssues.loadForFailures(failures); + if (indexInEs) { + await reportFailuresToEs(log, failures); + } - if (indexInEs) { - await reportFailuresToEs(log, failures); - } + for (const failure of failures) { + const pushMessage = (msg: string) => { + messages.push({ + classname: failure.classname, + name: failure.name, + message: msg, + }); + }; + + if (failure.likelyIrrelevant) { + pushMessage( + 'Failure is likely irrelevant' + + (updateGithub ? ', so an issue was not created or updated' : '') + ); + continue; + } - for (const failure of failures) { - const pushMessage = (msg: string) => { - messages.push({ - classname: failure.classname, - name: failure.name, - message: msg, - }); - }; - - if (failure.likelyIrrelevant) { - pushMessage( - 'Failure is likely irrelevant' + - (updateGithub ? ', so an issue was not created or updated' : '') - ); - continue; - } + const existingIssue = existingIssues.getForFailure(failure); + if (existingIssue) { + const { newBody, newCount } = await updateFailureIssue( + buildUrl, + existingIssue, + githubApi, + branch, + pipeline + ); + const url = existingIssue.github.htmlUrl; + existingIssue.github.body = newBody; + failure.githubIssue = url; + failure.failureCount = updateGithub ? newCount : newCount - 1; + pushMessage(`Test has failed ${newCount - 1} times on tracked branches: ${url}`); + if (updateGithub) { + pushMessage(`Updated existing issue: ${url} (fail count: ${newCount})`); + } + continue; + } - const existingIssue = existingIssues.getForFailure(failure); - if (existingIssue) { - const { newBody, newCount } = await updateFailureIssue( + const newIssue = await createFailureIssue( buildUrl, - existingIssue, + failure, githubApi, branch, - pipeline + pipeline, + prependTitle ); - const url = existingIssue.github.htmlUrl; - existingIssue.github.body = newBody; - failure.githubIssue = url; - failure.failureCount = updateGithub ? newCount : newCount - 1; - pushMessage(`Test has failed ${newCount - 1} times on tracked branches: ${url}`); + existingIssues.addNewlyCreated(failure, newIssue); + pushMessage('Test has not failed recently on tracked branches'); if (updateGithub) { - pushMessage(`Updated existing issue: ${url} (fail count: ${newCount})`); + pushMessage(`Created new issue: ${newIssue.html_url}`); + failure.githubIssue = newIssue.html_url; } - continue; - } - - const newIssue = await createFailureIssue( - buildUrl, - failure, - githubApi, - branch, - pipeline, - prependTitle - ); - existingIssues.addNewlyCreated(failure, newIssue); - pushMessage('Test has not failed recently on tracked branches'); - if (updateGithub) { - pushMessage(`Created new issue: ${newIssue.html_url}`); - failure.githubIssue = newIssue.html_url; + failure.failureCount = updateGithub ? 1 : 0; } - failure.failureCount = updateGithub ? 1 : 0; - } - // mutates report to include messages and writes updated report to disk - await addMessagesToReport({ - report, - messages, - log, - reportPath, - dryRun: !flags['report-update'], - }); + // mutates report to include messages and writes updated report to disk + await addMessagesToReport({ + report, + messages, + log, + reportPath, + dryRun: !flags['report-update'], + }); - await reportFailuresToFile(log, failures, bkMeta, getRootMetadata(report)); + await reportFailuresToFile(log, failures, bkMeta, getRootMetadata(report)); + } } // Scout test failures reporting @@ -196,7 +194,8 @@ run( }); if (dirs.length === 0) { - throw new Error(`No directories found matching pattern: ${scoutTestFailuresDirPattern}`); + throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`); + // log.info(`No directories found matching pattern: ${scoutTestFailuresDirPattern}`); } const dirPath = dirs[0]; // temp take the last one diff --git a/packages/kbn-scout/src/playwright/config/create_config.ts b/packages/kbn-scout/src/playwright/config/create_config.ts index 75b06bf3dfae6..6fa7ba18fc507 100644 --- a/packages/kbn-scout/src/playwright/config/create_config.ts +++ b/packages/kbn-scout/src/playwright/config/create_config.ts @@ -26,7 +26,7 @@ export function createPlaywrightConfig(options: ScoutPlaywrightOptions): Playwri /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, /* Retry on CI only */ - retries: process.env.CI ? 2 : 0, + retries: 0, // process.env.CI ? 2 : 0, /* Opt out of parallel tests on CI. */ workers: options.workers ?? 1, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ From 1f7a912f29238b2a3149c69eeb72bcc0497ebf9b Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Thu, 9 Jan 2025 16:42:21 +0100 Subject: [PATCH 12/21] package structure rafactoring --- .../src/cli/initialize_report_datastream.ts | 2 +- .../src/cli/upload_events.ts | 2 +- .../src/reporting/index.ts | 23 ++--- .../playwright/common/buildkite_meta.ts | 31 ------ .../playwright/common/get_run_target.test.ts | 68 +++++++++++++ .../playwright/common/get_run_target.ts | 28 ++++++ .../src/reporting/playwright/common/index.ts | 5 +- .../playwright/common/parse_stdout.test.ts | 62 ++++++++++++ .../playwright/common/parse_stdout.ts | 19 ++++ .../common/{strip_path.ts => strip_utils.ts} | 6 +- .../playwright/common/test_id_generator.ts | 18 ++++ .../src/reporting/playwright/events/index.ts | 12 +++ .../playwright/events/playwright_reporter.ts | 6 +- .../failed_test/failed_test_reporter.ts | 37 ++++--- .../reporting/playwright/failed_test/index.ts | 12 +++ .../src/reporting/report/{ => event}/event.ts | 2 +- .../src/reporting/report/event/index.ts | 11 +++ .../persistence/component_templates.ts | 0 .../report/{ => event}/persistence/index.ts | 0 .../persistence/index_templates.ts | 0 .../{ => event}/persistence/mappings.ts | 0 .../src/reporting/report/event/report.ts | 69 +++++++++++++ .../failed_test/build_test_failure_html.ts | 36 ++++++- .../failed_test/failure_report.ts | 46 ++------- .../src/reporting/report/failed_test/index.ts | 11 +++ .../failed_test/save_test_failures.ts | 0 .../failed_test/test_failure.ts} | 3 +- .../src/reporting/report/generic_report.ts | 70 ++++++++++++++ .../src/reporting/report/index.ts | 96 +------------------ 29 files changed, 457 insertions(+), 218 deletions(-) delete mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/common/buildkite_meta.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/common/get_run_target.test.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/common/get_run_target.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/common/parse_stdout.test.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/common/parse_stdout.ts rename packages/kbn-scout-reporting/src/reporting/playwright/common/{strip_path.ts => strip_utils.ts} (73%) create mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/common/test_id_generator.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/events/index.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts rename packages/kbn-scout-reporting/src/reporting/report/{ => event}/event.ts (96%) create mode 100644 packages/kbn-scout-reporting/src/reporting/report/event/index.ts rename packages/kbn-scout-reporting/src/reporting/report/{ => event}/persistence/component_templates.ts (100%) rename packages/kbn-scout-reporting/src/reporting/report/{ => event}/persistence/index.ts (100%) rename packages/kbn-scout-reporting/src/reporting/report/{ => event}/persistence/index_templates.ts (100%) rename packages/kbn-scout-reporting/src/reporting/report/{ => event}/persistence/mappings.ts (100%) create mode 100644 packages/kbn-scout-reporting/src/reporting/report/event/report.ts rename packages/kbn-scout-reporting/src/reporting/{playwright => report}/failed_test/build_test_failure_html.ts (87%) rename packages/kbn-scout-reporting/src/reporting/{playwright => report}/failed_test/failure_report.ts (76%) create mode 100644 packages/kbn-scout-reporting/src/reporting/report/failed_test/index.ts rename packages/kbn-scout-reporting/src/reporting/{playwright => report}/failed_test/save_test_failures.ts (100%) rename packages/kbn-scout-reporting/src/reporting/{test_failure.d.ts => report/failed_test/test_failure.ts} (94%) create mode 100644 packages/kbn-scout-reporting/src/reporting/report/generic_report.ts diff --git a/packages/kbn-scout-reporting/src/cli/initialize_report_datastream.ts b/packages/kbn-scout-reporting/src/cli/initialize_report_datastream.ts index 314794a9181b7..af3be7fe43b14 100644 --- a/packages/kbn-scout-reporting/src/cli/initialize_report_datastream.ts +++ b/packages/kbn-scout-reporting/src/cli/initialize_report_datastream.ts @@ -8,7 +8,7 @@ */ import { Command } from '@kbn/dev-cli-runner'; -import { ScoutReportDataStream } from '../reporting/report'; +import { ScoutReportDataStream } from '../reporting/report/event/report'; import { getValidatedESClient } from './common'; export const initializeReportDatastream: Command = { diff --git a/packages/kbn-scout-reporting/src/cli/upload_events.ts b/packages/kbn-scout-reporting/src/cli/upload_events.ts index 8c2ef1bd67347..f0eae0ec5df7e 100644 --- a/packages/kbn-scout-reporting/src/cli/upload_events.ts +++ b/packages/kbn-scout-reporting/src/cli/upload_events.ts @@ -10,7 +10,7 @@ import fs from 'node:fs'; import { Command } from '@kbn/dev-cli-runner'; import { createFlagError } from '@kbn/dev-cli-errors'; -import { ScoutReportDataStream } from '../reporting/report'; +import { ScoutReportDataStream } from '../reporting/report/event/report'; import { getValidatedESClient } from './common'; export const uploadEvents: Command = { diff --git a/packages/kbn-scout-reporting/src/reporting/index.ts b/packages/kbn-scout-reporting/src/reporting/index.ts index bcdacdee129de..28605c5452e7d 100644 --- a/packages/kbn-scout-reporting/src/reporting/index.ts +++ b/packages/kbn-scout-reporting/src/reporting/index.ts @@ -7,34 +7,23 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { createHash, randomBytes } from 'node:crypto'; import type { ReporterDescription } from 'playwright/test'; import { ScoutPlaywrightReporterOptions } from './scout_playwright_reporter'; -export * from './report'; +export * from './report/event/report'; -// ID helpers -export function generateTestRunId() { - return randomBytes(8).toString('hex'); -} - -export function getTestIDForTitle(title: string) { - return createHash('sha256').update(title).digest('hex').slice(0, 31); -} - -// Playwright reporting +// Playwright event-based reporting export const scoutPlaywrightReporter = ( options?: ScoutPlaywrightReporterOptions ): ReporterDescription => { - return ['@kbn/scout-reporting/src/reporting/playwright/events/playwright_reporter.ts', options]; + return ['@kbn/scout-reporting/src/reporting/playwright/events/index.ts', options]; }; // Playwright failed test reporting export const scoutFailedTestsReporter = ( options?: ScoutPlaywrightReporterOptions ): ReporterDescription => { - return [ - '@kbn/scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts', - options, - ]; + return ['@kbn/scout-reporting/src/reporting/playwright/failed_test/index.ts', options]; }; + +export { generateTestRunId } from './playwright/common'; diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/common/buildkite_meta.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/buildkite_meta.ts deleted file mode 100644 index 7bc6ebb39dba3..0000000000000 --- a/packages/kbn-scout-reporting/src/reporting/playwright/common/buildkite_meta.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -export function getBuildkiteMetadata() { - // Buildkite steps that use `parallelism` need a numerical suffix added to identify them - // We should also increment the number by one, since it's 0-based - const jobNumberSuffix = process.env.BUILDKITE_PARALLEL_JOB - ? ` #${parseInt(process.env.BUILDKITE_PARALLEL_JOB, 10) + 1}` - : ''; - - const buildUrl = process.env.BUILDKITE_BUILD_URL; - const jobUrl = process.env.BUILDKITE_JOB_ID - ? `${buildUrl}#${process.env.BUILDKITE_JOB_ID}` - : undefined; - - return { - buildId: process.env.BUJILDKITE_BUILD_ID, - jobId: process.env.BUILDKITE_JOB_ID, - url: buildUrl, - jobUrl, - jobName: process.env.BUILDKITE_LABEL - ? `${process.env.BUILDKITE_LABEL}${jobNumberSuffix}` - : undefined, - }; -} diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/common/get_run_target.test.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/get_run_target.test.ts new file mode 100644 index 0000000000000..588758ab3ced7 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/get_run_target.test.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { getRunTarget } from './get_run_target'; + +describe('getRunTarget', () => { + it(`should return the correct mode for '--grep=@svlSearch'`, () => { + const argv = [ + 'node', + 'scripts/scout.js', + 'run-tests', + '--config', + 'path/to/config', + '--grep=@svlSearch', + ]; + expect(getRunTarget(argv)).toBe('serverless-search'); + }); + + it(`should return the correct mode for '--grep @svlSearch'`, () => { + const argv = [ + 'node', + 'scripts/scout.js', + 'run-tests', + '--config', + 'path/to/config', + '--grep', + '@svlSearch', + ]; + expect(getRunTarget(argv)).toBe('serverless-search'); + }); + + it(`should return 'undefined' for an invalid --grep tag`, () => { + const argv = [ + 'node', + 'scripts/scout.js', + 'run-tests', + '--config', + 'path/to/config', + '--grep=@invalidTag', + ]; + expect(getRunTarget(argv)).toBe('undefined'); + }); + + it(`should return 'undefined' if --grep argument is not provided`, () => { + const argv = ['node', 'scripts/scout.js']; + expect(getRunTarget(argv)).toBe('undefined'); + }); + + it(`should default to 'undefined' if argv is not passed`, () => { + expect(getRunTarget()).toBe('undefined'); + }); + + it(`should return 'undefined' for '--grep='`, () => { + const argv = ['node', 'scripts/scout.js', '--grep=']; + expect(getRunTarget(argv)).toBe('undefined'); + }); + + it(`should return "undefined" if "--grep" argument is without value`, () => { + const argv = ['node', 'scripts/scout.js', '--grep']; + expect(getRunTarget(argv)).toBe('undefined'); + }); +}); diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/common/get_run_target.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/get_run_target.ts new file mode 100644 index 0000000000000..6f2bc412ab8f1 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/get_run_target.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export function getRunTarget(argv: string[] = process.argv): string { + const tagsToMode: Record = { + '@ess': 'stateful', + '@svlSearch': 'serverless-search', + '@svlOblt': 'serverless-oblt', + '@svlSecurity': 'serverless-security', + }; + + const grepIndex = argv.findIndex((arg) => arg === '--grep' || arg.startsWith('--grep=')); + if (grepIndex !== -1) { + const tag = argv[grepIndex].startsWith('--grep=') + ? argv[grepIndex].split('=')[1] + : argv[grepIndex + 1] || ''; // Look at the next argument if '--grep' is used without `=` + + return tagsToMode[tag] || 'undefined'; + } + + return 'undefined'; +} diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/common/index.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/index.ts index 61e5203da1454..a2334dfb0e1e6 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/common/index.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/index.ts @@ -8,4 +8,7 @@ */ export { getPluginManifestData, type PluginManifest } from './get_plugin_manifest_data'; -export { stripRunCommand, stripfilePath } from './strip_path'; +export { stripRunCommand, stripFilePath } from './strip_utils'; +export { parseStdout } from './parse_stdout'; +export { getRunTarget } from './get_run_target'; +export { getTestIDForTitle, generateTestRunId } from './test_id_generator'; diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/common/parse_stdout.test.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/parse_stdout.test.ts new file mode 100644 index 0000000000000..5bf7ed782d593 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/parse_stdout.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { parseStdout } from './parse_stdout'; +import stripANSI from 'strip-ansi'; + +jest.mock('strip-ansi', () => jest.fn((input) => input.replace(/\x1b\[[0-9;]*m/g, ''))); + +describe('parseStdout', () => { + it('should concatenate multiple strings and strip ANSI codes', () => { + const stdout = ['Line 1 with ANSI \x1b[31mred\x1b[0m text', '\nLine 2 plain text']; + const result = parseStdout(stdout); + + expect(stripANSI).toHaveBeenCalledWith( + 'Line 1 with ANSI \x1b[31mred\x1b[0m text\nLine 2 plain text' + ); + expect(result).toBe('Line 1 with ANSI red text\nLine 2 plain text'); + }); + + it('should concatenate multiple buffers and strip ANSI codes', () => { + const stdout = [ + Buffer.from('Buffer line 1 with ANSI \x1b[32mgreen\x1b[0m text'), + Buffer.from('\nBuffer line 2 plain text'), + ]; + const result = parseStdout(stdout); + + expect(stripANSI).toHaveBeenCalledWith( + 'Buffer line 1 with ANSI \x1b[32mgreen\x1b[0m text\nBuffer line 2 plain text' + ); + expect(result).toBe('Buffer line 1 with ANSI green text\nBuffer line 2 plain text'); + }); + + it('should handle an empty array and return an empty string', () => { + const stdout: Array = []; + const result = parseStdout(stdout); + + expect(stripANSI).toHaveBeenCalledWith(''); + expect(result).toBe(''); + }); + + it('should handle an array with only an empty string', () => { + const stdout = ['']; + const result = parseStdout(stdout); + + expect(stripANSI).toHaveBeenCalledWith(''); + expect(result).toBe(''); + }); + + it('should handle an array with only an empty buffer', () => { + const stdout = [Buffer.from('')]; + const result = parseStdout(stdout); + + expect(stripANSI).toHaveBeenCalledWith(''); + expect(result).toBe(''); + }); +}); diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/common/parse_stdout.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/parse_stdout.ts new file mode 100644 index 0000000000000..c1802b39ba2e2 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/parse_stdout.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import stripANSI from 'strip-ansi'; + +export function parseStdout(stdout: Array): string { + const stdoutContent = stdout + .map((chunk) => (Buffer.isBuffer(chunk) ? chunk.toString() : chunk)) + .join(''); + + // Escape special HTML characters + return stripANSI(stdoutContent); +} diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/common/strip_path.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/strip_utils.ts similarity index 73% rename from packages/kbn-scout-reporting/src/reporting/playwright/common/strip_path.ts rename to packages/kbn-scout-reporting/src/reporting/playwright/common/strip_utils.ts index 5c5906e2b4dca..25c26c98cbdb3 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/common/strip_path.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/strip_utils.ts @@ -7,13 +7,13 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ +import stripANSI from 'strip-ansi'; import { REPO_ROOT } from '@kbn/repo-info'; -export const stripfilePath = (filePath: string): string => filePath.replaceAll(`${REPO_ROOT}/`, ''); +export const stripFilePath = (filePath: string): string => + stripANSI(filePath.replaceAll(`${REPO_ROOT}/`, '')); export const stripRunCommand = (commandArgs: string[]): string => { - // "/Users/user/.nvm/versions/node/v20.15.1/bin/node /Users/dmle/github/kibana/node_modules/.bin/playwright test --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts --grep=svlSearch", - // "npx playwright test --config x-pack/plugins/discover_enhanced/ui_tests/playwright.config.ts --grep=svlSearch", if (!Array.isArray(commandArgs) || commandArgs.length < 3) { throw new Error(`Invalid command arguments: must include at least 'npx playwright test'`); } diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/common/test_id_generator.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/test_id_generator.ts new file mode 100644 index 0000000000000..525f5b46d40d8 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/test_id_generator.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { createHash, randomBytes } from 'node:crypto'; + +export function generateTestRunId() { + return randomBytes(8).toString('hex'); +} + +export function getTestIDForTitle(title: string) { + return createHash('sha256').update(title).digest('hex').slice(0, 31); +} diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/events/index.ts b/packages/kbn-scout-reporting/src/reporting/playwright/events/index.ts new file mode 100644 index 0000000000000..717d9804dc882 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright/events/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { ScoutPlaywrightReporter } from './playwright_reporter'; +// eslint-disable-next-line import/no-default-export +export default ScoutPlaywrightReporter; diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts b/packages/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts index c29a61cda4c7a..f71de1726bfee 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/events/playwright_reporter.ts @@ -28,9 +28,10 @@ import { getCodeOwnersEntries, getOwningTeamsForPath, } from '@kbn/code-owners'; -import { generateTestRunId, getTestIDForTitle, ScoutReport, ScoutReportEventAction } from '../..'; +import { ScoutReport, ScoutReportEventAction } from '../../report'; import { environmentMetadata } from '../../../datasources'; import type { ScoutPlaywrightReporterOptions } from '../../scout_playwright_reporter'; +import { generateTestRunId, getTestIDForTitle } from '../common'; /** * Scout Playwright reporter @@ -283,6 +284,3 @@ export class ScoutPlaywrightReporter implements Reporter { }); } } - -// eslint-disable-next-line import/no-default-export -export default ScoutPlaywrightReporter; diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts index 06eb100bc3af8..b1a56751f9baa 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts @@ -19,19 +19,23 @@ import type { import path from 'node:path'; import { ToolingLog } from '@kbn/tooling-log'; import { SCOUT_REPORT_OUTPUT_ROOT } from '@kbn/scout-info'; -import stripANSI from 'strip-ansi'; import { REPO_ROOT } from '@kbn/repo-info'; import { type CodeOwnersEntry, getCodeOwnersEntries, getOwningTeamsForPath, } from '@kbn/code-owners'; -import { generateTestRunId, getTestIDForTitle } from '../..'; -import type { TestFailure } from '../../test_failure'; +import type { TestFailure } from '../../report'; import type { ScoutPlaywrightReporterOptions } from '../../scout_playwright_reporter'; -import { stripRunCommand, stripfilePath } from '../common/strip_path'; -import { getPluginManifestData } from '../common/get_plugin_manifest_data'; -import { ScoutFailureReport } from './failure_report'; +import { stripRunCommand, stripFilePath } from '../common/strip_utils'; +import { ScoutFailureReport } from '../../report'; +import { + getRunTarget, + getPluginManifestData, + parseStdout, + generateTestRunId, + getTestIDForTitle, +} from '../common'; /** * Scout Failed Test reporter @@ -82,16 +86,13 @@ export class ScoutFailedTestReporter implements Reporter { group: metadata.group, }; } - // Get the target from the --grep argument, e.g. --grep=@svlSearch - const grepArg = process.argv.find((arg) => arg.includes('--grep')); - if (grepArg) { - this.target = grepArg.split('=')[1]; - } + + this.target = getRunTarget(); } onTestEnd(test: TestCase, result: TestResult) { if (result.status === 'failed') { - this.report.logFailure({ + this.report.logEvent({ id: getTestIDForTitle(test.titlePath().join(' ')), suite: test.parent.title, title: test.title, @@ -102,13 +103,10 @@ export class ScoutFailedTestReporter implements Reporter { plugin: this.plugin, duration: result.duration, error: { - message: result.error?.message - ? stripfilePath(stripANSI(result.error.message)) - : undefined, - stack_trace: result.error?.stack - ? stripfilePath(stripANSI(result.error.stack)) - : undefined, + message: result.error?.message ? stripFilePath(result.error.message) : undefined, + stack_trace: result.error?.stack ? stripFilePath(result.error.stack) : undefined, }, + stdout: result.stdout ? parseStdout(result.stdout) : undefined, attachments: result.attachments.map((attachment) => ({ name: attachment.name, path: attachment.path, @@ -127,6 +125,3 @@ export class ScoutFailedTestReporter implements Reporter { } } } - -// eslint-disable-next-line import/no-default-export -export default ScoutFailedTestReporter; diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts new file mode 100644 index 0000000000000..c37c286c6c4e0 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { ScoutFailedTestReporter } from './failed_test_reporter'; +// eslint-disable-next-line import/no-default-export +export default ScoutFailedTestReporter; diff --git a/packages/kbn-scout-reporting/src/reporting/report/event.ts b/packages/kbn-scout-reporting/src/reporting/report/event/event.ts similarity index 96% rename from packages/kbn-scout-reporting/src/reporting/report/event.ts rename to packages/kbn-scout-reporting/src/reporting/report/event/event.ts index 1f6f8251f3b60..441ef3484e112 100644 --- a/packages/kbn-scout-reporting/src/reporting/report/event.ts +++ b/packages/kbn-scout-reporting/src/reporting/report/event/event.ts @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -import { BuildkiteMetadata, HostMetadata } from '../../datasources'; +import { BuildkiteMetadata, HostMetadata } from '../../../datasources'; /** * Scout reporter event type diff --git a/packages/kbn-scout-reporting/src/reporting/report/event/index.ts b/packages/kbn-scout-reporting/src/reporting/report/event/index.ts new file mode 100644 index 0000000000000..b74e818552ccd --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/report/event/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export { ScoutReport } from './report'; +export { ScoutReportEventAction } from './event'; diff --git a/packages/kbn-scout-reporting/src/reporting/report/persistence/component_templates.ts b/packages/kbn-scout-reporting/src/reporting/report/event/persistence/component_templates.ts similarity index 100% rename from packages/kbn-scout-reporting/src/reporting/report/persistence/component_templates.ts rename to packages/kbn-scout-reporting/src/reporting/report/event/persistence/component_templates.ts diff --git a/packages/kbn-scout-reporting/src/reporting/report/persistence/index.ts b/packages/kbn-scout-reporting/src/reporting/report/event/persistence/index.ts similarity index 100% rename from packages/kbn-scout-reporting/src/reporting/report/persistence/index.ts rename to packages/kbn-scout-reporting/src/reporting/report/event/persistence/index.ts diff --git a/packages/kbn-scout-reporting/src/reporting/report/persistence/index_templates.ts b/packages/kbn-scout-reporting/src/reporting/report/event/persistence/index_templates.ts similarity index 100% rename from packages/kbn-scout-reporting/src/reporting/report/persistence/index_templates.ts rename to packages/kbn-scout-reporting/src/reporting/report/event/persistence/index_templates.ts diff --git a/packages/kbn-scout-reporting/src/reporting/report/persistence/mappings.ts b/packages/kbn-scout-reporting/src/reporting/report/event/persistence/mappings.ts similarity index 100% rename from packages/kbn-scout-reporting/src/reporting/report/persistence/mappings.ts rename to packages/kbn-scout-reporting/src/reporting/report/event/persistence/mappings.ts diff --git a/packages/kbn-scout-reporting/src/reporting/report/event/report.ts b/packages/kbn-scout-reporting/src/reporting/report/event/report.ts new file mode 100644 index 0000000000000..89ce478fdea55 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/report/event/report.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import os from 'node:os'; +import path from 'node:path'; +import fs from 'node:fs'; +import { ToolingLog } from '@kbn/tooling-log'; +import { ScoutReportEvent } from './event'; +import { GenericReport, ScoutReportError } from '../generic_report'; + +export class ScoutReport extends GenericReport { + log: ToolingLog; + workDir: string; + concluded = false; + + constructor(log?: ToolingLog) { + super(); + this.log = log || new ToolingLog(); + this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-report-')); + } + + public get eventLogPath(): string { + return path.join(this.workDir, `event-log.ndjson`); + } + + /** + * Logs an event to be processed by this reporter + * + * @param event {ScoutReportEvent} - Event to record + */ + logEvent(event: ScoutReportEvent) { + this.raiseIfConcluded('logging new events is no longer allowed'); + + if (event['@timestamp'] === undefined) { + event['@timestamp'] = new Date(); + } + + fs.appendFileSync(this.eventLogPath, JSON.stringify(event) + '\n'); + } + + /** + * Save the report to a non-ephemeral location + * + * @param destination - Full path to the save location. Must not exist. + */ + save(destination: string) { + this.raiseIfConcluded('nothing to save because workdir has been cleared'); + + if (fs.existsSync(destination)) { + throw new ScoutReportError(`Save destination path '${destination}' already exists`); + } + + // Create the destination directory + this.log.info(`Saving Scout report to ${destination}`); + fs.mkdirSync(destination, { recursive: true }); + + // Copy the workdir data to the destination + fs.cpSync(this.workDir, destination, { recursive: true }); + } +} + +export * from './event'; +export * from './persistence'; diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/build_test_failure_html.ts b/packages/kbn-scout-reporting/src/reporting/report/failed_test/build_test_failure_html.ts similarity index 87% rename from packages/kbn-scout-reporting/src/reporting/playwright/failed_test/build_test_failure_html.ts rename to packages/kbn-scout-reporting/src/reporting/report/failed_test/build_test_failure_html.ts index a682a6d900bea..0b4467b128874 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/build_test_failure_html.ts +++ b/packages/kbn-scout-reporting/src/reporting/report/failed_test/build_test_failure_html.ts @@ -8,11 +8,22 @@ */ import fs from 'fs'; -import { TestFailure } from '../../test_failure'; +import { TestFailure } from './test_failure'; export const buildFailureHtml = (testFailure: TestFailure): string => { - const { suite, title, target, command, location, owner, plugin, duration, error, attachments } = - testFailure; + const { + suite, + title, + target, + command, + location, + owner, + plugin, + duration, + error, + stdout, + attachments, + } = testFailure; const testDuration = duration < 1000 ? `${duration}ms` : `${(duration / 1000).toFixed(2)}s`; @@ -166,6 +177,25 @@ export const buildFailureHtml = (testFailure: TestFailure): string => { 
${error?.stack_trace || 'No stack trace available'}
+
+
+ + Failures in tracked branches: + 0 + + +
+
+ +
+
Output Logs
+
+ 
${stdout || 'No output available'}
+
+
+
Attachments
${screenshots.join('/n')} diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failure_report.ts b/packages/kbn-scout-reporting/src/reporting/report/failed_test/failure_report.ts similarity index 76% rename from packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failure_report.ts rename to packages/kbn-scout-reporting/src/reporting/report/failed_test/failure_report.ts index a0e65acb6370c..e44560d1415c2 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failure_report.ts +++ b/packages/kbn-scout-reporting/src/reporting/report/failed_test/failure_report.ts @@ -7,29 +7,22 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -// eslint-disable-next-line max-classes-per-file import os from 'node:os'; import path from 'node:path'; import fs from 'node:fs'; import { ToolingLog } from '@kbn/tooling-log'; -import { TestFailure } from '../../test_failure'; import { saveTestFailuresReport } from './save_test_failures'; import { buildFailureHtml } from './build_test_failure_html'; +import { GenericReport, ScoutReportError } from '../generic_report'; +import { TestFailure } from './test_failure'; -/** - * Generic error raised by a Scout report - */ -export class ScoutReportError extends Error {} - -/** - * - */ -export class ScoutFailureReport { +export class ScoutFailureReport extends GenericReport { log: ToolingLog; workDir: string; concluded = false; constructor(log?: ToolingLog) { + super(); this.log = log || new ToolingLog(); this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-failures-report-')); } @@ -38,24 +31,12 @@ export class ScoutFailureReport { return path.join(this.workDir, `test-failures.ndjson`); } - private raiseIfConcluded(additionalInfo?: string) { - if (this.concluded) { - let message = `Report at ${this.workDir} was concluded`; - - if (additionalInfo) { - message += `: ${additionalInfo}`; - } - - throw new ScoutReportError(message); - } - } - /** * Logs a failure to be processed by this reporter * * @param failure {TestFailure} - test failure to record */ - logFailure(failure: TestFailure) { + logEvent(failure: TestFailure) { this.raiseIfConcluded('logging new failures is no longer allowed'); fs.appendFileSync(this.testFailuresPath, JSON.stringify(failure) + '\n'); @@ -115,25 +96,10 @@ export class ScoutFailureReport { ); } - /** - * Call this when you're done adding information to this report. - * - * ⚠️**This will delete all the contents of the report's working directory** - */ - conclude() { - // Remove the working directory - this.log.info(`Removing Scout report working directory ${this.workDir}`); - fs.rmSync(this.workDir, { recursive: true, force: true }); - - // Mark this report as concluded - this.concluded = true; - this.log.success('Scout report has concluded.'); - } - /** * Reads all failures from the NDJSON file and parses them as TestFailure[]. */ - readFailuresFromNDJSON(): TestFailure[] { + private readFailuresFromNDJSON(): TestFailure[] { if (!fs.existsSync(this.testFailuresPath)) { return []; } diff --git a/packages/kbn-scout-reporting/src/reporting/report/failed_test/index.ts b/packages/kbn-scout-reporting/src/reporting/report/failed_test/index.ts new file mode 100644 index 0000000000000..ce868d1d9b8c3 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/report/failed_test/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +export { ScoutFailureReport } from './failure_report'; +export { TestFailure } from './test_failure'; diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts b/packages/kbn-scout-reporting/src/reporting/report/failed_test/save_test_failures.ts similarity index 100% rename from packages/kbn-scout-reporting/src/reporting/playwright/failed_test/save_test_failures.ts rename to packages/kbn-scout-reporting/src/reporting/report/failed_test/save_test_failures.ts diff --git a/packages/kbn-scout-reporting/src/reporting/test_failure.d.ts b/packages/kbn-scout-reporting/src/reporting/report/failed_test/test_failure.ts similarity index 94% rename from packages/kbn-scout-reporting/src/reporting/test_failure.d.ts rename to packages/kbn-scout-reporting/src/reporting/report/failed_test/test_failure.ts index 942bcbb9a92b0..aa19263531a6f 100644 --- a/packages/kbn-scout-reporting/src/reporting/test_failure.d.ts +++ b/packages/kbn-scout-reporting/src/reporting/report/failed_test/test_failure.ts @@ -7,7 +7,7 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -export declare interface TestFailure { +export interface TestFailure { id: string; suite: string; title: string; @@ -25,6 +25,7 @@ export declare interface TestFailure { message?: string; stack_trace?: string; }; + stdout?: string; attachments: Array<{ name: string; path?: string; diff --git a/packages/kbn-scout-reporting/src/reporting/report/generic_report.ts b/packages/kbn-scout-reporting/src/reporting/report/generic_report.ts new file mode 100644 index 0000000000000..da73907e6ec9b --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/report/generic_report.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +// eslint-disable-next-line max-classes-per-file +import os from 'node:os'; +import path from 'node:path'; +import fs from 'node:fs'; +import { ToolingLog } from '@kbn/tooling-log'; + +/** + * Generic error raised by a Scout report + */ +export class ScoutReportError extends Error {} + +export abstract class GenericReport { + log: ToolingLog; + workDir: string; + concluded = false; + + constructor(log?: ToolingLog) { + this.log = log || new ToolingLog(); + this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-report-')); + } + + public get eventLogPath(): string { + return path.join(this.workDir, `event-log.ndjson`); + } + + protected raiseIfConcluded(additionalInfo?: string) { + if (this.concluded) { + let message = `Report at ${this.workDir} was concluded`; + + if (additionalInfo) { + message += `: ${additionalInfo}`; + } + + throw new ScoutReportError(message); + } + } + + abstract logEvent(event: any): void; + + /** + * Save the report to a non-ephemeral location + * + * @param destination - Full path to the save location. Must not exist. + */ + abstract save(destination: string): void; + + /** + * Call this when you're done adding information to this report. + * + * ⚠️**This will delete all the contents of the report's working directory** + */ + conclude() { + // Remove the working directory + this.log.info(`Removing Scout report working directory ${this.workDir}`); + fs.rmSync(this.workDir, { recursive: true, force: true }); + + // Mark this report as concluded + this.concluded = true; + this.log.success('Scout report has concluded.'); + } +} diff --git a/packages/kbn-scout-reporting/src/reporting/report/index.ts b/packages/kbn-scout-reporting/src/reporting/report/index.ts index b678463c185f9..091ea2891c323 100644 --- a/packages/kbn-scout-reporting/src/reporting/report/index.ts +++ b/packages/kbn-scout-reporting/src/reporting/report/index.ts @@ -7,97 +7,5 @@ * License v3.0 only", or the "Server Side Public License, v 1". */ -// eslint-disable-next-line max-classes-per-file -import os from 'node:os'; -import path from 'node:path'; -import fs from 'node:fs'; -import { ToolingLog } from '@kbn/tooling-log'; -import { ScoutReportEvent } from './event'; - -/** - * Generic error raised by a Scout report - */ -export class ScoutReportError extends Error {} - -/** - * - */ -export class ScoutReport { - log: ToolingLog; - workDir: string; - concluded = false; - - constructor(log?: ToolingLog) { - this.log = log || new ToolingLog(); - this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-report-')); - } - - public get eventLogPath(): string { - return path.join(this.workDir, `event-log.ndjson`); - } - - private raiseIfConcluded(additionalInfo?: string) { - if (this.concluded) { - let message = `Report at ${this.workDir} was concluded`; - - if (additionalInfo) { - message += `: ${additionalInfo}`; - } - - throw new ScoutReportError(message); - } - } - - /** - * Logs an event to be processed by this reporter - * - * @param event {ScoutReportEvent} - Event to record - */ - logEvent(event: ScoutReportEvent) { - this.raiseIfConcluded('logging new events is no longer allowed'); - - if (event['@timestamp'] === undefined) { - event['@timestamp'] = new Date(); - } - - fs.appendFileSync(this.eventLogPath, JSON.stringify(event) + '\n'); - } - - /** - * Save the report to a non-ephemeral location - * - * @param destination - Full path to the save location. Must not exist. - */ - save(destination: string) { - this.raiseIfConcluded('nothing to save because workdir has been cleared'); - - if (fs.existsSync(destination)) { - throw new ScoutReportError(`Save destination path '${destination}' already exists`); - } - - // Create the destination directory - this.log.info(`Saving Scout report to ${destination}`); - fs.mkdirSync(destination, { recursive: true }); - - // Copy the workdir data to the destination - fs.cpSync(this.workDir, destination, { recursive: true }); - } - - /** - * Call this when you're done adding information to this report. - * - * ⚠️**This will delete all the contents of the report's working directory** - */ - conclude() { - // Remove the working directory - this.log.info(`Removing Scout report working directory ${this.workDir}`); - fs.rmSync(this.workDir, { recursive: true, force: true }); - - // Mark this report as concluded - this.concluded = true; - this.log.success('Scout report has concluded.'); - } -} - -export * from './event'; -export * from './persistence'; +export { ScoutReport, ScoutReportEventAction } from './event'; +export { ScoutFailureReport, TestFailure } from './failed_test'; From 775ca35d68a7a1d46688bdb74183947bcd703861 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Thu, 9 Jan 2025 16:49:12 +0100 Subject: [PATCH 13/21] add unique report names --- .../src/reporting/report/event/report.ts | 7 ++----- .../src/reporting/report/failed_test/failure_report.ts | 7 ++----- .../src/reporting/report/generic_report.ts | 8 +++++--- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/packages/kbn-scout-reporting/src/reporting/report/event/report.ts b/packages/kbn-scout-reporting/src/reporting/report/event/report.ts index 89ce478fdea55..fa2454a4918de 100644 --- a/packages/kbn-scout-reporting/src/reporting/report/event/report.ts +++ b/packages/kbn-scout-reporting/src/reporting/report/event/report.ts @@ -15,14 +15,11 @@ import { ScoutReportEvent } from './event'; import { GenericReport, ScoutReportError } from '../generic_report'; export class ScoutReport extends GenericReport { - log: ToolingLog; - workDir: string; - concluded = false; - constructor(log?: ToolingLog) { super(); this.log = log || new ToolingLog(); this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-report-')); + this.reportName = 'Scout report'; } public get eventLogPath(): string { @@ -57,7 +54,7 @@ export class ScoutReport extends GenericReport { } // Create the destination directory - this.log.info(`Saving Scout report to ${destination}`); + this.log.info(`Saving ${this.reportName} to ${destination}`); fs.mkdirSync(destination, { recursive: true }); // Copy the workdir data to the destination diff --git a/packages/kbn-scout-reporting/src/reporting/report/failed_test/failure_report.ts b/packages/kbn-scout-reporting/src/reporting/report/failed_test/failure_report.ts index e44560d1415c2..802db37517d94 100644 --- a/packages/kbn-scout-reporting/src/reporting/report/failed_test/failure_report.ts +++ b/packages/kbn-scout-reporting/src/reporting/report/failed_test/failure_report.ts @@ -17,14 +17,11 @@ import { GenericReport, ScoutReportError } from '../generic_report'; import { TestFailure } from './test_failure'; export class ScoutFailureReport extends GenericReport { - log: ToolingLog; - workDir: string; - concluded = false; - constructor(log?: ToolingLog) { super(); this.log = log || new ToolingLog(); this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-failures-report-')); + this.reportName = 'Scout Failure report'; } public get testFailuresPath(): string { @@ -63,7 +60,7 @@ export class ScoutFailureReport extends GenericReport { // Create the destination directory this.log.info( - `Saving Scout failures report to ${destination}: ${testFailures.length} failures reported` + `Saving ${this.reportName} to ${destination}: ${testFailures.length} failures reported` ); fs.mkdirSync(destination, { recursive: true }); diff --git a/packages/kbn-scout-reporting/src/reporting/report/generic_report.ts b/packages/kbn-scout-reporting/src/reporting/report/generic_report.ts index da73907e6ec9b..ecfb4a7138645 100644 --- a/packages/kbn-scout-reporting/src/reporting/report/generic_report.ts +++ b/packages/kbn-scout-reporting/src/reporting/report/generic_report.ts @@ -22,10 +22,12 @@ export abstract class GenericReport { log: ToolingLog; workDir: string; concluded = false; + reportName: string; constructor(log?: ToolingLog) { this.log = log || new ToolingLog(); this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-report-')); + this.reportName = 'generic-report'; } public get eventLogPath(): string { @@ -34,7 +36,7 @@ export abstract class GenericReport { protected raiseIfConcluded(additionalInfo?: string) { if (this.concluded) { - let message = `Report at ${this.workDir} was concluded`; + let message = `${this.reportName} at ${this.workDir} was concluded`; if (additionalInfo) { message += `: ${additionalInfo}`; @@ -60,11 +62,11 @@ export abstract class GenericReport { */ conclude() { // Remove the working directory - this.log.info(`Removing Scout report working directory ${this.workDir}`); + this.log.info(`Removing ${this.reportName} working directory ${this.workDir}`); fs.rmSync(this.workDir, { recursive: true, force: true }); // Mark this report as concluded this.concluded = true; - this.log.success('Scout report has concluded.'); + this.log.success(`${this.reportName} has concluded.`); } } From 079ae4e81218dd1bb30c6d0ddca63776454a511f Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Thu, 9 Jan 2025 16:58:34 +0100 Subject: [PATCH 14/21] update report logs --- .../src/reporting/report/failed_test/failure_report.ts | 4 ++-- packages/kbn-scout/src/playwright/config/create_config.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/kbn-scout-reporting/src/reporting/report/failed_test/failure_report.ts b/packages/kbn-scout-reporting/src/reporting/report/failed_test/failure_report.ts index 802db37517d94..67aa2cd3f548c 100644 --- a/packages/kbn-scout-reporting/src/reporting/report/failed_test/failure_report.ts +++ b/packages/kbn-scout-reporting/src/reporting/report/failed_test/failure_report.ts @@ -21,7 +21,7 @@ export class ScoutFailureReport extends GenericReport { super(); this.log = log || new ToolingLog(); this.workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'scout-failures-report-')); - this.reportName = 'Scout Failure report'; + this.reportName = 'Scout Test Failures report'; } public get testFailuresPath(): string { @@ -72,7 +72,7 @@ export class ScoutFailureReport extends GenericReport { htmlReportPath, htmlContent, this.log, - `html report for ${failure.id} is saved at ${htmlReportPath}` + `Html report for ${failure.id} is saved at ${htmlReportPath}` ); } diff --git a/packages/kbn-scout/src/playwright/config/create_config.ts b/packages/kbn-scout/src/playwright/config/create_config.ts index 6fa7ba18fc507..63df28f7a2f89 100644 --- a/packages/kbn-scout/src/playwright/config/create_config.ts +++ b/packages/kbn-scout/src/playwright/config/create_config.ts @@ -26,7 +26,7 @@ export function createPlaywrightConfig(options: ScoutPlaywrightOptions): Playwri /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, /* Retry on CI only */ - retries: 0, // process.env.CI ? 2 : 0, + retries: 0, // disable retry for Playwright runner /* Opt out of parallel tests on CI. */ workers: options.workers ?? 1, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ From 1b62a4cb8a00a01f5358128085ad71953ca512be Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Thu, 9 Jan 2025 17:23:45 +0100 Subject: [PATCH 15/21] update failed test reporter --- .../failed_tests_reporter_cli.ts | 56 +------------ .../generate_scout_test_failure_artifacts.ts | 79 +++++++++++++++++++ .../src/reporting/report/failed_test/index.ts | 2 +- .../src/reporting/report/index.ts | 2 +- 4 files changed, 83 insertions(+), 56 deletions(-) create mode 100644 packages/kbn-failed-test-reporter-cli/failed_tests_reporter/generate_scout_test_failure_artifacts.ts diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts index 0d56da397f432..981b768588c28 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -15,9 +15,6 @@ import { createFailError, createFlagError } from '@kbn/dev-cli-errors'; import { CiStatsReporter } from '@kbn/ci-stats-reporter'; import globby from 'globby'; import normalize from 'normalize-path'; -import fs from 'fs'; -import { createHash } from 'crypto'; - import { getFailures } from './get_failures'; import { GithubApi } from './github_api'; import { updateFailureIssue, createFailureIssue } from './report_failure'; @@ -28,6 +25,7 @@ import { reportFailuresToEs } from './report_failures_to_es'; import { reportFailuresToFile } from './report_failures_to_file'; import { getBuildkiteMetadata } from './buildkite_metadata'; import { ExistingFailedTestIssues } from './existing_failed_test_issues'; +import { generateScoutTestFailureArtifacts } from './generate_scout_test_failure_artifacts'; const DEFAULT_PATTERNS = [Path.resolve(REPO_ROOT, 'target/junit/**/*.xml')]; const DISABLE_MISSING_TEST_REPORT_ERRORS = @@ -186,57 +184,7 @@ run( } // Scout test failures reporting - log.info('Searching for Scout test failure reports'); - const scoutTestFailuresDirPattern = '.scout/reports/scout-playwright-test-failures-*'; - - const dirs = await globby(scoutTestFailuresDirPattern, { - onlyDirectories: true, - }); - - if (dirs.length === 0) { - throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`); - // log.info(`No directories found matching pattern: ${scoutTestFailuresDirPattern}`); - } - - const dirPath = dirs[0]; // temp take the last one - const summaryFilePath = Path.join(dirPath, 'test-failures-summary.json'); - - // Check if summary JSON exists - if (!fs.existsSync(summaryFilePath)) { - throw new Error(`Summary file not found in: ${dirPath}`); - } - - const summaryData: Array<{ name: string; htmlReportFilename: string }> = JSON.parse( - fs.readFileSync(summaryFilePath, 'utf-8') - ); - - log.info('Creating failure artifacts for', summaryData.length, 'test failures'); - for (const { name, htmlReportFilename } of summaryData) { - const htmlFilePath = Path.join(dirPath, htmlReportFilename); - const failureHTML = fs.readFileSync(htmlFilePath, 'utf-8'); - - const hash = createHash('md5').update(name).digest('hex'); // eslint-disable-line @kbn/eslint/no_unsafe_hash - const filenameBase = `${ - process.env.BUILDKITE_JOB_ID ? process.env.BUILDKITE_JOB_ID + '_' : '' - }${hash}`; - const dir = Path.join('target', 'test_failures'); - const failureJSON = JSON.stringify( - { - name, - hash, - buildId: bkMeta.buildId, - jobId: bkMeta.jobId, - url: bkMeta.url, - jobUrl: bkMeta.jobUrl, - jobName: bkMeta.jobName, - }, - null, - 2 - ); - fs.mkdirSync(dir, { recursive: true }); - fs.writeFileSync(Path.join(dir, `${filenameBase}.html`), failureHTML, 'utf8'); - fs.writeFileSync(Path.join(dir, `${filenameBase}.json`), failureJSON, 'utf8'); - } + await generateScoutTestFailureArtifacts({ log, bkMeta }); } finally { await CiStatsReporter.fromEnv(log).metrics([ { diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/generate_scout_test_failure_artifacts.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/generate_scout_test_failure_artifacts.ts new file mode 100644 index 0000000000000..b4641225af5c3 --- /dev/null +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/generate_scout_test_failure_artifacts.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import Path from 'path'; + +import globby from 'globby'; +import fs from 'fs'; +import { createHash } from 'crypto'; +import { ToolingLog } from '@kbn/tooling-log'; +import { BuildkiteMetadata } from './buildkite_metadata'; + +const SCOUT_TEST_FAILURE_DIR_PATTERN = '.scout/reports/scout-playwright-test-failures-*'; +const SUMMARY_REPORT_FILENAME = 'test-failures-summary.json'; + +export async function generateScoutTestFailureArtifacts({ + log, + bkMeta, +}: { + log: ToolingLog; + bkMeta: BuildkiteMetadata; +}) { + log.info('Searching for Scout test failure reports'); + + const dirs = await globby(SCOUT_TEST_FAILURE_DIR_PATTERN, { + onlyDirectories: true, + }); + + if (dirs.length === 0) { + log.info(`No directories found matching pattern: ${SCOUT_TEST_FAILURE_DIR_PATTERN}`); + return; + } + + log.info(`Found ${dirs.length} directories matching pattern: ${SCOUT_TEST_FAILURE_DIR_PATTERN}`); + for (const dirPath of dirs) { + const summaryFilePath = Path.join(dirPath, SUMMARY_REPORT_FILENAME); + // Check if summary JSON exists + if (!fs.existsSync(summaryFilePath)) { + throw new Error(`Summary file not found in: ${dirPath}`); + } + + const summaryData: Array<{ name: string; htmlReportFilename: string }> = JSON.parse( + fs.readFileSync(summaryFilePath, 'utf-8') + ); + + log.info(`Creating failure artifacts for report in ${dirPath}`); + for (const { name, htmlReportFilename } of summaryData) { + const htmlFilePath = Path.join(dirPath, htmlReportFilename); + const failureHTML = fs.readFileSync(htmlFilePath, 'utf-8'); + + const hash = createHash('md5').update(name).digest('hex'); // eslint-disable-line @kbn/eslint/no_unsafe_hash + const filenameBase = `${ + process.env.BUILDKITE_JOB_ID ? process.env.BUILDKITE_JOB_ID + '_' : '' + }${hash}`; + const dir = Path.join('target', 'test_failures'); + const failureJSON = JSON.stringify( + { + name, + hash, + buildId: bkMeta.buildId, + jobId: bkMeta.jobId, + url: bkMeta.url, + jobUrl: bkMeta.jobUrl, + jobName: bkMeta.jobName, + }, + null, + 2 + ); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(Path.join(dir, `${filenameBase}.html`), failureHTML, 'utf8'); + fs.writeFileSync(Path.join(dir, `${filenameBase}.json`), failureJSON, 'utf8'); + } + } +} diff --git a/packages/kbn-scout-reporting/src/reporting/report/failed_test/index.ts b/packages/kbn-scout-reporting/src/reporting/report/failed_test/index.ts index ce868d1d9b8c3..d10adbf9d3ce5 100644 --- a/packages/kbn-scout-reporting/src/reporting/report/failed_test/index.ts +++ b/packages/kbn-scout-reporting/src/reporting/report/failed_test/index.ts @@ -8,4 +8,4 @@ */ export { ScoutFailureReport } from './failure_report'; -export { TestFailure } from './test_failure'; +export type { TestFailure } from './test_failure'; diff --git a/packages/kbn-scout-reporting/src/reporting/report/index.ts b/packages/kbn-scout-reporting/src/reporting/report/index.ts index 091ea2891c323..f81fb2cacc325 100644 --- a/packages/kbn-scout-reporting/src/reporting/report/index.ts +++ b/packages/kbn-scout-reporting/src/reporting/report/index.ts @@ -8,4 +8,4 @@ */ export { ScoutReport, ScoutReportEventAction } from './event'; -export { ScoutFailureReport, TestFailure } from './failed_test'; +export { ScoutFailureReport, type TestFailure } from './failed_test'; From e7ec27ee565dce1bfd3326bd5cbd9c64a8ae06b0 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Thu, 9 Jan 2025 17:41:25 +0100 Subject: [PATCH 16/21] break FTR test, fix export --- .../failed_tests_reporter/failed_tests_reporter_cli.ts | 8 +++++--- packages/kbn-scout-reporting/src/reporting/index.ts | 2 +- .../test_suites/search/dashboards/build_dashboard.ts | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts index 981b768588c28..159b8995b8e36 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -101,6 +101,9 @@ run( return; } + // Scout test failures reporting + await generateScoutTestFailureArtifacts({ log, bkMeta }); + if (reportPaths.length) { log.info('found', reportPaths.length, 'junit reports', reportPaths); @@ -181,10 +184,9 @@ run( await reportFailuresToFile(log, failures, bkMeta, getRootMetadata(report)); } + } else { + throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`); } - - // Scout test failures reporting - await generateScoutTestFailureArtifacts({ log, bkMeta }); } finally { await CiStatsReporter.fromEnv(log).metrics([ { diff --git a/packages/kbn-scout-reporting/src/reporting/index.ts b/packages/kbn-scout-reporting/src/reporting/index.ts index 28605c5452e7d..78eebf6bc2223 100644 --- a/packages/kbn-scout-reporting/src/reporting/index.ts +++ b/packages/kbn-scout-reporting/src/reporting/index.ts @@ -26,4 +26,4 @@ export const scoutFailedTestsReporter = ( return ['@kbn/scout-reporting/src/reporting/playwright/failed_test/index.ts', options]; }; -export { generateTestRunId } from './playwright/common'; +export { generateTestRunId, getTestIDForTitle } from './playwright/common'; diff --git a/x-pack/test_serverless/functional/test_suites/search/dashboards/build_dashboard.ts b/x-pack/test_serverless/functional/test_suites/search/dashboards/build_dashboard.ts index aefd4c6da9832..f666516ba4dfc 100644 --- a/x-pack/test_serverless/functional/test_suites/search/dashboards/build_dashboard.ts +++ b/x-pack/test_serverless/functional/test_suites/search/dashboards/build_dashboard.ts @@ -62,7 +62,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { await PageObjects.dashboard.waitForRenderComplete(); const partitionVisExists = await testSubjects.exists('partitionVisChart'); - expect(partitionVisExists).to.be(true); + expect(partitionVisExists).to.be(false); }); it('can add a filter pill by clicking on the Lens chart', async () => { From a318a9efb4521dd668e8baa9c8abdd489e00d389 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Thu, 9 Jan 2025 17:59:17 +0100 Subject: [PATCH 17/21] use stripFilePath for location --- .../reporting/playwright/failed_test/failed_test_reporter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts index b1a56751f9baa..ab1a26faeddab 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts @@ -98,7 +98,7 @@ export class ScoutFailedTestReporter implements Reporter { title: test.title, target: this.target, command: this.command, - location: test.location.file.replace(`${REPO_ROOT}/`, ''), + location: stripFilePath(test.location.file), owner: this.getFileOwners(path.relative(REPO_ROOT, test.location.file)), plugin: this.plugin, duration: result.duration, From 54c9b5e8373bff6bc4f83b27de5cec882c3f877e Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Thu, 9 Jan 2025 18:25:54 +0100 Subject: [PATCH 18/21] no error if no FTR failures --- .../failed_tests_reporter/failed_tests_reporter_cli.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts index 159b8995b8e36..9cf108bb6bc7e 100644 --- a/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts +++ b/packages/kbn-failed-test-reporter-cli/failed_tests_reporter/failed_tests_reporter_cli.ts @@ -184,8 +184,6 @@ run( await reportFailuresToFile(log, failures, bkMeta, getRootMetadata(report)); } - } else { - throw createFailError(`Unable to find any junit reports with patterns [${patterns}]`); } } finally { await CiStatsReporter.fromEnv(log).metrics([ From 90fbec9900d5c20fb4891045f98809067db3265f Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Thu, 9 Jan 2025 20:33:20 +0100 Subject: [PATCH 19/21] fix create_config test --- .../playwright/config/create_config.test.ts | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/kbn-scout/src/playwright/config/create_config.test.ts b/packages/kbn-scout/src/playwright/config/create_config.test.ts index 730bdd5ef55e4..3fddefb02a1a2 100644 --- a/packages/kbn-scout/src/playwright/config/create_config.test.ts +++ b/packages/kbn-scout/src/playwright/config/create_config.test.ts @@ -10,9 +10,21 @@ import { SCOUT_SERVERS_ROOT } from '@kbn/scout-info'; import { createPlaywrightConfig } from './create_config'; import { VALID_CONFIG_MARKER } from '../types'; +import { generateTestRunId } from '@kbn/scout-reporting'; + +jest.mock('@kbn/scout-reporting', () => ({ + ...jest.requireActual('@kbn/scout-reporting'), + generateTestRunId: jest.fn(), +})); describe('createPlaywrightConfig', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + it('should return a valid default Playwright configuration', () => { + (generateTestRunId as jest.Mock).mockReturnValue('test-run-id'); + const testDir = './my_tests'; const config = createPlaywrightConfig({ testDir }); @@ -30,7 +42,14 @@ describe('createPlaywrightConfig', () => { expect(config.reporter).toEqual([ ['html', { open: 'never', outputFolder: './output/reports' }], ['json', { outputFile: './output/reports/test-results.json' }], - ['@kbn/scout-reporting/src/reporting/playwright.ts', { name: 'scout-playwright' }], + [ + '@kbn/scout-reporting/src/reporting/playwright/events/index.ts', + { name: 'scout-playwright', runId: 'test-run-id' }, + ], + [ + '@kbn/scout-reporting/src/reporting/playwright/failed_test/index.ts', + { name: 'scout-playwright-failed-tests', runId: 'test-run-id' }, + ], ]); expect(config.timeout).toBe(60000); expect(config.expect?.timeout).toBe(10000); From 8ee2c8438c0d5b1fd918374aa8ab288a0dcbf5af Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Fri, 10 Jan 2025 09:22:32 +0100 Subject: [PATCH 20/21] add more unit tests --- .../common/get_plugin_manifest_data.test.ts | 126 ++++++++++++++++++ .../playwright/common/get_run_target.test.ts | 6 +- .../playwright/common/strip_utils.test.ts | 58 ++++++++ .../failed_test/failed_test_reporter.ts | 5 +- 4 files changed, 188 insertions(+), 7 deletions(-) create mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/common/get_plugin_manifest_data.test.ts create mode 100644 packages/kbn-scout-reporting/src/reporting/playwright/common/strip_utils.test.ts diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/common/get_plugin_manifest_data.test.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/get_plugin_manifest_data.test.ts new file mode 100644 index 0000000000000..ae01df589eef5 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/get_plugin_manifest_data.test.ts @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import path from 'path'; +import fs from 'fs'; +import { + getManifestPath, + readPluginManifest, + getPluginManifestData, +} from './get_plugin_manifest_data'; + +jest.mock('fs'); + +describe('get_plugin_manufest_data', () => { + describe('getManifestPath', () => { + it('should resolve the manifest path correctly for a valid config path', () => { + const configPath = '/plugins/my_plugin/ui_tests/playwright.config.ts'; + const expectedPath = path.resolve('/plugins/my_plugin/kibana.jsonc'); + expect(getManifestPath(configPath)).toBe(expectedPath); + }); + + it(`should throw an error if 'ui_tests' is not in the path`, () => { + const configPath = '/plugins/my_plugin/tests/playwright.config.ts'; + expect(() => getManifestPath(configPath)).toThrow( + /Invalid path: "ui_tests" directory not found/ + ); + }); + }); + + describe('readPluginManifest', () => { + const filePath = '/plugins/my_plugin/kibana.jsonc'; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should read and parse the manifest file correctly', () => { + const fileContent = ` + { + "id": "my_plugin", + "group": "platform", + "visibility": "private", + "owner": ["team"], + "plugin": { "id": "my_plugin" } + } + `; + jest.spyOn(fs, 'existsSync').mockReturnValue(true); + jest.spyOn(fs, 'readFileSync').mockReturnValue(fileContent); + + const result = readPluginManifest(filePath); + expect(result).toEqual({ + id: 'my_plugin', + group: 'platform', + visibility: 'private', + owner: ['team'], + plugin: { id: 'my_plugin' }, + }); + }); + + it('should throw an error if the file does not exist', () => { + jest.spyOn(fs, 'existsSync').mockReturnValue(false); + expect(() => readPluginManifest(filePath)).toThrow(/Manifest file not found/); + }); + + it('should throw an error if the file cannot be read', () => { + jest.spyOn(fs, 'existsSync').mockReturnValue(true); + jest.spyOn(fs, 'readFileSync').mockImplementation(() => { + throw new Error('File read error'); + }); + expect(() => readPluginManifest(filePath)).toThrow(/Failed to read manifest file/); + }); + + it('should throw an error for invalid JSON content', () => { + jest.spyOn(fs, 'existsSync').mockReturnValue(true); + jest.spyOn(fs, 'readFileSync').mockReturnValue('{ invalid json }'); + expect(() => readPluginManifest(filePath)).toThrow(/Invalid JSON format in manifest file/); + }); + + it('should throw an error for missing required fields', () => { + const fileContent = `{ + "group": "platform", + "visibility": "public" + }`; + jest.spyOn(fs, 'existsSync').mockReturnValue(true); + jest.spyOn(fs, 'readFileSync').mockReturnValue(fileContent); + expect(() => readPluginManifest(filePath)).toThrow(/Invalid manifest structure/); + }); + }); + + describe('getPluginManifestData', () => { + const configPath = '/plugins/my_plugin/ui_tests/playwright.config.ts'; + const manifestContent = ` + { + "id": "my_plugin", + "group": "platform", + "visibility": "public", + "owner": ["team"], + "plugin": { "id": "my_plugin" } + } + `; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should resolve and parse the manifest data correctly', () => { + jest.spyOn(fs, 'existsSync').mockReturnValue(true); + jest.spyOn(fs, 'readFileSync').mockReturnValue(manifestContent); + + const result = getPluginManifestData(configPath); + expect(result).toEqual({ + id: 'my_plugin', + group: 'platform', + visibility: 'public', + owner: ['team'], + plugin: { id: 'my_plugin' }, + }); + }); + }); +}); diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/common/get_run_target.test.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/get_run_target.test.ts index 588758ab3ced7..af44837ce3c41 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/common/get_run_target.test.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/get_run_target.test.ts @@ -52,16 +52,12 @@ describe('getRunTarget', () => { expect(getRunTarget(argv)).toBe('undefined'); }); - it(`should default to 'undefined' if argv is not passed`, () => { - expect(getRunTarget()).toBe('undefined'); - }); - it(`should return 'undefined' for '--grep='`, () => { const argv = ['node', 'scripts/scout.js', '--grep=']; expect(getRunTarget(argv)).toBe('undefined'); }); - it(`should return "undefined" if "--grep" argument is without value`, () => { + it(`should return 'undefined' if '--grep' argument is without value`, () => { const argv = ['node', 'scripts/scout.js', '--grep']; expect(getRunTarget(argv)).toBe('undefined'); }); diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/common/strip_utils.test.ts b/packages/kbn-scout-reporting/src/reporting/playwright/common/strip_utils.test.ts new file mode 100644 index 0000000000000..9d353e433d100 --- /dev/null +++ b/packages/kbn-scout-reporting/src/reporting/playwright/common/strip_utils.test.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { stripRunCommand } from './strip_utils'; + +describe('stripRunCommand', () => { + it(`should return the correct run command when started with 'npx'`, () => { + const argv = ['npx', 'playwright', 'test', '--config', 'path/to/config', '--grep=@svlSearch']; + + expect(stripRunCommand(argv)).toBe( + 'npx playwright test --config path/to/config --grep=@svlSearch' + ); + }); + + it(`should return the correct run command when started with 'node'`, () => { + const argv = [ + '/Users/user/.nvm/versions/node/v20.15.1/bin/node', + 'node_modules/.bin/playwright', + 'test', + '--config', + 'path/to/config', + '--grep=@svlSearch', + ]; + + expect(stripRunCommand(argv)).toBe( + 'npx playwright test --config path/to/config --grep=@svlSearch' + ); + }); + + it(`should throw error if command has less than 3 arguments`, () => { + const argv = [ + '/Users/user/.nvm/versions/node/v20.15.1/bin/node', + 'node_modules/.bin/playwright', + ]; + expect(() => stripRunCommand(argv)).toThrow( + /Invalid command arguments: must include at least 'npx playwright test'/ + ); + }); + + it(`should throw error if command does not start with 'node' or 'npx'`, () => { + const argv = [ + 'node_modules/.bin/playwright', + 'test', + '--config', + 'path/to/config', + '--grep=@svlSearch', + ]; + expect(() => stripRunCommand(argv)).toThrow( + /Invalid command structure: Expected "node test" or "npx playwright test"/ + ); + }); +}); diff --git a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts index ab1a26faeddab..e313786d1b6b9 100644 --- a/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts +++ b/packages/kbn-scout-reporting/src/reporting/playwright/failed_test/failed_test_reporter.ts @@ -26,15 +26,16 @@ import { getOwningTeamsForPath, } from '@kbn/code-owners'; import type { TestFailure } from '../../report'; -import type { ScoutPlaywrightReporterOptions } from '../../scout_playwright_reporter'; -import { stripRunCommand, stripFilePath } from '../common/strip_utils'; import { ScoutFailureReport } from '../../report'; +import type { ScoutPlaywrightReporterOptions } from '../../scout_playwright_reporter'; import { getRunTarget, getPluginManifestData, parseStdout, generateTestRunId, getTestIDForTitle, + stripRunCommand, + stripFilePath, } from '../common'; /** From 9c34a64941fb757097c4677480e7eb2c405e5296 Mon Sep 17 00:00:00 2001 From: Dzmitry Lemechko Date: Fri, 10 Jan 2025 17:54:57 +0100 Subject: [PATCH 21/21] revert test changes --- .../ui_tests/tests/value_suggestions.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts b/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts index aeb57eeedbf04..40b07726b31e4 100644 --- a/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts +++ b/x-pack/platform/plugins/private/discover_enhanced/ui_tests/tests/value_suggestions.spec.ts @@ -34,7 +34,7 @@ test.describe( test('dont show up if outside of range', async ({ page, pageObjects }) => { await pageObjects.datePicker.setAbsoluteRange(testData.LOGSTASH_OUT_OF_RANGE_DATES); await page.testSubj.fill('queryInput', 'extension.raw : '); - await expect(page.testSubj.locator('autoCompleteSuggestionText')).toHaveCount(4); + await expect(page.testSubj.locator('autoCompleteSuggestionText')).toHaveCount(0); }); test('show up if in range', async ({ page, pageObjects }) => { @@ -47,7 +47,7 @@ test.describe( const actualSuggestions = await page.testSubj .locator('autoCompleteSuggestionText') .allTextContents(); - expect(actualSuggestions.join(',')).toContain('xyz'); + expect(actualSuggestions.join(',')).toContain('jpg'); }); test('also displays descriptions for operators', async ({ page, pageObjects }) => {