import { format } from 'node:util' import prettyMs from 'pretty-ms' import type { Capabilities } from '@wdio/types' import { Chalk, type ChalkInstance } from 'chalk' import WDIOReporter, { TestStats, getBrowserName } from '@wdio/reporter' import type { SuiteStats, HookStats, RunnerStats, Argument } from '@wdio/reporter' import { buildTableData, printTable, getFormattedRows, sauceAuthenticationToken } from './utils.js' import { ChalkColors, type SpecReporterOptions, type TestLink, type StateCount, type Symbols, State } from './types.js' const DEFAULT_INDENT = ' ' interface CapabilitiesWithSessionId extends WebdriverIO.Capabilities { sessionId: string } export default class SpecReporter extends WDIOReporter { private _suiteUids = new Set() private _indents = 0 private _suiteIndents: Record = {} private _orderedSuites: SuiteStats[] = [] private _consoleOutput = '' private _suiteIndent = '' private _preface = '' private _consoleLogs: string[] = [] private _pendingReasons: string[] = [] private _originalStdoutWrite = process.stdout.write.bind(process.stdout) private _addConsoleLogs = false private _realtimeReporting = false private _showPreface = true private _suiteName = '' private _isSuiteRetry = false private _passingTestsSinceLastRetry = 0 // Keep track of the order that suites were called private _stateCounts: StateCount = { passed: 0, failed: 0, skipped: 0, pending: 0, retried: 0 } private _symbols: Symbols = { passed: '✓', skipped: '-', pending: '?', failed: '✖', retried: '↻' } private _chalk: ChalkInstance private _onlyFailures = false private _sauceLabsSharableLinks = true constructor (options: SpecReporterOptions) { /** * make spec reporter to write to output stream by default */ super(Object.assign({ stdout: true } as Record, options)) this._symbols = { ...this._symbols, ...this.options.symbols || {} } this._onlyFailures = options.onlyFailures || false this._realtimeReporting = options.realtimeReporting || false this._showPreface = options.showPreface !== false this._sauceLabsSharableLinks = 'sauceLabsSharableLinks' in options ? options.sauceLabsSharableLinks as boolean : this._sauceLabsSharableLinks // eslint-disable-next-line @typescript-eslint/no-explicit-any const processObj = process as any if (options.addConsoleLogs || this._addConsoleLogs) { processObj.stdout.write = (chunk: string, encoding: BufferEncoding, callback: ((err?: Error) => void)) => { if (typeof chunk === 'string' && !chunk.includes('mwebdriver')) { this._consoleOutput += chunk } return this._originalStdoutWrite(chunk, encoding, callback) } } this._chalk = new Chalk(options.color === false ? { level:0 } : {}) } /** * @param state state of test execution * @param msg the message to print in terminal * @returns colord value based on chalk to print in terminal */ setMessageColor(message:string, state?: State): string{ return this._chalk[this.getColor(state)](message) } onRunnerStart (runner: RunnerStats) { this._preface = this._showPreface ? `[${this.getEnviromentCombo(runner.capabilities, false, runner.isMultiremote).trim()} #${runner.cid}]` : '' } onSuiteStart (suite: SuiteStats) { this._suiteName = suite.file?.replace(process.cwd(), '') this.printCurrentStats(suite) this._suiteUids.add(suite.uid) if (suite.type === 'feature') { this._indents = 0 this._suiteIndents[suite.uid] = this._indents } else { this._suiteIndents[suite.uid] = ++this._indents } } onSuiteEnd () { this._indents-- this._isSuiteRetry = false this._passingTestsSinceLastRetry = 0 } onSuiteRetry(): void { this._stateCounts.failed-- this._stateCounts.retried++ this._stateCounts.passed -= this._passingTestsSinceLastRetry this._isSuiteRetry = true this._passingTestsSinceLastRetry = 0 } onHookEnd (hook: HookStats) { this.printCurrentStats(hook) if (hook.error) { this._stateCounts.failed++ } } onTestStart () { this._consoleOutput = '' } onTestPass (testStat: TestStats) { this.printCurrentStats(testStat) this._consoleLogs.push(this._consoleOutput) this._stateCounts.passed++ if (!this._isSuiteRetry) { this._passingTestsSinceLastRetry++ } } onTestFail (testStat: TestStats) { this.printCurrentStats(testStat) this._consoleLogs.push(this._consoleOutput) this._stateCounts.failed++ } onTestSkip (testStat: TestStats) { this.printCurrentStats(testStat) this._pendingReasons.push(testStat.pendingReason as string) this._consoleLogs.push(this._consoleOutput) this._stateCounts.skipped++ } onTestPending(testStat: TestStats) { this.printCurrentStats(testStat) this._pendingReasons.push(testStat.pendingReason as string) this._consoleLogs.push(this._consoleOutput) this._stateCounts.pending++ } onRunnerEnd (runner: RunnerStats) { this.printReport(runner) } /** * Print the report to the stdout realtime */ printCurrentStats (stat: TestStats | HookStats | SuiteStats) { if (!this._realtimeReporting) { return } const title = stat.title, state = (stat as TestStats).state as State const divider = '------------------------------------------------------------------' const indent = (stat.type==='test') ? `${DEFAULT_INDENT}${this._suiteIndent}` : this.indent(stat.uid) const suiteStartBanner = (stat.type === 'feature' || stat.type === 'suite' || stat.type === 'suite:start') ? `${this._preface} ${divider}\n`+ `${this._preface} Suite started: \n`+ `${this._preface} » ${this._suiteName}\n` : '\n' const content = stat.type === 'test' ? `${this._preface} ${indent}` + `${this.setMessageColor(this.getSymbol(state), state)} ${title}` + ` » ${this.setMessageColor('[', state)} ${this._suiteName} ${this.setMessageColor(']', state)}` : stat.type !== 'hook' ? `${suiteStartBanner}${this._preface} ${title}` : title ? `${this._preface} Hook executed: ${title}` : undefined /** * only send event upstream, * - if we are in a child process * - there is content to send * - we are not running a unit test */ if (process.send && content && !process.env.WDIO_UNIT_TESTS) { process.send({ name: 'reporterRealTime', content }) } } /** * Print the report to the screen */ printReport(runner: RunnerStats) { // Don't print non failed tests if (runner.failures === 0 && this._onlyFailures === true) { return } const duration = `(${prettyMs(runner._duration)})` const preface = `[${this.getEnviromentCombo(runner.capabilities, false, runner.isMultiremote).trim()} #${runner.cid}]` const divider = '------------------------------------------------------------------' // Get the results const results = this.getResultDisplay(preface) if (results.length === 0 && runner.error) { results.push( this.setMessageColor(`${this.getSymbol(State.FAILED)} Failed to create a session:`, State.FAILED), runner.error, '' ) } // If there are no test results then return nothing if (results.length === 0) { return } const testLinks = runner.isMultiremote ? Object.entries(runner.capabilities).map(([instanceName, capabilities]: [string, CapabilitiesWithSessionId]) => this.getTestLink({ capabilities, sessionId: capabilities.sessionId, isMultiremote: runner.isMultiremote, instanceName })).filter((links) => links.length) : this.getTestLink(runner) const output = [ ...this.getHeaderDisplay(runner), '', ...results, ...this.getCountDisplay(duration), ...this.getFailureDisplay(), ...(testLinks.length /** * if we have test links add an empty line */ ? ['', ...testLinks] : [] ) ] // Prefix all values with the browser information const prefacedOutput = this._showPreface ? output.map((value) => { return value ? `${preface} ${value}` : preface }) : output // Output the results this.write(`${divider}\n${prefacedOutput.join('\n')}\n`) } /** * get link to saucelabs job */ getTestLink ({ sessionId, isMultiremote, instanceName, capabilities }: TestLink) { const config = this.runnerStat && this.runnerStat.instanceOptions[sessionId] const isSauceJob = ( (config && config.hostname && config.hostname.includes('saucelabs')) || // only show if multiremote is not used capabilities['sauce:options'] ) if (isSauceJob && config && config.user && config.key && sessionId) { const multiremoteNote = isMultiremote ? ` ${instanceName}` : '' const note = 'Check out%s job at %s' // The report url of RDC is in the caps that are returned if ('testobject_test_report_url' in capabilities){ return [format(note, multiremoteNote, capabilities.testobject_test_report_url)] } // VDC urls can be constructed / be made shared const isUSEast4 = ['us-east-4'].includes(config?.region || '') || (config.hostname?.includes('us-east-4')) const isEUCentral = ['eu', 'eu-central-1'].includes(config?.region || '') || (config.hostname?.includes('eu-central')) const dc = isUSEast4 ? '.us-east-4' : isEUCentral ? '.eu-central-1' : '' const sauceLabsSharableLinks = this._sauceLabsSharableLinks ? sauceAuthenticationToken( config.user, config.key, sessionId ) : '' const sauceUrl = `https://app${dc}.saucelabs.com/tests/${sessionId}${sauceLabsSharableLinks}` return [format(note, multiremoteNote, sauceUrl)] } return [] } /** * Get the header display for the report * @param {Object} runner Runner data * @return {Array} Header data */ getHeaderDisplay (runner: RunnerStats) { const combo = this.getEnviromentCombo(runner.capabilities, undefined, runner.isMultiremote).trim() // Spec file name and enviroment information const output = [`Running: ${combo}`] /** * print session ID if not multiremote */ // @ts-expect-error if (runner.capabilities.sessionId) { // @ts-expect-error output.push(`Session ID: ${runner.capabilities.sessionId}`) } return output } /** * returns everything worth reporting from a suite * @param {Object} suite test suite containing tests and hooks * @return {Object[]} list of events to report */ getEventsToReport (suite: SuiteStats) { return [ /** * Generate a report that shows all tests except those that failed but passed on retry, and only display failed hooks. */ ...suite.hooksAndTests.reduce((accumulator, currentItem) => { if (currentItem instanceof TestStats) { const existingTestIndex = accumulator.findIndex((test) => test instanceof TestStats && test.fullTitle === currentItem.fullTitle) if (existingTestIndex === -1) { accumulator.push(currentItem) } else { const existingTest = accumulator[existingTestIndex] as TestStats if (currentItem.retries !== undefined && existingTest.retries !== undefined) { if (currentItem.retries > existingTest.retries) { accumulator.splice(existingTestIndex, 1, currentItem) } else { accumulator.push(currentItem) } } } } else { accumulator.push(currentItem) } return accumulator }, [] as (HookStats | TestStats)[]).filter((item) => Object.keys(item).length > 0).filter((item) => { return item.type === 'test' || Boolean(item.error) }) ] } /** * Get the results from the tests * @param {Array} suites Runner suites * @return {Array} Display output list */ getResultDisplay (prefaceString?: string) { const output = [] const preface = this._showPreface ? prefaceString : '' const suites = this.getOrderedSuites() const specFileReferences: string[] = [] for (const suite of suites) { // Don't do anything if a suite has no tests or sub suites if (suite.tests.length === 0 && suite.suites.length === 0 && suite.hooks.length === 0) { continue } // Get the indent/starting point for this suite const suiteIndent = this.indent(suite.uid) // Display file path of spec if (suite.file && !specFileReferences.includes(suite.file)) { output.push(`${suiteIndent}» ${suite.file.replace(process.cwd(), '').slice(1)}`) specFileReferences.push(suite.file) } let retryAnnotation = '' if (suite.retries > 0) { retryAnnotation = this._chalk.yellow(` (${suite.retries}x retries)`) } // Display the title of the suite output.push(`${suiteIndent}${suite.title}${retryAnnotation}`) // display suite description (Cucumber only) if (suite.description) { output.push(...suite.description.trim().split('\n') .map((l) => `${suiteIndent}${this.setMessageColor(l.trim())}`)) output.push('') // empty line } // display suite rule (Cucumber only) if (suite.rule) { output.push(...suite.rule.trim().split('\n') .map((l) => `${suiteIndent}${this.setMessageColor(l.trim())}`)) } const eventsToReport = this.getEventsToReport(suite) for (const test of eventsToReport) { const testRetryAnnotation = (test instanceof TestStats && test.retries && test.retries > 0) ? this._chalk.yellow(`(${test.retries}x retries)`) : '' const testTitle = `${test.title} ${testRetryAnnotation}` const state = test.state as State const testIndent = `${DEFAULT_INDENT}${suiteIndent}` // Output for a single test output.push(`${testIndent}${this.setMessageColor(this.getSymbol(state), state)} ${testTitle.trim()}`) // print cucumber data table cells and docstring const arg = (test as TestStats).argument if (typeof(arg) === 'string'){ // Doc string is the same with the indent on the output for a single test const docstringIndent = ' ' const docstringMark = `${testIndent}${docstringIndent}"""` const docstring = String(arg) const formattedDocstringLines = docstring.split('\n').filter(line => line) .map((line: string) => `${testIndent}${docstringIndent}${line}`) output.push(...[docstringMark, ...formattedDocstringLines, docstringMark]) } else { const dataTable = arg as Argument if (dataTable && dataTable.rows && dataTable.rows.length) { const data = buildTableData(dataTable.rows) const rawTable = printTable(data) const table = getFormattedRows(rawTable, testIndent) output.push(...table) } } // print pending reasons const pendingItem = this._pendingReasons.shift() if (pendingItem) { output.push('') output.push(testIndent.repeat(2) + '.........Pending Reasons.........') output.push(testIndent.repeat(3) + pendingItem?.replace(/\n/g, '\n'.concat(preface + ' ', testIndent.repeat(3)))) } // print console output const logItem = this._consoleLogs.shift() if (logItem) { output.push('') output.push(testIndent.repeat(2) + '.........Console Logs.........') output.push(testIndent.repeat(3) + logItem?.replace(/\n/g, '\n'.concat(preface + ' ', testIndent.repeat(3)))) } } // Put a line break after each suite (only if tests exist in that suite) if (eventsToReport.length) { output.push('') } } return output } /** * Get the display for passing, failing and skipped * @param {string} duration Duration string * @return {Array} Count display */ getCountDisplay (duration: string) { const output: string[] = [] // Get the passes if (this._stateCounts.passed > 0) { const text = `${this._stateCounts.passed} passing ${duration}` output.push(this.setMessageColor(text, State.PASSED)) duration = '' } // Get the failures if (this._stateCounts.failed > 0) { const text = `${this._stateCounts.failed} failing ${duration}`.trim() output.push(this.setMessageColor(text, State.FAILED)) duration = '' } // Get the skipped tests if (this._stateCounts.skipped > 0) { const text = `${this._stateCounts.skipped} skipped ${duration}`.trim() output.push(this.setMessageColor(text, State.SKIPPED)) } // Get the pending tests if (this._stateCounts.pending > 0) { const text = `${this._stateCounts.pending} pending ${duration}`.trim() output.push(this.setMessageColor(text, State.PENDING)) duration = '' } // Get the skipped tests if (this._stateCounts.retried > 0) { const text = `${this._stateCounts.retried} retried ${duration}`.trim() output.push(this.setMessageColor(text, State.RETRIED)) } return output } /** * Get display for failed tests, e.g. stack trace * @return {Array} Stack trace output */ getFailureDisplay () { let failureLength = 0 const output = [] const suites = this.getOrderedSuites() for (const suite of suites) { const suiteTitle = suite.title const eventsToReport = this.getEventsToReport(suite) for (const test of eventsToReport) { if (test.state !== State.FAILED) { continue } const testTitle = test.title const errors: Error[] = test.errors || (test.error ? [test.error] : []) // If we get here then there is a failed test output.push( '', `${++failureLength}) ${suiteTitle} ${testTitle}`, ) for (const error of errors) { if (!error?.stack?.includes('new AssertionError')) { output.push(this.setMessageColor(error.message, State.FAILED)) } else { output.push(...error.message.split('\n')) } if (error.stack) { output.push(...error.stack.split(/\n/g).map(value => this.setMessageColor(value))) } } } } return output } /** * Get suites in the order they were called * @return {Array} Ordered suites */ getOrderedSuites () { if (this._orderedSuites.length) { return this._orderedSuites } this._orderedSuites = [] for (const uid of this._suiteUids) { for (const [suiteUid, suite] of Object.entries(this.suites)) { if (suiteUid !== uid) { continue } this._orderedSuites.push(suite) } } /** * ensure we include root suite hook errors */ const rootSuite = this.currentSuites[0] if (rootSuite) { const baseRootSuite = { ...rootSuite, type: 'suite', title: '(root)', fullTitle: '(root)', suites: [] } const beforeAllHooks = rootSuite.hooks.filter((hook) => hook.state && hook.title.startsWith('"before') && hook.title.endsWith('"{root}"')) const afterAllHooks = rootSuite.hooks.filter((hook) => hook.state && hook.title.startsWith('"after') && hook.title.endsWith('"{root}"')) this._orderedSuites.unshift(Object.assign({} as SuiteStats, baseRootSuite, { hooks: beforeAllHooks, hooksAndTests: beforeAllHooks })) this._orderedSuites.push(Object.assign({} as SuiteStats, baseRootSuite, { hooks: afterAllHooks, hooksAndTests: afterAllHooks })) } return this._orderedSuites } /** * Indent a suite based on where how it's nested * @param {string} uid Unique suite key * @return {String} Spaces for indentation */ indent (uid: string) { const indents = this._suiteIndents[uid] return indents === 0 ? '' : Array(indents).join(' ') } /** * Get a symbol based on state * @param {string} state State of a test * @return {String} Symbol to display */ getSymbol (state?: keyof Symbols) { return (state && this._symbols[state]) || '?' } /** * Get a color based on a given state * @param {string} state Test state * @return {String} State color */ getColor (state?: string): ChalkColors { // In case of an unknown state let color = ChalkColors.GRAY switch (state) { case State.PASSED: color = ChalkColors.GREEN break case State.PENDING: case State.SKIPPED: color = ChalkColors.CYAN break case State.FAILED: color = ChalkColors.RED break case State.RETRIED: color = ChalkColors.YELLOW break } return color } /** * Get information about the enviroment * @param capability * @param {Boolean} verbose * @param isMultiremote * @return {String} Enviroment string */ getEnviromentCombo (capability: Capabilities.ResolvedTestrunnerCapabilities, verbose = true, isMultiremote = false): string { if (isMultiremote) { const browserNames = Object.values(capability).map((c) => c.browserName) const browserName = browserNames.length > 1 ? `${browserNames.slice(0, -1).join(', ')} and ${browserNames.pop()}` : browserNames.pop() return `MultiremoteBrowser on ${browserName}` } const caps = 'alwaysMatch' in capability ? capability.alwaysMatch : capability const device = caps['appium:deviceName'] const browser = getBrowserName(caps) /** * fallback to different capability types: * browserVersion: W3C format * version: JSONWP format * platformVersion: mobile format * browser_version: invalid BS capability */ // @ts-expect-error outdated JSONWP capabilities const version = caps.browserVersion || caps.version || caps['appium:platformVersion'] || caps.browser_version /** * fallback to different capability types: * platformName: W3C format * platform: JSONWP format * os, os_version: invalid BS capability */ // @ts-expect-error outdated JSONWP capabilities const platform = caps.platformName || caps['appium:platformName'] || caps.platform || (caps.os ? caps.os + (caps.os_version ? ` ${caps.os_version}` : '') : '(unknown)') // Mobile capabilities if (device) { const program = getBrowserName(caps) const executing = program ? `executing ${program}` : '' if (!verbose) { return `${device} ${platform} ${version}` } return `${device} on ${platform} ${version} ${executing}`.trim() } if (!verbose) { return (browser + (version ? ` ${version} ` : ' ') + (platform)).trim() } return browser + (version ? ` (v${version})` : '') + (` on ${platform}`) } }