FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

fix(browser): take failure screenshot if `toMatchScreenshot` can't ca… · vitest-dev/vitest@faace1f · GitHub

/ vitest Public

Commit faace1f

Browse files
macarie
authored
fix(browser): take failure screenshot if toMatchScreenshot can't capture a stable screenshot (#9847)
1 parent 7c06598 commit faace1f

7 files changed

Lines changed: 142 additions & 9 deletions

File tree

‎packages/browser/src/client/tester/expect/toMatchScreenshot.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,5 +120,8 @@ export default async function toMatchScreenshot(
120120
]
121121
.filter(element => element !== null)
122122
.join('\n'),
123+
meta: {
124+
outcome: result.outcome,
125+
},
123126
}
124127
}

‎packages/browser/src/client/tester/runner.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -156,14 +156,16 @@ export function createBrowserRunner(
156156
}
157157

158158
onTaskFinished = async (task: Task) => {
159+
const lastErrorContext = task.result?.errors?.at(-1)?.context
159160
if (
160161
this.config.browser.screenshotFailures
161162
&& document.body.clientHeight > 0
162163
&& task.result?.state === 'fail'
163164
&& task.type === 'test'
164-
&& task.artifacts.every(
165-
artifact => artifact.type !== 'internal:toMatchScreenshot',
166-
)
165+
&& !(
166+
lastErrorContext
167+
&& Reflect.get(lastErrorContext, 'assertionName') === 'toMatchScreenshot'
168+
&& Reflect.get(lastErrorContext, 'meta')?.outcome !== 'unstable-screenshot')
167169
) {
168170
const screenshot = await page.screenshot({
169171
timeout: this.config.browser.providerOptions?.actionTimeout ?? 5_000,

‎packages/browser/src/node/commands/screenshotMatcher/index.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,7 @@ function buildOutput(
273273
case 'unstable-screenshot':
274274
return {
275275
pass: false,
276+
outcome: outcome.type,
276277
reference: outcome.reference && {
277278
path: outcome.reference.path,
278279
width: outcome.reference.image.metadata.width,
@@ -286,6 +287,7 @@ function buildOutput(
286287
case 'missing-reference': {
287288
return {
288289
pass: false,
290+
outcome: outcome.type,
289291
reference: {
290292
path: outcome.reference.path,
291293
width: outcome.reference.image.metadata.width,
@@ -302,11 +304,12 @@ function buildOutput(
302304
case 'update-reference':
303305
case 'matched-immediately':
304306
case 'matched-after-comparison':
305-
return { pass: true }
307+
return { pass: true, outcome: outcome.type }
306308

307309
case 'mismatch':
308310
return {
309311
pass: false,
312+
outcome: outcome.type,
310313
reference: {
311314
path: outcome.reference.path,
312315
width: outcome.reference.image.metadata.width,
@@ -333,6 +336,7 @@ function buildOutput(
333336

334337
return {
335338
pass: false,
339+
outcome: null as never,
336340
actual: null,
337341
reference: null,
338342
diff: null,

‎packages/browser/src/shared/screenshotMatcher/types.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,20 @@ interface ScreenshotData { path: string; width: number; height: number }
1717
export type ScreenshotMatcherOutput = Promise<
1818
{
1919
pass: false
20+
outcome:
21+
| 'unstable-screenshot'
22+
| 'missing-reference'
23+
| 'mismatch'
2024
reference: ScreenshotData | null
2125
actual: ScreenshotData | null
2226
diff: ScreenshotData | null
2327
message: string
2428
}
2529
| {
2630
pass: true
31+
outcome:
32+
| 'update-reference'
33+
| 'matched-immediately'
34+
| 'matched-after-comparison'
2735
}
2836
>

‎packages/expect/src/jest-extend.ts‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,12 @@ function getMatcherState(
6565
}
6666

6767
class JestExtendError extends Error {
68-
constructor(message: string, public actual?: any, public expected?: any) {
68+
constructor(
69+
message: string,
70+
public actual?: any,
71+
public expected?: any,
72+
public context?: { assertionName: string; meta?: object },
73+
) {
6974
super(message)
7075
}
7176
}
@@ -92,23 +97,33 @@ function JestExtendPlugin(
9297
&& typeof (result as any).then === 'function'
9398
) {
9499
const thenable = result as PromiseLike<SyncExpectationResult>
95-
return thenable.then(({ pass, message, actual, expected }) => {
100+
return thenable.then(({ pass, message, actual, expected, meta }) => {
96101
if ((pass && isNot) || (!pass && !isNot)) {
97102
const errorMessage = customMessage != null
98103
? customMessage
99104
: message()
100-
throw new JestExtendError(errorMessage, actual, expected)
105+
throw new JestExtendError(
106+
errorMessage,
107+
actual,
108+
expected,
109+
{ assertionName: expectAssertionName, meta },
110+
)
101111
}
102112
})
103113
}
104114

105-
const { pass, message, actual, expected } = result as SyncExpectationResult
115+
const { pass, message, actual, expected, meta } = result as SyncExpectationResult
106116

107117
if ((pass && isNot) || (!pass && !isNot)) {
108118
const errorMessage = customMessage != null
109119
? customMessage
110120
: message()
111-
throw new JestExtendError(errorMessage, actual, expected)
121+
throw new JestExtendError(
122+
errorMessage,
123+
actual,
124+
expected,
125+
{ assertionName: expectAssertionName, meta },
126+
)
112127
}
113128
}
114129

‎packages/expect/src/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ export interface SyncExpectationResult {
9191
message: () => string
9292
actual?: any
9393
expected?: any
94+
meta?: object
9495
}
9596

9697
export type AsyncExpectationResult = Promise<SyncExpectationResult>

‎test/browser/specs/failure-screenshot.test.ts‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
import type { TestFsStructure } from '../../test-utils'
2+
import { describe, expect, test } from 'vitest'
3+
import { runInlineTests } from '../../test-utils'
4+
import utilsContent from '../fixtures/expect-dom/utils?raw'
5+
import { instances, provider } from '../settings'
6+
7+
const testFilename = 'basic.test.ts'
8+
9+
async function runBrowserTests(
10+
structure: TestFsStructure,
11+
) {
12+
return runInlineTests({
13+
...structure,
14+
'vitest.config.js': `
15+
import { ${provider.name} } from '@vitest/browser-${provider.name}'
16+
export default {
17+
test: {
18+
browser: {
19+
enabled: true,
20+
screenshotFailures: true,
21+
provider: ${provider.name}(),
22+
ui: false,
23+
headless: true,
24+
instances: ${JSON.stringify(instances.slice(0, 1) /* logic not bound to browser instance */)},
25+
},
26+
reporters: ['verbose'],
27+
update: 'new',
28+
},
29+
}`,
30+
})
31+
}
32+
33+
describe('failure screenshots', () => {
34+
describe('`toMatchScreenshot`', () => {
35+
test('usually does NOT produce a failure screenshot', async () => {
36+
const { stderr } = await runBrowserTests(
37+
{
38+
[testFilename]: /* ts */`
39+
import { page } from 'vitest/browser'
40+
import { test } from 'vitest'
41+
import { render } from './utils'
42+
43+
test('screenshot-initial', async ({ expect }) => {
44+
render('<div data-testid="el">Test</div>')
45+
await expect(page.getByTestId('el')).toMatchScreenshot()
46+
})
47+
`,
48+
'utils.ts': utilsContent,
49+
},
50+
)
51+
52+
expect(stderr).toContain('No existing reference screenshot found; a new one was created.')
53+
expect(stderr).not.toContain('Failure screenshot:')
54+
})
55+
56+
test('unstable screenshot fails produces a failure screenshot', async () => {
57+
const { stderr } = await runBrowserTests(
58+
{
59+
[testFilename]: /* ts */`
60+
import { page } from 'vitest/browser'
61+
import { test } from 'vitest'
62+
import { render } from './utils'
63+
64+
test('screenshot-unstable', async ({ expect }) => {
65+
render('<div data-testid="el">Test</div>')
66+
await expect(page.getByTestId('el')).toMatchScreenshot({ timeout: 1 })
67+
})
68+
`,
69+
'utils.ts': utilsContent,
70+
},
71+
)
72+
73+
expect(stderr).toContain('Could not capture a stable screenshot within 1ms.')
74+
expect(stderr).toContain('Failure screenshot:')
75+
})
76+
77+
test('`expect.soft` produces a failure screenshot', async () => {
78+
const { stderr } = await runBrowserTests(
79+
{
80+
[testFilename]: /* ts */`
81+
import { page } from 'vitest/browser'
82+
import { test } from 'vitest'
83+
import { render } from './utils'
84+
85+
test('screenshot-soft-then-fail', async ({ expect }) => {
86+
render('<div data-testid="el">Test</div>')
87+
await expect.soft(page.getByTestId('el')).toMatchScreenshot()
88+
expect(1).toBe(2)
89+
})
90+
`,
91+
'utils.ts': utilsContent,
92+
},
93+
)
94+
95+
expect(stderr).toContain('No existing reference screenshot found; a new one was created.')
96+
expect(stderr).toContain('expected 1 to be 2')
97+
expect(stderr).toContain('Failure screenshot:')
98+
})
99+
})
100+
})

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL