feat: auto-generate toBeVisible assertion while recording (microsoft#36812)

Author: dgozman

packages/injected/src/ariaSnapshot.ts

@@ -48,7 +48,9 @@ type AriaRef = {
 
 let lastRef = 0;
 
-export function generateAriaTree(rootElement: Element, options?: { forAI?: boolean, refPrefix?: string }): AriaSnapshot {
+export type AriaTreeOptions = { forAI?: boolean, refPrefix?: string, refs?: boolean, visibleOnly?: boolean };
+
+export function generateAriaTree(rootElement: Element, options?: AriaTreeOptions): AriaSnapshot {
   const visited = new Set<Node>();
 
   const snapshot: AriaSnapshot = {
@@ -91,7 +93,7 @@ export function generateAriaTree(rootElement: Element, options?: { forAI?: boole
       }
     }
 
-    const visible = !isElementHiddenForAria || isElementVisible(element);
+    const visible = options?.visibleOnly ? isElementVisible(element) : !isElementHiddenForAria || isElementVisible(element);
     const childAriaNode = visible ? toAriaNode(element, options) : null;
     if (childAriaNode) {
       if (childAriaNode.ref) {
@@ -155,8 +157,8 @@ export function generateAriaTree(rootElement: Element, options?: { forAI?: boole
   return snapshot;
 }
 
-function ariaRef(element: Element, role: string, name: string, options?: { forAI?: boolean, refPrefix?: string }): string | undefined {
-  if (!options?.forAI)
+function ariaRef(element: Element, role: string, name: string, options?: AriaTreeOptions): string | undefined {
+  if (!options?.forAI && !options?.refs)
     return undefined;
 
   let ariaRef: AriaRef | undefined;
@@ -168,7 +170,7 @@ function ariaRef(element: Element, role: string, name: string, options?: { forAI
   return ariaRef.ref;
 }
 
-function toAriaNode(element: Element, options?: { forAI?: boolean, refPrefix?: string }): AriaNode | null {
+function toAriaNode(element: Element, options?: AriaTreeOptions): AriaNode | null {
   const active = element.ownerDocument.activeElement === element;
   if (element.nodeName === 'IFRAME') {
     return {
@@ -409,7 +411,7 @@ function matchesNodeDeep(root: AriaNode, template: AriaTemplateNode, collectAll:
   return results;
 }
 
-export function renderAriaTree(ariaSnapshot: AriaSnapshot, options?: { mode?: 'raw' | 'regex', forAI?: boolean }): string {
+export function renderAriaTree(ariaSnapshot: AriaSnapshot, options?: { mode?: 'raw' | 'regex', forAI?: boolean, refs?: boolean }): string {
   const lines: string[] = [];
   const includeText = options?.mode === 'regex' ? textContributesInfo : () => true;
   const renderString = options?.mode === 'regex' ? convertToBestGuessRegex : (str: string) => str;
@@ -450,11 +452,12 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, options?: { mode?: 'r
       key += ` [pressed]`;
     if (ariaNode.selected === true)
       key += ` [selected]`;
-    if (options?.forAI && receivesPointerEvents(ariaNode)) {
-      const ref = ariaNode.ref;
-      const cursor = hasPointerCursor(ariaNode) ? ' [cursor=pointer]' : '';
-      if (ref)
-        key += ` [ref=${ref}]${cursor}`;
+
+    const includeRef = (options?.forAI && receivesPointerEvents(ariaNode)) || options?.refs;
+    if (includeRef && ariaNode.ref) {
+      key += ` [ref=${ariaNode.ref}]`;
+      if (hasPointerCursor(ariaNode))
+        key += ' [cursor=pointer]';
     }
 
     const escapedKey = indent + '- ' + yamlEscapeKeyIfNeeded(key);

packages/injected/src/injectedScript.ts

@@ -39,7 +39,7 @@ import type { CSSComplexSelectorList } from '@isomorphic/cssParser';
 import type { Language } from '@isomorphic/locatorGenerators';
 import type { NestedSelectorBody, ParsedSelector, ParsedSelectorPart } from '@isomorphic/selectorParser';
 import type * as channels from '@protocol/channels';
-import type { AriaSnapshot } from './ariaSnapshot';
+import type { AriaSnapshot, AriaTreeOptions } from './ariaSnapshot';
 import type { LayoutSelectorName } from './layoutSelectorUtils';
 import type { SelectorEngine, SelectorRoot } from './selectorEngine';
 import type { GenerateSelectorOptions } from './selectorGenerator';
@@ -297,7 +297,7 @@ export class InjectedScript {
     return new Set<Element>(result.map(r => r.element));
   }
 
-  ariaSnapshot(node: Node, options?: { mode?: 'raw' | 'regex', forAI?: boolean, refPrefix?: string }): string {
+  ariaSnapshot(node: Node, options?: { mode?: 'raw' | 'regex' } & AriaTreeOptions): string {
     if (node.nodeType !== Node.ELEMENT_NODE)
       throw this.createStacklessError('Can only capture aria snapshot of Element nodes.');
     this._lastAriaSnapshot = generateAriaTree(node as Element, options);

packages/injected/src/recorder/recorder.ts

@@ -383,7 +383,7 @@ class RecordActionTool implements RecorderTool {
     const target = this._recorder.deepEventTarget(event);
 
     if (target.nodeName === 'INPUT' && (target as HTMLInputElement).type.toLowerCase() === 'file') {
-      this._recorder.recordAction({
+      this._recordAction({
         name: 'setInputFiles',
         selector: this._activeModel!.selector,
         signals: [],
@@ -393,7 +393,7 @@ class RecordActionTool implements RecorderTool {
     }
 
     if (isRangeInput(target)) {
-      this._recorder.recordAction({
+      this._recordAction({
         name: 'fill',
         // must use hoveredModel instead of activeModel for it to work in webkit
         selector: this._hoveredModel!.selector,
@@ -412,7 +412,7 @@ class RecordActionTool implements RecorderTool {
       // Non-navigating actions are simply recorded by Playwright.
       if (this._consumedDueWrongTarget(event))
         return;
-      this._recorder.recordAction({
+      this._recordAction({
         name: 'fill',
         selector: this._activeModel!.selector,
         signals: [],
@@ -545,9 +545,21 @@ class RecordActionTool implements RecorderTool {
       consumeEvent(event);
   }
 
+  private _captureAriaSnapshotForAction(action: actions.Action) {
+    const documentElement = this._recorder.injectedScript.document.documentElement;
+    if (documentElement)
+      action.ariaSnapshot = this._recorder.injectedScript.ariaSnapshot(documentElement, { mode: 'raw', refs: true, visibleOnly: true });
+  }
+
+  private _recordAction(action: actions.Action) {
+    this._captureAriaSnapshotForAction(action);
+    this._recorder.recordAction(action);
+  }
+
   private _performAction(action: actions.PerformOnRecordAction) {
     this._recorder.updateHighlight(null, false);
 
+    this._captureAriaSnapshotForAction(action);
     this._performingActions.add(action);
 
     const promise = this._recorder.performAction(action).then(() => {

packages/playwright-core/src/server/codegen/jsonl.ts

@@ -31,6 +31,7 @@ export class JsonlLanguageGenerator implements LanguageGenerator {
       ...actionInContext.action,
       ...actionInContext.frame,
       locator,
+      ariaSnapshot: undefined,
     };
     return JSON.stringify(entry);
   }

packages/playwright-core/src/server/frameSelectors.ts

@@ -49,7 +49,7 @@ export class FrameSelectors {
     return this.frame._page.browserContext.selectors().parseSelector(selector, strict);
   }
 
-  async query(selector: string, options?: types.StrictOptions, scope?: ElementHandle): Promise<ElementHandle<Element> | null> {
+  async query(selector: string, options?: types.StrictOptions & { mainWorld?: boolean }, scope?: ElementHandle): Promise<ElementHandle<Element> | null> {
     const resolved = await this.resolveInjectedForSelector(selector, options, scope);
     // Be careful, |this.frame| can be different from |resolved.frame|.
     if (!resolved)

packages/playwright-core/src/server/frames.ts

@@ -1179,8 +1179,8 @@ export class Frame extends SdkObject {
     dom.assertDone(await this._retryWithProgressIfNotConnected(progress, selector, options.strict, true /* performActionPreChecks */, handle => handle._blur(progress)));
   }
 
-  async resolveSelector(progress: Progress, selector: string): Promise<{ resolvedSelector: string }> {
-    const element = await progress.race(this.selectors.query(selector));
+  async resolveSelector(progress: Progress, selector: string, options: { mainWorld?: boolean } = {}): Promise<{ resolvedSelector: string }> {
+    const element = await progress.race(this.selectors.query(selector, options));
     if (!element)
       throw new Error(`No element matching ${selector}`);
 

packages/playwright-core/src/server/recorder.ts

@@ -20,7 +20,7 @@ import fs from 'fs';
 import { isUnderTest } from '../utils';
 import { BrowserContext } from './browserContext';
 import { Debugger } from './debugger';
-import { buildFullSelector, generateFrameSelector, metadataToCallLog } from './recorder/recorderUtils';
+import { buildFullSelector, generateFrameSelector, isAssertAction, metadataToCallLog, shouldMergeAction } from './recorder/recorderUtils';
 import { locatorOrSelectorAsSelector } from '../utils/isomorphic/locatorParser';
 import { stringifySelector } from '../utils/isomorphic/selectorParser';
 import { ProgressController } from './progress';
@@ -31,6 +31,8 @@ import { eventsHelper, monotonicTime } from './../utils';
 import { Frame } from './frames';
 import { Page } from './page';
 import { performAction } from './recorder/recorderRunner';
+import { findNewElementRef } from '../utils/isomorphic/ariaSnapshot';
+import { yaml } from '../utilsBundle';
 
 import type { Language } from './codegen/types';
 import type { CallMetadata, InstrumentationListener, SdkObject } from './instrumentation';
@@ -520,16 +522,69 @@ export class Recorder extends EventEmitter<RecorderEventMap> implements Instrume
     return actionInContext;
   }
 
+  private async _maybeGenerateAssertAction(frame: Frame, actionInContext: actions.ActionInContext) {
+    const lastAction = getLastFrameAction(frame);
+    if (isAssertAction(actionInContext.action))
+      return;
+    if (lastAction && (isAssertAction(lastAction.action) || shouldMergeAction(actionInContext, lastAction)))
+      return;
+    const newSnapshot = actionInContext.action.ariaSnapshot;
+    if (!newSnapshot)
+      return;
+    const lastSnapshot = lastAction?.action.ariaSnapshot || `- document [ref=e1]\n`;
+    if (!lastSnapshot)
+      return;
+    const callMetadata = serverSideCallMetadata();
+    const controller = new ProgressController(callMetadata, frame);
+    const selector = await controller.run(async progress => {
+      const ref = findNewElementRef(yaml, lastSnapshot, newSnapshot);
+      if (!ref)
+        return;
+      // Note: recorder runs in the main world, so we need to resolve the ref in the main world.
+      // Note: resolveSelector always returns a |page|-based selector, not |frame|-based.
+      const { resolvedSelector } = await frame.resolveSelector(progress, `aria-ref=${ref}`, { mainWorld: true }).catch(() => ({ resolvedSelector: undefined }));
+      if (!resolvedSelector)
+        return;
+      const isVisible = await frame._page.mainFrame().isVisible(progress, resolvedSelector, { strict: true }).catch(() => false);
+      return isVisible ? resolvedSelector : undefined;
+    }).catch(() => undefined);
+    if (!selector)
+      return;
+    if (!actionInContext.frame.framePath.length && 'selector' in actionInContext.action && actionInContext.action.selector === selector)
+      return;  // Do not assert the action target, auto-waiting takes care of it.
+    const assertActionInContext: actions.ActionInContext = {
+      frame: {
+        pageGuid: actionInContext.frame.pageGuid,
+        pageAlias: actionInContext.frame.pageAlias,
+        framePath: [],
+      },
+      action: {
+        name: 'assertVisible',
+        selector,
+        signals: [],
+      },
+      startTime: actionInContext.startTime,
+      endTime: actionInContext.startTime,
+    };
+    return assertActionInContext;
+  }
+
   private async _performAction(frame: Frame, action: actions.PerformOnRecordAction) {
     const actionInContext = await this._createActionInContext(frame, action);
+    const assertActionInContext = await this._maybeGenerateAssertAction(frame, actionInContext);
+    if (assertActionInContext)
+      this._signalProcessor.addAction(assertActionInContext);
     this._signalProcessor.addAction(actionInContext);
+    setLastFrameAction(frame, actionInContext);
     if (actionInContext.action.name !== 'openPage' && actionInContext.action.name !== 'closePage')
       await performAction(this._pageAliases, actionInContext);
     actionInContext.endTime = monotonicTime();
   }
 
   private async _recordAction(frame: Frame, action: actions.Action) {
-    this._signalProcessor.addAction(await this._createActionInContext(frame, action));
+    const actionInContext = await this._createActionInContext(frame, action);
+    this._signalProcessor.addAction(actionInContext);
+    setLastFrameAction(frame, actionInContext);
   }
 
   private _onFrameNavigated(frame: Frame, page: Page) {
@@ -569,3 +624,11 @@ function languageForFile(file: string) {
     return 'csharp';
   return 'javascript';
 }
+
+const kLastFrameActionSymbol = Symbol('lastFrameAction');
+function getLastFrameAction(frame: Frame): actions.ActionInContext | undefined {
+  return (frame as any)[kLastFrameActionSymbol];
+}
+function setLastFrameAction(frame: Frame, action: actions.ActionInContext | undefined) {
+  (frame as any)[kLastFrameActionSymbol] = action;
+}

packages/playwright-core/src/server/recorder/recorderUtils.ts

@@ -73,6 +73,10 @@ export async function frameForAction(pageAliases: Map<Page, string>, actionInCon
   return result.frame;
 }
 
+export function isAssertAction(action: actions.Action): action is actions.AssertAction {
+  return action.name.startsWith('assert');
+}
+
 function isSameAction(a: actions.ActionInContext, b: actions.ActionInContext): boolean {
   return a.action.name === b.action.name && a.frame.pageAlias === b.frame.pageAlias && a.frame.framePath.join('|') === b.frame.framePath.join('|');
 }