chore: include action ref in generated actions (microsoft#36818)

Author: pavelfeldman

packages/injected/src/recorder/recorder.ts

@@ -650,12 +650,13 @@ class JsonRecordActionTool implements RecorderTool {
       return;
 
     const checkbox = asCheckbox(element);
-    const { ariaSnapshot, selector } = this._ariaSnapshot(element);
+    const { ariaSnapshot, selector, ref } = this._ariaSnapshot(element);
     if (checkbox && event.detail === 1) {
       // Interestingly, inputElement.checked is reversed inside this event handler.
       this._recorder.recordAction({
         name: checkbox.checked ? 'check' : 'uncheck',
         selector,
+        ref,
         signals: [],
         ariaSnapshot,
       });
@@ -665,6 +666,7 @@ class JsonRecordActionTool implements RecorderTool {
     this._recorder.recordAction({
       name: 'click',
       selector,
+      ref,
       ariaSnapshot,
       position: positionForEvent(event),
       signals: [],
@@ -681,27 +683,29 @@ class JsonRecordActionTool implements RecorderTool {
     if (this._shouldIgnoreMouseEvent(event))
       return;
 
-    const { ariaSnapshot, selector } = this._ariaSnapshot(element);
+    const { ariaSnapshot, selector, ref } = this._ariaSnapshot(element);
     this._recorder.recordAction({
       name: 'click',
       selector,
+      ref,
       ariaSnapshot,
       position: positionForEvent(event),
       signals: [],
       button: buttonForEvent(event),
       modifiers: modifiersForEvent(event),
-      clickCount: event.detail
+      clickCount: event.detail,
     });
   }
 
   onInput(event: Event) {
     const element = this._recorder.deepEventTarget(event);
 
-    const { ariaSnapshot, selector } = this._ariaSnapshot(element);
+    const { ariaSnapshot, selector, ref } = this._ariaSnapshot(element);
     if (isRangeInput(element)) {
       this._recorder.recordAction({
         name: 'fill',
         selector,
+        ref,
         ariaSnapshot,
         signals: [],
         text: element.value,
@@ -717,6 +721,7 @@ class JsonRecordActionTool implements RecorderTool {
 
       this._recorder.recordAction({
         name: 'fill',
+        ref,
         selector,
         ariaSnapshot,
         signals: [],
@@ -730,6 +735,7 @@ class JsonRecordActionTool implements RecorderTool {
       this._recorder.recordAction({
         name: 'select',
         selector,
+        ref,
         ariaSnapshot,
         options: [...selectElement.selectedOptions].map(option => option.value),
         signals: []
@@ -743,7 +749,7 @@ class JsonRecordActionTool implements RecorderTool {
       return;
 
     const element = this._recorder.deepEventTarget(event);
-    const { ariaSnapshot, selector } = this._ariaSnapshot(element);
+    const { ariaSnapshot, selector, ref } = this._ariaSnapshot(element);
 
     // Similarly to click, trigger checkbox on key event, not input.
     if (event.key === ' ') {
@@ -752,6 +758,7 @@ class JsonRecordActionTool implements RecorderTool {
         this._recorder.recordAction({
           name: checkbox.checked ? 'uncheck' : 'check',
           selector,
+          ref,
           ariaSnapshot,
           signals: [],
         });
@@ -762,6 +769,7 @@ class JsonRecordActionTool implements RecorderTool {
     this._recorder.recordAction({
       name: 'press',
       selector,
+      ref,
       ariaSnapshot,
       signals: [],
       key: event.key,
@@ -819,12 +827,12 @@ class JsonRecordActionTool implements RecorderTool {
     return false;
   }
 
-  private _ariaSnapshot(element: HTMLElement): { ariaSnapshot: string, selector: string };
-  private _ariaSnapshot(element: HTMLElement | undefined): { ariaSnapshot: string, selector?: string } {
+  private _ariaSnapshot(element: HTMLElement): { ariaSnapshot: string, selector: string, ref?: string };
+  private _ariaSnapshot(element: HTMLElement | undefined): { ariaSnapshot: string, selector?: string, ref?: string } {
     const { ariaSnapshot, refs } = this._recorder.injectedScript.ariaSnapshotForRecorder();
     const ref = element ? refs.get(element) : undefined;
-    const selector = ref ? `aria-ref=${ref}` : undefined;
-    return { ariaSnapshot, selector };
+    const elementInfo = element ? this._recorder.injectedScript.generateSelector(element, { testIdAttributeName: this._recorder.state.testIdAttributeName }) : undefined;
+    return { ariaSnapshot, selector: elementInfo?.selector, ref };
   }
 }
 

packages/playwright-core/src/client/browserContext.ts

@@ -50,9 +50,9 @@ import type * as channels from '@protocol/channels';
 import type * as actions from '@recorder/actions';
 
 interface RecorderEventSink {
-  actionAdded(page: Page, actionInContext: actions.ActionInContext): void;
-  actionUpdated(page: Page, actionInContext: actions.ActionInContext): void;
-  signalAdded(page: Page, signal: actions.SignalInContext): void;
+  actionAdded?(page: Page, actionInContext: actions.ActionInContext, code: string[]): void;
+  actionUpdated?(page: Page, actionInContext: actions.ActionInContext, code: string[]): void;
+  signalAdded?(page: Page, signal: actions.SignalInContext): void;
 }
 
 export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel> implements api.BrowserContext {
@@ -148,13 +148,13 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
     this._channel.on('requestFailed', ({ request, failureText, responseEndTiming, page }) => this._onRequestFailed(network.Request.from(request), responseEndTiming, failureText, Page.fromNullable(page)));
     this._channel.on('requestFinished', params => this._onRequestFinished(params));
     this._channel.on('response', ({ response, page }) => this._onResponse(network.Response.from(response), Page.fromNullable(page)));
-    this._channel.on('recorderEvent', ({ event, data, page }) => {
+    this._channel.on('recorderEvent', ({ event, data, page, code }) => {
       if (event === 'actionAdded')
-        this._onRecorderEventSink?.actionAdded(Page.from(page), data as actions.ActionInContext);
+        this._onRecorderEventSink?.actionAdded?.(Page.from(page), data as actions.ActionInContext, code);
       else if (event === 'actionUpdated')
-        this._onRecorderEventSink?.actionUpdated(Page.from(page), data as actions.ActionInContext);
+        this._onRecorderEventSink?.actionUpdated?.(Page.from(page), data as actions.ActionInContext, code);
       else if (event === 'signalAdded')
-        this._onRecorderEventSink?.signalAdded(Page.from(page), data as actions.SignalInContext);
+        this._onRecorderEventSink?.signalAdded?.(Page.from(page), data as actions.SignalInContext);
     });
     this._closedPromise = new Promise(f => this.once(Events.BrowserContext.Close, f));
 

packages/playwright-core/src/protocol/validator.ts

@@ -984,6 +984,7 @@ scheme.BrowserContextRecorderEventEvent = tObject({
   event: tEnum(['actionAdded', 'actionUpdated', 'signalAdded']),
   data: tAny,
   page: tChannel(['Page']),
+  code: tArray(tString),
 });
 scheme.BrowserContextAddCookiesParams = tObject({
   cookies: tArray(tType('SetNetworkCookie')),

packages/playwright-core/src/server/dispatchers/browserContextDispatcher.ts

@@ -35,7 +35,7 @@ import { WritableStreamDispatcher } from './writableStreamDispatcher';
 import { createGuid } from '../utils/crypto';
 import { urlMatches } from '../../utils/isomorphic/urlMatch';
 import { Recorder } from '../recorder';
-import { ProgrammaticRecorderApp, RecorderApp } from '../recorder/recorderApp';
+import { RecorderApp } from '../recorder/recorderApp';
 
 import type { Artifact } from '../artifact';
 import type { ConsoleMessage } from '../console';
@@ -200,8 +200,8 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
         page: PageDispatcher.fromNullable(this, request.frame()?._page.initializedOrUndefined()),
       });
     });
-    this.addObjectListener(BrowserContext.Events.RecorderEvent, ({ event, data, page }: { event: 'actionAdded' | 'actionUpdated' | 'signalAdded', data: any, page: Page }) => {
-      this._dispatchEvent('recorderEvent', { event, data, page: PageDispatcher.from(this, page) });
+    this.addObjectListener(BrowserContext.Events.RecorderEvent, ({ event, data, page, code }: { event: 'actionAdded' | 'actionUpdated' | 'signalAdded', data: any, page: Page, code: string[] }) => {
+      this._dispatchEvent('recorderEvent', { event, data, code, page: PageDispatcher.from(this, page) });
     });
   }
 
@@ -336,11 +336,6 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
   }
 
   async enableRecorder(params: channels.BrowserContextEnableRecorderParams, progress: Progress): Promise<void> {
-    const recorder = await Recorder.forContext(this._context, params);
-    if (params.recorderMode === 'api') {
-      await ProgrammaticRecorderApp.run(this._context, recorder);
-      return;
-    }
     await RecorderApp.show(this._context, params);
   }
 

packages/playwright-core/src/server/recorder/recorderApp.ts

@@ -166,7 +166,8 @@ export class RecorderApp {
       return;
     const recorder = await Recorder.forContext(context, params);
     if (params.recorderMode === 'api') {
-      await ProgrammaticRecorderApp.run(context, recorder);
+      const browserName = context._browser.options.name;
+      await ProgrammaticRecorderApp.run(context, recorder, browserName, params);
       return;
     }
     await RecorderApp._show(recorder, context, params);
@@ -364,21 +365,33 @@ export class RecorderApp {
 }
 
 export class ProgrammaticRecorderApp {
-  static async run(inspectedContext: BrowserContext, recorder: Recorder) {
+  static async run(inspectedContext: BrowserContext, recorder: Recorder, browserName: string, params: channels.BrowserContextEnableRecorderParams) {
     let lastAction: actions.ActionInContext | null = null;
+    const languages = [...languageSet()];
+
+    const languageGeneratorOptions = {
+      browserName: browserName,
+      launchOptions: { headless: false, ...params.launchOptions, tracesDir: undefined },
+      contextOptions: { ...params.contextOptions },
+      deviceName: params.device,
+      saveStorage: params.saveStorage,
+    };
+    const languageGenerator = languages.find(l => l.id === params.language) ?? languages.find(l => l.id === 'playwright-test')!;
+
     recorder.on(RecorderEvent.ActionAdded, action => {
       const page = findPageByGuid(inspectedContext, action.frame.pageGuid);
       if (!page)
         return;
+      const { actionTexts } = generateCode([action], languageGenerator, languageGeneratorOptions);
       if (!lastAction || !shouldMergeAction(action, lastAction))
-        inspectedContext.emit(BrowserContext.Events.RecorderEvent, { event: 'actionAdded', data: action, page });
+        inspectedContext.emit(BrowserContext.Events.RecorderEvent, { event: 'actionAdded', data: action, page, code: actionTexts });
       else
-        inspectedContext.emit(BrowserContext.Events.RecorderEvent, { event: 'actionUpdated', data: action, page });
+        inspectedContext.emit(BrowserContext.Events.RecorderEvent, { event: 'actionUpdated', data: action, page, code: actionTexts });
       lastAction = action;
     });
     recorder.on(RecorderEvent.SignalAdded, signal => {
       const page = findPageByGuid(inspectedContext, signal.frame.pageGuid);
-      inspectedContext.emit(BrowserContext.Events.RecorderEvent, { event: 'signalAdded', data: signal, page });
+      inspectedContext.emit(BrowserContext.Events.RecorderEvent, { event: 'signalAdded', data: signal, page, code: [] });
     });
   }
 }

packages/protocol/src/channels.d.ts

@@ -1721,6 +1721,7 @@ export type BrowserContextRecorderEventEvent = {
   event: 'actionAdded' | 'actionUpdated' | 'signalAdded',
   data: any,
   page: PageChannel,
+  code: string[],
 };
 export type BrowserContextAddCookiesParams = {
   cookies: SetNetworkCookie[],

packages/protocol/src/protocol.yml

@@ -1549,6 +1549,9 @@ BrowserContext:
           - signalAdded
         data: json
         page: Page
+        code:
+          type: array
+          items: string
 
 Page:
   type: interface

packages/recorder/src/actions.d.ts

@@ -41,6 +41,7 @@ export type ActionBase = {
 
 export type ActionWithSelector = ActionBase & {
   selector: string,
+  ref?: string,
 };
 
 export type ClickAction = ActionWithSelector & {
@@ -78,9 +79,8 @@ export type ClosesPageAction = ActionBase & {
   name: 'closePage',
 };
 
-export type PressAction = ActionBase & {
+export type PressAction = ActionWithSelector & {
   name: 'press',
-  selector: string,
   key: string,
   modifiers: number,
 };

tests/library/inspector/recorder-api.spec.ts

@@ -0,0 +1,83 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './inspectorTest';
+
+import type { Page } from '@playwright/test';
+import type * as actions from '@recorder/actions';
+
+class RecorderLog {
+  actions: (actions.ActionInContext & { code: string[] })[] = [];
+
+  actionAdded(page: Page, actionInContext: actions.ActionInContext, code: string[]): void {
+    this.actions.push({ ...actionInContext, code });
+  }
+
+  actionUpdated(page: Page, actionInContext: actions.ActionInContext, code: string[]): void {
+    this.actions[this.actions.length - 1] = { ...actionInContext, code };
+  }
+}
+
+async function startRecording(context) {
+  const log = new RecorderLog();
+  await (context as any)._enableRecorder({
+    mode: 'recording',
+    recorderMode: 'api',
+  }, log);
+  return {
+    lastAction: () => log.actions[log.actions.length - 1],
+  };
+}
+
+test('should click', async ({ context }) => {
+  const log = await startRecording(context);
+  const page = await context.newPage();
+  await page.setContent(`<button onclick="console.log('click')">Submit</button>`);
+
+  await page.getByRole('button', { name: 'Submit' }).click();
+
+  expect(log.lastAction()).toEqual(
+      expect.objectContaining({
+        action: expect.objectContaining({
+          name: 'click',
+          selector: 'internal:role=button[name="Submit"i]',
+          ref: 'e2',
+          ariaSnapshot: '- button "Submit" [active] [ref=e2] [cursor=pointer]',
+        }),
+        code: [`  await page.getByRole('button', { name: 'Submit' }).click();`],
+        startTime: expect.any(Number),
+      }));
+});
+
+test('should type', async ({ context }) => {
+  const log = await startRecording(context);
+  const page = await context.newPage();
+  await page.setContent(`<input type="text" />`);
+
+  await page.getByRole('textbox').pressSequentially('Hello');
+
+  expect(log.lastAction()).toEqual(
+      expect.objectContaining({
+        action: expect.objectContaining({
+          name: 'fill',
+          selector: 'internal:role=textbox',
+          ref: 'e2',
+          ariaSnapshot: '- textbox [active] [ref=e2] [cursor=pointer]: Hello',
+        }),
+        code: [`  await page.getByRole('textbox').fill('Hello');`],
+        startTime: expect.any(Number),
+      }));
+});