[pull] main from microsoft:main

packages/injected/src/ariaSnapshot.ts

@@ -77,7 +77,7 @@ function toInternalOptions(options: AriaTreeOptions): InternalOptions {
   }
   if (options.mode === 'autoexpect') {
     // To auto-generate assertions on visible elements.
-    return { visibility: 'ariaAndVisible', refs: 'all' };
+    return { visibility: 'ariaAndVisible', refs: 'none' };
   }
   if (options.mode === 'codegen') {
     // To generate aria assertion with regex heurisitcs.

packages/injected/src/injectedScript.ts

@@ -107,6 +107,7 @@ export class InjectedScript {
     isInsideScope,
     normalizeWhiteSpace,
     parseAriaSnapshot,
+    generateAriaTree,
     // Builtins protect injected code from clock emulation.
     builtins: null as unknown as Builtins,
   };

packages/injected/src/recorder/recorder.ts

@@ -23,6 +23,7 @@ import type { ElementText } from '../selectorUtils';
 import type * as actions from '@recorder/actions';
 import type { ElementInfo, Mode, OverlayState, UIState } from '@recorder/recorderTypes';
 import type { Language } from '@isomorphic/locatorGenerators';
+import type { AriaNode, AriaSnapshot } from '@injected/ariaSnapshot';
 
 const HighlightColors = {
   multiple: '#f6b26b7f',
@@ -543,21 +544,13 @@ class RecordActionTool implements RecorderTool {
       consumeEvent(event);
   }
 
-  private _captureAriaSnapshotForAction(action: actions.Action) {
-    const documentElement = this._recorder.injectedScript.document.documentElement;
-    if (documentElement)
-      action.ariaSnapshot = this._recorder.injectedScript.ariaSnapshot(documentElement, { mode: 'autoexpect' });
-  }
-
   private _recordAction(action: actions.Action) {
-    this._captureAriaSnapshotForAction(action);
     this._recorder.recordAction(action);
   }
 
   private _performAction(action: actions.PerformOnRecordAction) {
     this._recorder.updateHighlight(null, false);
 
-    this._captureAriaSnapshotForAction(action);
     this._performingActions.add(action);
 
     const promise = this._recorder.performAction(action).then(() => {
@@ -1267,6 +1260,7 @@ export class Recorder {
   private _tools: Record<Mode, RecorderTool>;
   private _lastHighlightedSelector: string | undefined = undefined;
   private _lastHighlightedAriaTemplateJSON: string = 'undefined';
+  private _lastActionAutoexpectSnapshot: AriaSnapshot | undefined;
   readonly highlight: Highlight;
   readonly overlay: Overlay | undefined;
   private _stylesheet: CSSStyleSheet;
@@ -1585,11 +1579,25 @@ export class Recorder {
     void this._delegate.setMode?.(mode);
   }
 
+  private _captureAutoExpectSnapshot() {
+    const documentElement = this.injectedScript.document.documentElement;
+    return documentElement ? this.injectedScript.utils.generateAriaTree(documentElement, { mode: 'autoexpect' }) : undefined;
+  }
+
   async performAction(action: actions.PerformOnRecordAction) {
+    const previousSnapshot = this._lastActionAutoexpectSnapshot;
+    this._lastActionAutoexpectSnapshot = this._captureAutoExpectSnapshot();
+    if (!isAssertAction(action) && this._lastActionAutoexpectSnapshot) {
+      const element = findNewElement(previousSnapshot, this._lastActionAutoexpectSnapshot);
+      action.preconditionSelector = element ? this.injectedScript.generateSelector(element, { testIdAttributeName: this.state.testIdAttributeName }).selector : undefined;
+      if (action.preconditionSelector === action.selector)
+        action.preconditionSelector = undefined;
+    }
     await this._delegate.performAction?.(action).catch(() => {});
   }
 
   recordAction(action: actions.Action) {
+    this._lastActionAutoexpectSnapshot = this._captureAutoExpectSnapshot();
     void this._delegate.recordAction?.(action);
   }
 
@@ -1794,3 +1802,56 @@ function createSvgElement(doc: Document, { tagName, attrs, children }: SvgJson):
 
   return elem;
 }
+
+function isAssertAction(action: actions.Action): action is actions.AssertAction {
+  return action.name.startsWith('assert');
+}
+
+function findNewElement(from: AriaSnapshot | undefined, to: AriaSnapshot): Element | undefined {
+  type ByRoleAndName = Map<string, Map<string, { node: AriaNode, sizeAndPosition: number }>>;
+
+  function fillMap(root: AriaNode, map: ByRoleAndName, position: number) {
+    let size = 1;
+    let childPosition = position + size;
+    for (const child of root.children || []) {
+      if (typeof child === 'string') {
+        size++;
+        childPosition++;
+      } else {
+        size += fillMap(child, map, childPosition);
+        childPosition += size;
+      }
+    }
+    if (!['none', 'presentation', 'fragment', 'iframe', 'generic'].includes(root.role) && root.name) {
+      let byRole = map.get(root.role);
+      if (!byRole) {
+        byRole = new Map();
+        map.set(root.role, byRole);
+      }
+      const existing = byRole.get(root.name);
+      // This heuristic prioritizes elements at the top of the page, even if somewhat smaller.
+      const sizeAndPosition = size * 100 - position;
+      if (!existing || existing.sizeAndPosition < sizeAndPosition)
+        byRole.set(root.name, { node: root, sizeAndPosition });
+    }
+    return size;
+  }
+
+  const fromMap: ByRoleAndName = new Map();
+  if (from)
+    fillMap(from.root, fromMap, 0);
+
+  const toMap: ByRoleAndName = new Map();
+  fillMap(to.root, toMap, 0);
+
+  const result: { node: AriaNode, sizeAndPosition: number }[] = [];
+  for (const [role, byRole] of toMap) {
+    for (const [name, byName] of byRole) {
+      const inFrom = fromMap.get(role)?.get(name);
+      if (!inFrom)
+        result.push(byName);
+    }
+  }
+  result.sort((a, b) => b.sizeAndPosition - a.sizeAndPosition);
+  return result[0]?.node.element;
+}

packages/playwright-core/src/remote/playwrightServer.ts

@@ -107,21 +107,21 @@ export class PlaywrightServer {
         const allowFSPaths = isExtension;
         launchOptions = filterLaunchOptions(launchOptions, allowFSPaths);
 
-        if (process.env.PW_BROWSER_SERVER && url.searchParams.has('connect')) {
-          const filter = url.searchParams.get('connect');
-          if (filter !== 'first')
-            throw new Error(`Unknown connect filter: ${filter}`);
-          return new PlaywrightConnection(
-              browserSemaphore,
-              ws,
-              false,
-              this._playwright,
-              () => this._initConnectMode(id, filter, browserName, launchOptions),
-              id,
-          );
-        }
-
         if (isExtension) {
+          const connectFilter = url.searchParams.get('connect');
+          if (connectFilter) {
+            if (connectFilter !== 'first')
+              throw new Error(`Unknown connect filter: ${connectFilter}`);
+            return new PlaywrightConnection(
+                browserSemaphore,
+                ws,
+                false,
+                this._playwright,
+                () => this._initConnectMode(id, connectFilter, browserName, launchOptions),
+                id,
+            );
+          }
+
           if (url.searchParams.has('debug-controller')) {
             return new PlaywrightConnection(
                 controllerSemaphore,

packages/playwright-core/src/server/bidi/bidiBrowser.ts

@@ -180,7 +180,6 @@ export class BidiBrowser extends Browser {
 export class BidiBrowserContext extends BrowserContext {
   declare readonly _browser: BidiBrowser;
   private _originToPermissions = new Map<string, string[]>();
-  private _blockingPageCreations: Set<Promise<unknown>> = new Set();
   private _initScriptIds = new Map<InitScript, string>();
 
   constructor(browser: BidiBrowser, browserContextId: string | undefined, options: types.BrowserContextOptions) {
@@ -212,30 +211,12 @@ export class BidiBrowserContext extends BrowserContext {
     return this._bidiPages().map(bidiPage => bidiPage._page);
   }
 
-  override async doCreateNewPage(markAsServerSideOnly?: boolean): Promise<Page> {
-    const promise = this._createNewPageImpl(markAsServerSideOnly);
-    if (markAsServerSideOnly)
-      this._blockingPageCreations.add(promise);
-    try {
-      return await promise;
-    } finally {
-      this._blockingPageCreations.delete(promise);
-    }
-  }
-
-  private async _createNewPageImpl(markAsServerSideOnly?: boolean): Promise<Page> {
+  override async doCreateNewPage(): Promise<Page> {
     const { context } = await this._browser._browserSession.send('browsingContext.create', {
       type: bidi.BrowsingContext.CreateType.Window,
       userContext: this._browserContextId,
     });
-    const page = this._browser._bidiPages.get(context)!._page;
-    if (markAsServerSideOnly)
-      page.markAsServerSideOnly();
-    return page;
-  }
-
-  async waitForBlockingPageCreations() {
-    await Promise.all([...this._blockingPageCreations].map(command => command.catch(() => {})));
+    return this._browser._bidiPages.get(context)!._page;
   }
 
   async doGetCookies(urls: string[]): Promise<channels.NetworkCookie[]> {

packages/playwright-core/src/server/bidi/bidiPage.ts

@@ -94,10 +94,6 @@ export class BidiPage implements PageDelegate {
       this.updateRequestInterception(),
       // If the page is created by the Playwright client's call, some initialization
       // may be pending. Wait for it to complete before reporting the page as new.
-      //
-      // TODO: ideally we'd wait only for the commands that created this page, but currently
-      // there is no way in Bidi to track which command created this page.
-      this._browserContext.waitForBlockingPageCreations(),
     ]);
   }
 

packages/playwright-core/src/server/browserContext.ts

@@ -244,7 +244,7 @@ export abstract class BrowserContext extends SdkObject {
 
   // BrowserContext methods.
   abstract possiblyUninitializedPages(): Page[];
-  abstract doCreateNewPage(markAsServerSideOnly?: boolean): Promise<Page>;
+  abstract doCreateNewPage(): Promise<Page>;
   abstract addCookies(cookies: channels.SetNetworkCookie[]): Promise<void>;
   abstract setGeolocation(geolocation?: types.Geolocation): Promise<void>;
   abstract setUserAgent(userAgent: string | undefined): Promise<void>;
@@ -418,7 +418,7 @@ export abstract class BrowserContext extends SdkObject {
       // Workaround for:
       // - chromium fails to change isMobile for existing page;
       // - webkit fails to change locale for existing page.
-      await this.newPage(progress, false);
+      await this.newPage(progress);
       await defaultPage.close();
     }
   }
@@ -538,9 +538,11 @@ export abstract class BrowserContext extends SdkObject {
     await this._closePromise;
   }
 
-  async newPage(progress: Progress, isServerSide: boolean): Promise<Page> {
-    const page = await progress.race(this.doCreateNewPage(isServerSide));
+  async newPage(progress: Progress, forStorageState?: boolean): Promise<Page> {
+    let page: Page | undefined;
     try {
+      this._creatingStorageStatePage = !!forStorageState;
+      page = await progress.race(this.doCreateNewPage());
       const pageOrError = await progress.race(page.waitForInitializedOrError());
       if (pageOrError instanceof Page) {
         if (pageOrError.isClosed())
@@ -549,8 +551,10 @@ export abstract class BrowserContext extends SdkObject {
       }
       throw pageOrError;
     } catch (error) {
-      await page.close({ reason: 'Failed to create page' }).catch(() => {});
+      await page?.close({ reason: 'Failed to create page' }).catch(() => {});
       throw error;
+    } finally {
+      this._creatingStorageStatePage = false;
     }
   }
 
@@ -589,7 +593,7 @@ export abstract class BrowserContext extends SdkObject {
 
     // If there are still origins to save, create a blank page to iterate over origins.
     if (originsToSave.size)  {
-      const page = await this.newPage(progress, true);
+      const page = await this.newPage(progress, true /* forStorageState */);
       try {
         await page.addRequestInterceptor(progress, route => {
           route.fulfill({ body: '<html></html>' }).catch(() => {});
@@ -629,14 +633,8 @@ export abstract class BrowserContext extends SdkObject {
       if (allOrigins.size) {
         if (mode === 'resetForReuse')
           page = this.pages()[0];
-        if (!page) {
-          try {
-            this._creatingStorageStatePage = mode !== 'resetForReuse';
-            page = await this.newPage(progress, this._creatingStorageStatePage);
-          } finally {
-            this._creatingStorageStatePage = false;
-          }
-        }
+        if (!page)
+          page = await this.newPage(progress, mode !== 'resetForReuse' /* forStorageState */);
 
         interceptor = (route: network.Route) => {
           route.fulfill({ body: '<html></html>' }).catch(() => {});

packages/playwright-core/src/server/chromium/crBrowser.ts

@@ -370,12 +370,9 @@ export class CRBrowserContext extends BrowserContext {
     return this._crPages().map(crPage => crPage._page);
   }
 
-  override async doCreateNewPage(markAsServerSideOnly?: boolean): Promise<Page> {
+  override async doCreateNewPage(): Promise<Page> {
     const { targetId } = await this._browser._session.send('Target.createTarget', { url: 'about:blank', browserContextId: this._browserContextId });
-    const page = this._browser._crPages.get(targetId)!._page;
-    if (markAsServerSideOnly)
-      page.markAsServerSideOnly();
-    return page;
+    return this._browser._crPages.get(targetId)!._page;
   }
 
   async doGetCookies(urls: string[]): Promise<channels.NetworkCookie[]> {

packages/playwright-core/src/server/chromium/crPage.ts

@@ -446,16 +446,15 @@ class FrameSession {
   }
 
   async _initialize(hasUIWindow: boolean) {
-    const isSettingStorageState = this._page.browserContext.isCreatingStorageStatePage();
-    if (!isSettingStorageState && hasUIWindow &&
+    if (!this._page.isStorageStatePage && hasUIWindow &&
       !this._crPage._browserContext._browser.isClank() &&
       !this._crPage._browserContext._options.noDefaultViewport) {
       const { windowId } = await this._client.send('Browser.getWindowForTarget');
       this._windowId = windowId;
     }
 
     let screencastOptions: types.PageScreencastOptions | undefined;
-    if (!isSettingStorageState && this._isMainFrame() && this._crPage._browserContext._options.recordVideo && hasUIWindow) {
+    if (!this._page.isStorageStatePage && this._isMainFrame() && this._crPage._browserContext._options.recordVideo && hasUIWindow) {
       const screencastId = createGuid();
       const outputFile = path.join(this._crPage._browserContext._options.recordVideo.dir, screencastId + '.webm');
       screencastOptions = {
@@ -527,7 +526,7 @@ class FrameSession {
         ]
       }),
     ];
-    if (!isSettingStorageState) {
+    if (!this._page.isStorageStatePage) {
       if (this._crPage._browserContext.needsPlaywrightBinding())
         promises.push(this.exposePlaywrightBinding());
       if (this._isMainFrame())

packages/playwright-core/src/server/codegen/language.ts

@@ -22,11 +22,33 @@ import type * as actions from '@recorder/actions';
 export function generateCode(actions: actions.ActionInContext[], languageGenerator: LanguageGenerator, options: LanguageGeneratorOptions) {
   const header = languageGenerator.generateHeader(options);
   const footer = languageGenerator.generateFooter(options.saveStorage);
-  const actionTexts = actions.map(a => languageGenerator.generateAction(a)).filter(Boolean);
+  const actionTexts = actions.map(a => generateActionText(languageGenerator, a)).filter(Boolean) as string[];
   const text = [header, ...actionTexts, footer].join('\n');
   return { header, footer, actionTexts, text };
 }
 
+function generateActionText(generator: LanguageGenerator, action: actions.ActionInContext): string | undefined {
+  let text = generator.generateAction(action);
+  if (!text)
+    return;
+  if (action.action.preconditionSelector) {
+    const expectAction: actions.ActionInContext = {
+      frame: action.frame,
+      startTime: action.startTime,
+      endTime: action.startTime,
+      action: {
+        name: 'assertVisible',
+        selector: action.action.preconditionSelector,
+        signals: [],
+      },
+    };
+    const expectText = generator.generateAction(expectAction);
+    if (expectText)
+      text = expectText + '\n' + text;
+  }
+  return text;
+}
+
 export function sanitizeDeviceOptions(device: any, options: BrowserContextOptions): BrowserContextOptions {
   // Filter out all the properties from the device descriptor.
   const cleanedOptions: Record<string, any> = {};

packages/playwright-core/src/server/debugController.ts

@@ -91,7 +91,7 @@ export class DebugController extends SdkObject {
     if (!pages.length) {
       const [browser] = this._playwright.allBrowsers();
       const context = await browser.newContextForReuse(progress, {});
-      await context.newPage(progress, false /* isServerSide */);
+      await context.newPage(progress);
     }
     // Update test id attribute.
     if (params.testIdAttributeName) {

packages/playwright-core/src/server/dispatchers/browserContextDispatcher.ts

@@ -249,7 +249,7 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
   }
 
   async newPage(params: channels.BrowserContextNewPageParams, progress: Progress): Promise<channels.BrowserContextNewPageResult> {
-    return { page: PageDispatcher.from(this, await this._context.newPage(progress, false /* isServerSide */)) };
+    return { page: PageDispatcher.from(this, await this._context.newPage(progress)) };
   }
 
   async cookies(params: channels.BrowserContextCookiesParams, progress: Progress): Promise<channels.BrowserContextCookiesResult> {