fix(getContainingBlock): reorder isTopLayer check (floating-ui#2979)

Co-authored-by: GitHub Actions <github-actions@github.com>

Author: atomiks

.changeset/khaki-beds-lie.md

@@ -0,0 +1,5 @@
+---
+'@floating-ui/utils': patch
+---
+
+fix(getContainingBlock): reorder `isTopLayer` check. Fixes regression when a top layer element like `<dialog>` is a containing block (e.g. it has a `transform` style) and a floating element is being positioned inside of it.

packages/dom/test/functional/top-layer.test.ts

@@ -245,3 +245,14 @@ test('fixed inside dialog with outer containing block', async ({page}) => {
     `top-layer.dialog.outer.png`,
   );
 });
+
+test('top layer containing block with inner floating element', async ({
+  page,
+}) => {
+  await page.goto('http://localhost:1234/top-layer');
+  await click(page, '[data-testid="inner-true"]');
+
+  expect(await page.locator('#inner-dialog').screenshot()).toMatchSnapshot(
+    `top-layer.dialog.inner.png`,
+  );
+});

packages/dom/test/functional/top-layer.test.ts-snapshots/top-layer-dialog-inner-linux.png

@@ -30,16 +30,40 @@ function OuterTopLayer() {
   }, []);
 
   return (
-    <>
-      <div style={{containerType: 'inline-size'}}>
-        <dialog id="outer-dialog" ref={handleDialogRef}>
-          <button ref={refs.setReference}>My button</button>
-          <div ref={refs.setFloating} style={floatingStyles}>
-            My tooltip
-          </div>
-        </dialog>
+    <div style={{containerType: 'inline-size'}}>
+      <dialog id="outer-dialog" ref={handleDialogRef}>
+        <button ref={refs.setReference}>My button</button>
+        <div ref={refs.setFloating} style={floatingStyles}>
+          My tooltip
+        </div>
+      </dialog>
+    </div>
+  );
+}
+
+function InnerTopLayer() {
+  const {refs, floatingStyles} = useFloating({
+    strategy: 'fixed',
+    whileElementsMounted: autoUpdate,
+  });
+
+  const handleDialogRef = useCallback((node: HTMLDialogElement | null) => {
+    if (node) {
+      node.showModal();
+    }
+  }, []);
+
+  return (
+    <dialog
+      id="inner-dialog"
+      ref={handleDialogRef}
+      style={{containerType: 'inline-size', width: 300, height: 300}}
+    >
+      <button ref={refs.setReference}>My button</button>
+      <div ref={refs.setFloating} style={floatingStyles}>
+        My tooltip
       </div>
-    </>
+    </dialog>
   );
 }
 
@@ -174,6 +198,7 @@ export function TopLayer() {
   const [layoutStyles, setLayoutStyles] = useState(true);
   const [strategy, setStrategy] = useState<'absolute' | 'fixed'>('fixed');
   const [outer, setOuter] = useState(false);
+  const [inner, setInner] = useState(false);
 
   const {refs, floatingStyles, x, y} = useFloating({
     strategy,
@@ -222,6 +247,7 @@ export function TopLayer() {
         Top Layer
       </h1>
       {outer && <OuterTopLayer />}
+      {inner && <InnerTopLayer />}
       <Stack {...stackProps}>
         <div
           className={classes}
@@ -406,6 +432,22 @@ export function TopLayer() {
           </button>
         ))}
       </Controls>
+
+      <h2>inner</h2>
+      <Controls>
+        {BOOLS.map((bool) => (
+          <button
+            key={String(bool)}
+            data-testid={`inner-${bool}`}
+            onClick={() => setInner(bool)}
+            style={{
+              backgroundColor: bool === inner ? 'black' : '',
+            }}
+          >
+            {String(bool)}
+          </button>
+        ))}
+      </Controls>
     </>
   );
 }

packages/dom/test/visual/spec/TopLayer.tsx

@@ -92,12 +92,10 @@ export function getContainingBlock(element: Element): HTMLElement | null {
   let currentNode: Node | null = getParentNode(element);
 
   while (isHTMLElement(currentNode) && !isLastTraversableNode(currentNode)) {
-    if (isTopLayer(currentNode)) {
-      return null;
-    }
-
     if (isContainingBlock(currentNode)) {
       return currentNode;
+    } else if (isTopLayer(currentNode)) {
+      return null;
     }
 
     currentNode = getParentNode(currentNode);