More configurable order for mesh rendering (#7422)

* More configurable order for mesh rendering

* lint

---------

Co-authored-by: Martin Valigursky <mvaligursky@snapchat.com>

Author: mvaligursky

src/scene/constants.js

@@ -164,7 +164,6 @@ export const fresnelNames = {
 // Legacy
 export const LAYER_HUD = 0;
 export const LAYER_GIZMO = 1;
-export const LAYER_FX = 2;
 // 3 - 14 are custom user layers
 export const LAYER_WORLD = 15;
 
@@ -820,9 +819,6 @@ export const SHADOWUPDATE_THISFRAME = 1;
  */
 export const SHADOWUPDATE_REALTIME = 2;
 
-export const SORTKEY_FORWARD = 0;
-export const SORTKEY_DEPTH = 1;
-
 // flags used on the mask property of the Light, and also on mask property of the MeshInstance
 export const MASK_AFFECT_DYNAMIC = 1;
 export const MASK_AFFECT_LIGHTMAPPED = 2;

src/scene/layer.js

@@ -2,8 +2,6 @@ import { Debug } from '../core/debug.js';
 import { hash32Fnv1a } from '../core/hash.js';
 import {
     LIGHTTYPE_DIRECTIONAL,
-    LAYER_FX,
-    SORTKEY_FORWARD,
     SORTMODE_BACK2FRONT, SORTMODE_CUSTOM, SORTMODE_FRONT2BACK, SORTMODE_MATERIALMESH, SORTMODE_NONE
 } from './constants.js';
 import { Material } from './materials/material.js';
@@ -28,20 +26,20 @@ function sortManual(drawCallA, drawCallB) {
 }
 
 function sortMaterialMesh(drawCallA, drawCallB) {
-    const keyA = drawCallA._key[SORTKEY_FORWARD];
-    const keyB = drawCallB._key[SORTKEY_FORWARD];
-    if (keyA === keyB && drawCallA.mesh && drawCallB.mesh) {
+    const keyA = drawCallA._sortKeyForward;
+    const keyB = drawCallB._sortKeyForward;
+    if (keyA === keyB) {
         return drawCallB.mesh.id - drawCallA.mesh.id;
     }
     return keyB - keyA;
 }
 
 function sortBackToFront(drawCallA, drawCallB) {
-    return drawCallB.zdist - drawCallA.zdist;
+    return drawCallB._sortKeyDynamic - drawCallA._sortKeyDynamic;
 }
 
 function sortFrontToBack(drawCallA, drawCallB) {
-    return drawCallA.zdist - drawCallB.zdist;
+    return drawCallA._sortKeyDynamic - drawCallB._sortKeyDynamic;
 }
 
 const sortCallbacks = [null, sortManual, sortMaterialMesh, sortBackToFront, sortFrontToBack];
@@ -797,24 +795,32 @@ class Layer {
 
     /**
      * @param {MeshInstance[]} drawCalls - Array of mesh instances.
-     * @param {number} drawCallsCount - Number of mesh instances.
      * @param {Vec3} camPos - Camera position.
      * @param {Vec3} camFwd - Camera forward vector.
      * @private
      */
-    _calculateSortDistances(drawCalls, drawCallsCount, camPos, camFwd) {
-        for (let i = 0; i < drawCallsCount; i++) {
+    _calculateSortDistances(drawCalls, camPos, camFwd) {
+        const count = drawCalls.length;
+        const { x: px, y: py, z: pz } = camPos;
+        const { x: fx, y: fy, z: fz } = camFwd;
+
+        for (let i = 0; i < count; i++) {
             const drawCall = drawCalls[i];
-            if (drawCall.layer <= LAYER_FX) continue; // Only alpha sort mesh instances in the main world (backwards comp)
+
+            // compute distance from camera to mesh along the forward vector
+            let zDist;
             if (drawCall.calculateSortDistance) {
-                drawCall.zdist = drawCall.calculateSortDistance(drawCall, camPos, camFwd);
-                continue;
+                zDist = drawCall.calculateSortDistance(drawCall, camPos, camFwd);
+            } else {
+                const meshPos = drawCall.aabb.center;
+                zDist = (meshPos.x - px) * fx + (meshPos.y - py) * fy + (meshPos.z - pz) * fz;
             }
-            const meshPos = drawCall.aabb.center;
-            const tempx = meshPos.x - camPos.x;
-            const tempy = meshPos.y - camPos.y;
-            const tempz = meshPos.z - camPos.z;
-            drawCall.zdist = tempx * camFwd.x + tempy * camFwd.y + tempz * camFwd.z;
+
+            // scale the bucket to give it a significantly higher magnitude than distance (1 billion)
+            const bucket = drawCall._drawBucket * 1e9;
+
+            // create sorting key based on the drawBucket and distance
+            drawCall._sortKeyDynamic = bucket + zDist;
         }
     }
 
@@ -840,6 +846,7 @@ class Layer {
      * @ignore
      */
     sortVisible(camera, transparent) {
+
         const sortMode = transparent ? this.transparentSortMode : this.opaqueSortMode;
         if (sortMode === SORTMODE_NONE) {
             return;
@@ -863,7 +870,7 @@ class Layer {
             if (sortMode === SORTMODE_BACK2FRONT || sortMode === SORTMODE_FRONT2BACK) {
                 const sortPos = cameraNode.getPosition();
                 const sortDir = cameraNode.forward;
-                this._calculateSortDistances(instances, instances.length, sortPos, sortDir);
+                this._calculateSortDistances(instances, sortPos, sortDir);
             }
 
             instances.sort(sortCallbacks[sortMode]);

src/scene/mesh-instance.js

@@ -4,14 +4,12 @@ import { BoundingSphere } from '../core/shape/bounding-sphere.js';
 import { BindGroup } from '../platform/graphics/bind-group.js';
 import { UniformBuffer } from '../platform/graphics/uniform-buffer.js';
 import {
-    BLEND_NONE, BLEND_NORMAL,
     LAYER_WORLD,
     MASK_AFFECT_DYNAMIC, MASK_BAKE, MASK_AFFECT_LIGHTMAPPED,
     RENDERSTYLE_SOLID,
     SHADERDEF_UV0, SHADERDEF_UV1, SHADERDEF_VCOLOR, SHADERDEF_TANGENTS, SHADERDEF_NOSHADOW, SHADERDEF_SKIN,
     SHADERDEF_SCREENSPACE, SHADERDEF_MORPH_POSITION, SHADERDEF_MORPH_NORMAL, SHADERDEF_BATCH,
-    SHADERDEF_LM, SHADERDEF_DIRLM, SHADERDEF_LMAMBIENT, SHADERDEF_INSTANCING, SHADERDEF_MORPH_TEXTURE_BASED_INT,
-    SORTKEY_FORWARD
+    SHADERDEF_LM, SHADERDEF_DIRLM, SHADERDEF_LMAMBIENT, SHADERDEF_INSTANCING, SHADERDEF_MORPH_TEXTURE_BASED_INT
 } from './constants.js';
 import { GraphNode } from './graph-node.js';
 import { getDefaultMaterial } from './materials/default-material.js';
@@ -208,6 +206,12 @@ class MeshInstance {
      */
     drawOrder = 0;
 
+    /**
+     * @type {number}
+     * @ignore
+     */
+    _drawBucket = 127;
+
     /**
      * The graph node defining the transform for this instance.
      *
@@ -322,8 +326,28 @@ class MeshInstance {
     /** @private */
     _updateAabbFunc = null;
 
-    /** @private */
-    _key = [0, 0];
+    /**
+     * The internal sorting key used by the shadow renderer.
+     *
+     * @ignore
+     */
+    _sortKeyShadow = 0;
+
+    /**
+     * The internal sorting key used by the forward renderer, in case SORTMODE_MATERIALMESH sorting
+     * is used.
+     *
+     * @private
+     */
+    _sortKeyForward = 0;
+
+    /**
+     * The internal sorting key used by the forward renderer, in case SORTMODE_BACK2FRONT or
+     * SORTMODE_FRONT2BACK sorting is used.
+     *
+     * @ignore
+     */
+    _sortKeyDynamic = 0;
 
     /** @private */
     _layer = LAYER_WORLD;
@@ -419,6 +443,31 @@ class MeshInstance {
         this.updateKey();
     }
 
+    /**
+     * Sets the draw bucket for mesh instances. The draw bucket, an integer from 0 to 255 (default
+     * 127), serves as the primary sort key for mesh rendering. Meshes are sorted in ascending order
+     * by draw bucket, then by sort mode. This setting is only effective when mesh instances are
+     * added to a {@link Layer} with its {@link Layer#opaqueSortMode} or
+     * {@link Layer#transparentSortMode} (depending on the material) set to
+     * {@link SORTMODE_BACK2FRONT}, {@link SORTMODE_FRONT2BACK}, or {@link SORTMODE_MATERIALMESH}.
+     *
+     * @type {number}
+     */
+    set drawBucket(bucket) {
+        // 8bit integer
+        this._drawBucket = Math.floor(bucket) & 0xff;
+        this.updateKey();
+    }
+
+    /**
+     * Gets the draw bucket for mesh instance.
+     *
+     * @type {number}
+     */
+    get drawBucket() {
+        return this._drawBucket;
+    }
+
     /**
      * Sets the render style of the mesh instance. Can be:
      *
@@ -702,15 +751,6 @@ class MeshInstance {
         return this._material;
     }
 
-    set layer(layer) {
-        this._layer = layer;
-        this.updateKey();
-    }
-
-    get layer() {
-        return this._layer;
-    }
-
     /**
      * @param {number} shaderDefs - The shader definitions to set.
      * @private
@@ -827,11 +867,11 @@ class MeshInstance {
     }
 
     set key(val) {
-        this._key[SORTKEY_FORWARD] = val;
+        this._sortKeyForward = val;
     }
 
     get key() {
-        return this._key[SORTKEY_FORWARD];
+        return this._sortKeyForward;
     }
 
     /**
@@ -965,21 +1005,15 @@ class MeshInstance {
 
     updateKey() {
 
-        // render alphatest/atoc after opaque
-        const material = this.material;
-        const blendType = (material.alphaToCoverage || material.alphaTest) ? BLEND_NORMAL : material.blendType;
-
-        // Key definition:
-        // Bit
         // 31      : sign bit (leave)
-        // 27 - 30 : layer
-        // 26      : translucency type (opaque/transparent)
-        // 25      : unused
-        // 0 - 24  : Material ID (if opaque) or 0 (if transparent - will be depth)
-        this._key[SORTKEY_FORWARD] =
-            ((this.layer & 0x0f) << 27) |
-            ((blendType === BLEND_NONE ? 1 : 0) << 26) |
-            ((material.id & 0x1ffffff) << 0);
+        // 30 - 25 : 8 bits for draw bucket - this is the highest priority for sorting
+        // 24      : 1 bit for alpha test / coverage, to render them after opaque to keep GPU efficiency
+        // 23 - 0  : 24 bits for material ID
+        const { material } = this;
+        this._sortKeyForward =
+            (this._drawBucket << 25) |
+            ((material.alphaToCoverage || material.alphaTest) ? 0x1000000 : 0) |
+            (material.id & 0xffffff);
     }
 
     /**

src/scene/renderer/renderer.js

@@ -22,7 +22,6 @@ import { BindGroup, DynamicBindGroup } from '../../platform/graphics/bind-group.
 import { UniformFormat, UniformBufferFormat } from '../../platform/graphics/uniform-buffer-format.js';
 import { BindGroupFormat, BindUniformBufferFormat, BindTextureFormat } from '../../platform/graphics/bind-group-format.js';
 import {
-    SORTKEY_DEPTH, SORTKEY_FORWARD,
     VIEW_CENTER, PROJECTION_ORTHOGRAPHIC,
     LIGHTTYPE_DIRECTIONAL, MASK_AFFECT_DYNAMIC, MASK_AFFECT_LIGHTMAPPED, MASK_BAKE,
     SHADOWUPDATE_NONE, SHADOWUPDATE_THISFRAME,
@@ -266,50 +265,6 @@ class Renderer {
         this.lightTextureAtlas = null;
     }
 
-    sortCompare(drawCallA, drawCallB) {
-        if (drawCallA.layer === drawCallB.layer) {
-            if (drawCallA.drawOrder && drawCallB.drawOrder) {
-                return drawCallA.drawOrder - drawCallB.drawOrder;
-            } else if (drawCallA.zdist && drawCallB.zdist) {
-                return drawCallB.zdist - drawCallA.zdist; // back to front
-            } else if (drawCallA.zdist2 && drawCallB.zdist2) {
-                return drawCallA.zdist2 - drawCallB.zdist2; // front to back
-            }
-        }
-
-        return drawCallB._key[SORTKEY_FORWARD] - drawCallA._key[SORTKEY_FORWARD];
-    }
-
-    sortCompareMesh(drawCallA, drawCallB) {
-        if (drawCallA.layer === drawCallB.layer) {
-            if (drawCallA.drawOrder && drawCallB.drawOrder) {
-                return drawCallA.drawOrder - drawCallB.drawOrder;
-            } else if (drawCallA.zdist && drawCallB.zdist) {
-                return drawCallB.zdist - drawCallA.zdist; // back to front
-            }
-        }
-
-        const keyA = drawCallA._key[SORTKEY_FORWARD];
-        const keyB = drawCallB._key[SORTKEY_FORWARD];
-
-        if (keyA === keyB && drawCallA.mesh && drawCallB.mesh) {
-            return drawCallB.mesh.id - drawCallA.mesh.id;
-        }
-
-        return keyB - keyA;
-    }
-
-    sortCompareDepth(drawCallA, drawCallB) {
-        const keyA = drawCallA._key[SORTKEY_DEPTH];
-        const keyB = drawCallB._key[SORTKEY_DEPTH];
-
-        if (keyA === keyB && drawCallA.mesh && drawCallB.mesh) {
-            return drawCallB.mesh.id - drawCallA.mesh.id;
-        }
-
-        return keyB - keyA;
-    }
-
     /**
      * Set up the viewport and the scissor for camera rendering.
      *

src/scene/renderer/shadow-renderer.js

@@ -12,7 +12,6 @@ import {
     LIGHTTYPE_DIRECTIONAL, LIGHTTYPE_OMNI,
     SHADER_SHADOW,
     SHADOWUPDATE_NONE, SHADOWUPDATE_THISFRAME,
-    SORTKEY_DEPTH,
     shadowTypeInfo
 } from '../constants.js';
 import { ShaderPass } from '../shader-pass.js';
@@ -192,7 +191,18 @@ class ShadowRenderer {
         }
 
         // this sorts the shadow casters by the shader id
-        visible.sort(this.renderer.sortCompareDepth);
+        visible.sort(this.sortCompareShader);
+    }
+
+    sortCompareShader(drawCallA, drawCallB) {
+        const keyA = drawCallA._sortKeyShadow;
+        const keyB = drawCallB._sortKeyShadow;
+
+        if (keyA === keyB) {
+            return drawCallB.mesh.id - drawCallA.mesh.id;
+        }
+
+        return keyB - keyA;
     }
 
     setupRenderState(device, light) {
@@ -316,7 +326,7 @@ class ShadowRenderer {
             Debug.assert(shadowShader, `no shader for pass ${shadowPass}`, material);
 
             // sort shadow casters by shader
-            meshInstance._key[SORTKEY_DEPTH] = shadowShader.id;
+            meshInstance._sortKeyShadow = shadowShader.id;
 
             device.setShader(shadowShader);