Fix SortPolicy sorting key generation

Change-Id: Ib06d84088d93e00ffdcaa6baa9d34e03358943cb
Reviewed-by: Svenn-Arne Dragly <svenn-arne.dragly@qt.io>
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

src/render/backend/renderview.cpp

@@ -557,7 +557,12 @@ QVector<RenderCommand *> RenderView::buildDrawRenderCommands(const QVector<Entit
             for (const RenderPassParameterData &passData : renderPassData) {
                 // Add the RenderPass Parameters
                 RenderCommand *command = new RenderCommand();
-                command->m_depth = m_data.m_eyePos.distanceToPoint(node->worldBoundingVolume()->center());
+
+                // Project the camera-to-object-center vector onto the camera
+                // view vector. This gives a depth value suitable as the key
+                // for BackToFront sorting.
+                command->m_depth = QVector3D::dotProduct(node->worldBoundingVolume()->center() - m_data.m_eyePos, m_data.m_eyeViewDir);
+
                 command->m_geometry = geometryHandle;
                 command->m_geometryRenderer = geometryRendererHandle;
                 command->m_material = materialHandle;
@@ -716,6 +721,12 @@ void RenderView::updateMatrices()
         const QMatrix4x4 inverseWorldTransform = viewMatrix().inverted();
         const QVector3D eyePosition(inverseWorldTransform.column(3));
         setEyePosition(eyePosition);
+
+        // Get the viewing direction of the camera. Use the normal matrix to
+        // ensure non-uniform scale works too.
+        QMatrix3x3 normalMat = m_data.m_viewMatrix.normalMatrix();
+        // dir = normalize(QVector3D(0, 0, -1) * normalMat)
+        setEyeViewDirection(QVector3D(-normalMat(2, 0), -normalMat(2, 1), -normalMat(2, 2)).normalized());
     }
 }
 

src/render/backend/renderview_p.h

@@ -148,6 +148,9 @@ public:
     inline void setEyePosition(const QVector3D &eyePos) Q_DECL_NOTHROW { m_data.m_eyePos = eyePos; }
     inline QVector3D eyePosition() const Q_DECL_NOTHROW { return m_data.m_eyePos; }
 
+    inline void setEyeViewDirection(const QVector3D &dir) Q_DECL_NOTHROW { m_data.m_eyeViewDir = dir; }
+    inline QVector3D eyeViewDirection() const Q_DECL_NOTHROW { return m_data.m_eyeViewDir; }
+
     inline void setHasLayerFilter(bool filter) Q_DECL_NOTHROW { m_data.m_hasLayerFilter = filter; }
     inline bool hasLayerFilter() const Q_DECL_NOTHROW { return m_data.m_hasLayerFilter; }
     inline void appendLayerFilter(const Qt3DCore::QNodeIdVector &layerIds) Q_DECL_NOTHROW { m_data.m_layerIds << layerIds; }
@@ -247,6 +250,7 @@ public:
         Qt3DCore::QNodeIdVector m_layerIds;
         QVector<Qt3DRender::QSortPolicy::SortType> m_sortingTypes;
         QVector3D m_eyePos;
+        QVector3D m_eyeViewDir;
     };
 
     bool isDownloadBuffersEnable() const;

src/render/framegraph/qsortpolicy.cpp

@@ -88,10 +88,20 @@ QSortPolicyPrivate::QSortPolicyPrivate()
 /*!
     \enum QSortPolicy::SortType
 
-    This enum type describes sort types that can be employed
-    \value StateChangeCost sort the objects so as to minimize the cost of changing from the currently rendered state
-    \value BackToFront sort the objects from back to front inverted z order
+    This enum type describes the available sort types.
+
+    \value StateChangeCost sort the objects so as to minimize the cost of
+    changing from the currently rendered state
+
+    \value BackToFront sort the objects from back to front based on inverted z
+    order. More accurately, the sorting key is the z component of the
+    projection of the camera-to-object-center vector onto the camera's view
+    vector.
+
     \value Material sort the objects based on their material value
+
+    \value FrontToBack sort the objects from front to back. The opposite of
+    BackToFront.
 */
 
 /*!