SL-13490 fix debug normals under non-uniform scale

2 files changed, 87 insertions, 41 deletions

diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml
index 0b9cc78132..62ceb9077a 100644
--- a/indra/newview/app_settings/settings.xml
+++ b/indra/newview/app_settings/settings.xml
@@ -9091,7 +9091,7 @@
     <key>Type</key>
     <string>F32</string>
     <key>Value</key>
-    <real>0.03</real>
+    <real>0.1</real>
   </map>
   <key>RenderDebugPipeline</key>
     <map>
diff --git a/indra/newview/llspatialpartition.cpp b/indra/newview/llspatialpartition.cpp
index 8fc2405f0a..b87551dab5 100644
--- a/indra/newview/llspatialpartition.cpp
+++ b/indra/newview/llspatialpartition.cpp
@@ -2253,52 +2253,98 @@ void renderBoundingBox(LLDrawable* drawable, BOOL set_color = TRUE)
 	}
 }
 
-void renderNormals(LLDrawable* drawablep)
+void renderNormals(LLDrawable *drawablep)
 {
-	LLVertexBuffer::unbind();
+    if (!drawablep->isVisible())
+        return;
 
-	LLVOVolume* vol = drawablep->getVOVolume();
-	if (vol)
-	{
-		LLVolume* volume = vol->getVolume();
-		gGL.pushMatrix();
-		gGL.multMatrix((F32*) vol->getRelativeXform().mMatrix);
-		
-		gGL.getTexUnit(0)->unbind(LLTexUnit::TT_TEXTURE);
+    LLVertexBuffer::unbind();
 
-		LLVector4a scale(gSavedSettings.getF32("RenderDebugNormalScale"));
+    LLVOVolume *vol = drawablep->getVOVolume();
 
-		for (S32 i = 0; i < volume->getNumVolumeFaces(); ++i)
-		{
-			const LLVolumeFace& face = volume->getVolumeFace(i);
+    if (vol)
+    {
+        LLVolume *volume = vol->getVolume();
 
-			for (S32 j = 0; j < face.mNumVertices; ++j)
-			{
-				gGL.begin(LLRender::LINES);
-				LLVector4a n,p;
-				
-				n.setMul(face.mNormals[j], scale);
-				p.setAdd(face.mPositions[j], n);
-				
-				gGL.diffuseColor4f(1,1,1,1);
-				gGL.vertex3fv(face.mPositions[j].getF32ptr());
-				gGL.vertex3fv(p.getF32ptr());
-				
-				if (face.mTangents)
-				{
-					n.setMul(face.mTangents[j], scale);
-					p.setAdd(face.mPositions[j], n);
-				
-					gGL.diffuseColor4f(0,1,1,1);
-					gGL.vertex3fv(face.mPositions[j].getF32ptr());
-					gGL.vertex3fv(p.getF32ptr());
-				}	
-				gGL.end();
-			}
-		}
+        // Drawable's normals & tangents are stored in model space, i.e. before any scaling is applied.
+        //
+        // SL-13490, using pos + normal to compute the 2nd vertex of a normal line segment doesn't
+        // work when there's a non-uniform scale in the mix. Normals require MVP-inverse-transpose
+        // transform. We get that effect here by pre-applying the inverse scale (twice, because
+        // one forward scale will be re-applied via the MVP in the vertex shader)
 
-		gGL.popMatrix();
-	}
+        LLVector3  scale_v3 = vol->getScale();
+        LLVector4a obj_scale(scale_v3.mV[VX], scale_v3.mV[VY], scale_v3.mV[VZ]);
+        obj_scale.normalize3();
+
+        float draw_length = gSavedSettings.getF32("RenderDebugNormalScale");
+
+        // Create inverse-scale vector for normals
+        LLVector4a inv_scale(1.0 / scale_v3.mV[VX], 1.0 / scale_v3.mV[VY], 1.0 / scale_v3.mV[VZ]);
+        inv_scale.mul(inv_scale);  // Squared, to apply inverse scale twice
+        inv_scale.normalize3fast();
+
+        gGL.pushMatrix();
+        gGL.multMatrix((F32 *) vol->getRelativeXform().mMatrix);
+
+        gGL.getTexUnit(0)->unbind(LLTexUnit::TT_TEXTURE);
+
+        for (S32 i = 0; i < volume->getNumVolumeFaces(); ++i)
+        {
+            const LLVolumeFace &face = volume->getVolumeFace(i);
+
+            gGL.flush();
+            gGL.diffuseColor4f(1, 1, 0, 1);
+            gGL.begin(LLRender::LINES);
+            for (S32 j = 0; j < face.mNumVertices; ++j)
+            {
+                LLVector4a n, p;
+
+                n.setMul(face.mNormals[j], 1.0);
+                n.mul(inv_scale);  // Pre-scale normal, so it's left with an inverse-transpose xform after MVP
+                n.normalize3fast();
+
+                // Since we send 2 vertices instead of a vertex and a vector, the drawn normal length ends up
+                // getting stretched along with the object. To minimize that effect (imperfectly), reduce its
+                // length by a dot factor with the dominant scale direction.
+                float mvp_scale_factor = 0.95 * abs(n.dot3(obj_scale).getF32());
+                n.mul((1.0 - mvp_scale_factor) * draw_length);
+                p.setAdd(face.mPositions[j], n);
+
+                gGL.vertex3fv(face.mPositions[j].getF32ptr());
+                gGL.vertex3fv(p.getF32ptr());
+            }
+            gGL.end();
+
+            // Tangents are simple vectors and do not require reorientation via pre-scaling
+            if (face.mTangents)
+            {
+                gGL.flush();
+                gGL.diffuseColor4f(0, 1, 1, 1);
+                gGL.begin(LLRender::LINES);
+                for (S32 j = 0; j < face.mNumVertices; ++j)
+                {
+                    LLVector4a t, p;
+
+                    t.setMul(face.mTangents[j], 1.0f);
+                    t.normalize3fast();
+
+                    // Since we send 2 vertices instead of a vertex and a vector, the drawn tangent length ends up
+                    // getting stretched along with the object. To minimize that effect (imperfectly), reduce its
+                    // length by a dot factor with the dominant scale direction.
+                    float mvp_scale_factor = 0.95 * abs(t.dot3(obj_scale).getF32());
+                    t.mul((1.0 - mvp_scale_factor) * draw_length);
+                    p.setAdd(face.mPositions[j], t);
+
+                    gGL.vertex3fv(face.mPositions[j].getF32ptr());
+                    gGL.vertex3fv(p.getF32ptr());
+                }
+                gGL.end();
+            }
+        }
+
+        gGL.popMatrix();
+    }
 }
 
 S32 get_physics_detail(const LLVolumeParams& volume_params, const LLVector3& scale)