SL-19808: Fix orientation of PBR normal texture when texture transform and/or texture animation is applied

3 files changed, 52 insertions, 6 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/pbralphaV.glsl b/indra/newview/app_settings/shaders/class1/deferred/pbralphaV.glsl
index e9515a9187..81482e2954 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/pbralphaV.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/pbralphaV.glsl
@@ -71,6 +71,7 @@ flat out float vary_sign;
 out vec3 vary_normal;
 
 vec2 texture_transform(vec2 vertex_texcoord, vec4[2] khr_gltf_transform, mat4 sl_animation_transform);
+vec3 tangent_space_transform(vec4 vertex_tangent, vec3 vertex_normal, vec4[2] khr_gltf_transform, mat4 sl_animation_transform);
 
 
 void main()
@@ -94,12 +95,13 @@ void main()
     metallic_roughness_texcoord = texture_transform(texcoord0, texture_metallic_roughness_transform, texture_matrix0);
     emissive_texcoord = texture_transform(texcoord0, texture_emissive_transform, texture_matrix0);
 
+    vec3 tex_tangent = tangent_space_transform(tangent, normal.xyz, texture_normal_transform, texture_matrix0);
 #ifdef HAS_SKIN
 	vec3 n = (mat*vec4(normal.xyz+position.xyz,1.0)).xyz-pos.xyz;
-	vec3 t = (mat*vec4(tangent.xyz+position.xyz,1.0)).xyz-pos.xyz;
+	vec3 t = (mat*vec4(tex_tangent.xyz+position.xyz,1.0)).xyz-pos.xyz;
 #else //HAS_SKIN
 	vec3 n = normal_matrix * normal;
-  	vec3 t = normal_matrix * tangent.xyz;
+  	vec3 t = normal_matrix * tex_tangent.xyz;
 #endif //HAS_SKIN
 
     vary_tangent = normalize(t);
diff --git a/indra/newview/app_settings/shaders/class1/deferred/pbropaqueV.glsl b/indra/newview/app_settings/shaders/class1/deferred/pbropaqueV.glsl
index e2c23ac8f0..2ad2a015fc 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/pbropaqueV.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/pbropaqueV.glsl
@@ -61,6 +61,7 @@ flat out float vary_sign;
 out vec3 vary_normal;
 
 vec2 texture_transform(vec2 vertex_texcoord, vec4[2] khr_gltf_transform, mat4 sl_animation_transform);
+vec3 tangent_space_transform(vec4 vertex_tangent, vec3 vertex_normal, vec4[2] khr_gltf_transform, mat4 sl_animation_transform);
 
 void main()
 {
@@ -83,12 +84,13 @@ void main()
     metallic_roughness_texcoord = texture_transform(texcoord0, texture_metallic_roughness_transform, texture_matrix0);
     emissive_texcoord = texture_transform(texcoord0, texture_emissive_transform, texture_matrix0);
 
+    vec3 tex_tangent = tangent_space_transform(tangent, normal.xyz, texture_normal_transform, texture_matrix0);
 #ifdef HAS_SKIN
 	vec3 n = (mat*vec4(normal.xyz+position.xyz,1.0)).xyz-pos.xyz;
-	vec3 t = (mat*vec4(tangent.xyz+position.xyz,1.0)).xyz-pos.xyz;
+	vec3 t = (mat*vec4(tex_tangent.xyz+position.xyz,1.0)).xyz-pos.xyz;
 #else //HAS_SKIN
 	vec3 n = normal_matrix * normal;
-	vec3 t = normal_matrix * tangent.xyz;
+	vec3 t = normal_matrix * tex_tangent.xyz;
 #endif
 
     vary_tangent = normalize(t);
@@ -107,8 +109,6 @@ uniform mat4 modelview_projection_matrix;
 uniform mat4 texture_matrix0;
 
 uniform vec4[2] texture_base_color_transform;
-uniform vec4[2] texture_normal_transform;
-uniform vec4[2] texture_metallic_roughness_transform;
 uniform vec4[2] texture_emissive_transform;
 
 in vec3 position;
diff --git a/indra/newview/app_settings/shaders/class1/deferred/textureUtilV.glsl b/indra/newview/app_settings/shaders/class1/deferred/textureUtilV.glsl
index 71f7ec52c4..636dfed4ba 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/textureUtilV.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/textureUtilV.glsl
@@ -76,3 +76,47 @@ vec2 texture_transform(vec2 vertex_texcoord, vec4[2] khr_gltf_transform, mat4 sl
 
     return texcoord;
 }
+
+// Take the rotation only from both transforms and apply to the tangent. This
+// accounts for the change of the topology of the normal texture when a texture
+// rotation is applied to it.
+// *HACK: Assume the imported GLTF model did not have both normal texture
+// transforms and tangent vertices. The use of this function is inconsistent
+// with the GLTF sample viewer when that is the case. See getNormalInfo in
+// https://raw.githubusercontent.com/KhronosGroup/glTF-Sample-Viewer/47a191931461a6f2e14de48d6da0f0eb6ec2d147/source/Renderer/shaders/material_info.glsl
+// We may want to account for this case during GLTF model import.
+// -Cosmic,2023-06-06
+vec3 tangent_space_transform(vec4 vertex_tangent, vec3 vertex_normal, vec4[2] khr_gltf_transform, mat4 sl_animation_transform)
+{
+    vec2 weights = vec2(0, 1);
+
+    // Apply texture animation first to avoid shearing and other artifacts (rotation only)
+    mat2 sl_rot_scale;
+    sl_rot_scale[0][0] = sl_animation_transform[0][0];
+    sl_rot_scale[0][1] = sl_animation_transform[0][1];
+    sl_rot_scale[1][0] = sl_animation_transform[1][0];
+    sl_rot_scale[1][1] = sl_animation_transform[1][1];
+    weights = sl_rot_scale * weights;
+    // Remove scale
+    weights = normalize(weights);
+
+    // Convert to left-handed coordinate system
+    weights.y = -weights.y;
+
+    // Apply KHR_texture_transform (rotation only)
+    float khr_rotation = khr_gltf_transform[0].z;
+    mat2 khr_rotation_mat = mat2(
+        cos(khr_rotation),-sin(khr_rotation),
+        sin(khr_rotation), cos(khr_rotation)
+    );
+    weights = khr_rotation_mat * weights;
+
+    // Convert back to right-handed coordinate system
+    weights.y = -weights.y;
+
+    // Similar to the MikkTSpace-compatible method of extracting the binormal
+    // from the normal and tangent, as seen in the fragment shader
+    vec3 vertex_binormal = vertex_tangent.w * cross(vertex_normal, vertex_tangent.xyz);
+
+    return (weights.x * vertex_binormal.xyz) + (weights.y * vertex_tangent.xyz);
+}