DRTVWR-592: (WIP) PBR Terrain: Improve orientation of textures and normal maps for triplanar mapping, minor cleanup

4 files changed, 93 insertions, 39 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/pbrterrainF.glsl b/indra/newview/app_settings/shaders/class1/deferred/pbrterrainF.glsl
index b9073a9361..7febbe280e 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/pbrterrainF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/pbrterrainF.glsl
@@ -35,7 +35,7 @@ out vec4 frag_data[4];
 
 uniform sampler2D alpha_ramp;
 
-// *TODO: Configurable quality level which disables PBR features on machines
+// *TODO: More configurable quality level which disables PBR features on machines
 // with limited texture availability
 // https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html#additional-textures
 uniform sampler2D detail_0_base_color;
@@ -57,7 +57,6 @@ uniform sampler2D detail_2_emissive;
 uniform sampler2D detail_3_emissive;
 #endif
 
-// *TODO: More efficient packing?
 uniform vec4[4] baseColorFactors; // See also vertex_color in pbropaqueV.glsl
 uniform vec4 metallicFactors;
 uniform vec4 roughnessFactors;
@@ -66,7 +65,9 @@ uniform vec3[4] emissiveColors;
 #endif
 uniform vec4 minimum_alphas; // PBR alphaMode: MASK, See: mAlphaCutoff, setAlphaCutoff()
 
+#if TERRAIN_PLANAR_TEXTURE_SAMPLE_COUNT == 3
 in vec4[2] vary_coords;
+#endif
 in vec3 vary_normal;
 in vec3 vary_tangent;
 flat in float vary_sign;
@@ -79,7 +80,7 @@ float terrain_mix(vec4 samples, float alpha1, float alpha2, float alphaFinal);
 vec3 sample_and_mix_color3(float alpha1, float alpha2, float alphaFinal, TerrainCoord texcoord, vec3[4] factors, sampler2D tex0, sampler2D tex1, sampler2D tex2, sampler2D tex3);
 vec4 sample_and_mix_color4(float alpha1, float alpha2, float alphaFinal, TerrainCoord texcoord, vec4[4] factors, sampler2D tex0, sampler2D tex1, sampler2D tex2, sampler2D tex3);
 vec3 sample_and_mix_vector3(float alpha1, float alpha2, float alphaFinal, TerrainCoord texcoord, vec3[4] factors, sampler2D tex0, sampler2D tex1, sampler2D tex2, sampler2D tex3);
-vec3 sample_and_mix_vector3_no_scale(float alpha1, float alpha2, float alphaFinal, TerrainCoord texcoord, sampler2D tex0, sampler2D tex1, sampler2D tex2, sampler2D tex3);
+vec3 sample_and_mix_normal(float alpha1, float alpha2, float alphaFinal, TerrainCoord texcoord, sampler2D tex0, sampler2D tex1, sampler2D tex2, sampler2D tex3);
 
 void main()
 {
@@ -101,7 +102,6 @@ void main()
         discard;
     }
 
-    vec3 normal_texture = sample_and_mix_vector3_no_scale(alpha1, alpha2, alphaFinal, terrain_texcoord, detail_0_normal, detail_1_normal, detail_2_normal, detail_3_normal);
 
     vec3[4] orm_factors;
     orm_factors[0] = vec3(1.0, roughnessFactors.x, metallicFactors.x);
@@ -124,7 +124,7 @@ void main()
     float base_color_factor_alpha = terrain_mix(vec4(baseColorFactors[0].z, baseColorFactors[1].z, baseColorFactors[2].z, baseColorFactors[3].z), alpha1, alpha2, alphaFinal);
 
     // from mikktspace.com
-    vec3 vNt = normal_texture.xyz*2.0-1.0;
+    vec3 vNt = sample_and_mix_normal(alpha1, alpha2, alphaFinal, terrain_texcoord, detail_0_normal, detail_1_normal, detail_2_normal, detail_3_normal);
     float sign = vary_sign;
     vec3 vN = vary_normal;
     vec3 vT = vary_tangent.xyz;
diff --git a/indra/newview/app_settings/shaders/class1/deferred/pbrterrainUtilF.glsl b/indra/newview/app_settings/shaders/class1/deferred/pbrterrainUtilF.glsl
index ef2d030320..a9155cc629 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/pbrterrainUtilF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/pbrterrainUtilF.glsl
@@ -65,14 +65,56 @@ vec4 terrain_mix(vec4[4] samples, float alpha1, float alpha2, float alphaFinal)
 }
 
 #if TERRAIN_PLANAR_TEXTURE_SAMPLE_COUNT == 3
-// Pre-transformed texture coordinates for each axial uv slice (Packing: xy, yz, zx, unused)
-#define TerrainCoord vec4[2]
 // Triplanar mapping
+
+// Pre-transformed texture coordinates for each axial uv slice (Packing: xy, yz, (-x)z, unused)
+#define TerrainCoord vec4[2]
+
+vec4 _t_texture(sampler2D tex, vec2 uv_unflipped, float sign)
+{
+    // If the vertex normal is negative, flip the texture back
+    // right-side up.
+    vec2 uv = uv_unflipped * vec2(sign, 1);
+    return texture(tex, uv);
+}
+
+vec3 _t_texture_n(sampler2D tex, vec2 uv_unflipped, float sign)
+{
+    // Unpack normal from pixel to vector
+    vec3 n = _t_texture(tex, uv_unflipped, sign).xyz*2.0-1.0;
+    // If the sign is negative, rotate normal by 180 degrees
+    n.xy = (min(0, sign) * n.xy) + (min(0, -sign) * -n.xy);
+    return n;
+}
+
 vec4 terrain_texture(sampler2D tex, TerrainCoord terrain_coord)
 {
-    vec4 x = texture(tex, terrain_coord[0].zw);
-    vec4 y = texture(tex, terrain_coord[1].xy);
-    vec4 z = texture(tex, terrain_coord[0].xy);
+    // Multiplying the UVs by the sign of the normal flips the texture upright.
+    vec4 x = _t_texture(tex, terrain_coord[0].zw, sign(vary_vertex_normal.x));
+    vec4 y = _t_texture(tex, terrain_coord[1].xy, sign(vary_vertex_normal.y));
+    vec4 z = _t_texture(tex, terrain_coord[0].xy, sign(vary_vertex_normal.z));
+    float sharpness = TERRAIN_TRIPLANAR_BLEND_FACTOR;
+    vec3 weight = pow(abs(vary_vertex_normal), vec3(sharpness));
+    return ((x * weight.x) + (y * weight.y) + (z * weight.z)) / (weight.x + weight.y + weight.z);
+}
+
+// Specialized triplanar normal texture sampling implementation, taking into
+// account how the rotation of the texture affects the lighting and trying to
+// negate that.
+// *TODO: Decide if we want this. It may be better to just calculate the
+// tangents on-the-fly here rather than messing with the normals, due to the
+// subtleties of the effects of triplanar mapping on UVs. These sampled normals
+// are only valid on the faces of a cube, and the pregenerated tangents are
+// only valid for uv = xy.
+// *NOTE: Bottom face has not been tested
+vec3 terrain_texture_normal(sampler2D tex, TerrainCoord terrain_coord)
+{
+    vec3 x = _t_texture_n(tex, terrain_coord[0].zw, sign(vary_vertex_normal.x));
+    x.xy = vec2(-x.y, x.x);
+    vec3 y = _t_texture_n(tex, terrain_coord[1].xy, sign(vary_vertex_normal.y));
+    y.xy = -y.xy;
+    vec3 z = _t_texture_n(tex, terrain_coord[0].xy, sign(vary_vertex_normal.z));
+
     float sharpness = TERRAIN_TRIPLANAR_BLEND_FACTOR;
     vec3 weight = pow(abs(vary_vertex_normal), vec3(sharpness));
     return ((x * weight.x) + (y * weight.y) + (z * weight.z)) / (weight.x + weight.y + weight.z);
@@ -83,6 +125,11 @@ vec4 terrain_texture(sampler2D tex, TerrainCoord terrain_coord)
 {
     return texture(tex, terrain_coord);
 }
+
+vec3 terrain_texture_normal(sampler2D tex, TerrainCoord terrain_coord)
+{
+    return texture(tex, terrain_coord).xyz;
+}
 #endif
 
 vec3 sample_and_mix_color3(float alpha1, float alpha2, float alphaFinal, TerrainCoord texcoord, vec3[4] factors, sampler2D tex0, sampler2D tex1, sampler2D tex2, sampler2D tex3)
@@ -135,12 +182,13 @@ vec3 sample_and_mix_vector3(float alpha1, float alpha2, float alphaFinal, Terrai
     return terrain_mix(samples, alpha1, alpha2, alphaFinal);
 }
 
-vec3 sample_and_mix_vector3_no_scale(float alpha1, float alpha2, float alphaFinal, TerrainCoord texcoord, sampler2D tex0, sampler2D tex1, sampler2D tex2, sampler2D tex3)
+// Returns the unpacked normal texture in range [-1, 1]
+vec3 sample_and_mix_normal(float alpha1, float alpha2, float alphaFinal, TerrainCoord texcoord, sampler2D tex0, sampler2D tex1, sampler2D tex2, sampler2D tex3)
 {
     vec3[4] samples;
-    samples[0] = terrain_texture(tex0, texcoord).xyz;
-    samples[1] = terrain_texture(tex1, texcoord).xyz;
-    samples[2] = terrain_texture(tex2, texcoord).xyz;
-    samples[3] = terrain_texture(tex3, texcoord).xyz;
+    samples[0] = terrain_texture_normal(tex0, texcoord).xyz;
+    samples[1] = terrain_texture_normal(tex1, texcoord).xyz;
+    samples[2] = terrain_texture_normal(tex2, texcoord).xyz;
+    samples[3] = terrain_texture_normal(tex3, texcoord).xyz;
     return terrain_mix(samples, alpha1, alpha2, alphaFinal);
 }
diff --git a/indra/newview/app_settings/shaders/class1/deferred/pbrterrainV.glsl b/indra/newview/app_settings/shaders/class1/deferred/pbrterrainV.glsl
index 991783d242..2df5faf037 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/pbrterrainV.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/pbrterrainV.glsl
@@ -34,7 +34,9 @@ in vec4 diffuse_color;
 in vec2 texcoord0;
 in vec2 texcoord1;
 
+#if TERRAIN_PLANAR_TEXTURE_SAMPLE_COUNT == 3
 out vec4[2] vary_coords;
+#endif
 out vec3 vary_vertex_normal; // Used by pbrterrainUtilF.glsl
 out vec3 vary_normal;
 out vec3 vary_tangent;
@@ -42,22 +44,13 @@ flat out float vary_sign;
 out vec4 vary_texcoord0;
 out vec4 vary_texcoord1;
 
-uniform vec4 object_plane_s;
-uniform vec4 object_plane_t;
+// *HACK: tangent_space_transform should use texture_normal_transform, or maybe
+// we shouldn't use tangent_space_transform at all. See the call to
+// tangent_space_transform below.
+uniform vec4[2] texture_base_color_transform;
 
-vec4 texgen_object_pbr(vec4 tc, mat4 mat, vec4 tp0, vec4 tp1)
-{
-    vec4 tcoord;
-    
-    tcoord.x = dot(tc, tp0);
-    tcoord.y = dot(tc, tp1);
-    tcoord.z = tcoord.z;
-    tcoord.w = tcoord.w;
-    
-    tcoord = mat * tcoord;
-    
-    return tcoord;
-}
+vec2 texture_transform(vec2 vertex_texcoord, vec4[2] khr_gltf_transform, mat4 sl_animation_transform);
+vec3 tangent_space_transform(vec4 vertex_tangent, vec3 vertex_normal, vec4[2] khr_gltf_transform, mat4 sl_animation_transform);
 
 void main()
 {
@@ -65,23 +58,36 @@ void main()
 	gl_Position = modelview_projection_matrix * vec4(position.xyz, 1.0); 
 
 	vec3 n = normal_matrix * normal;
-    // *TODO: Looks like terrain normals are per-vertex when they should be per-triangle instead, causing incorrect values on triangles touching steep edges of terrain
     vary_vertex_normal = normal;
 	vec3 t = normal_matrix * tangent.xyz;
 
     vary_tangent = normalize(t);
+    // *TODO: Decide if we want this. It may be better to just calculate the
+    // tangents on-the-fly in the fragment shader, due to the subtleties of the
+    // effect of triplanar mapping on UVs.
+    // *HACK: Should be using texture_normal_transform here. The KHR texture
+    // transform spec requires handling texture transforms separately for each
+    // individual texture.
+    vary_tangent = normalize(tangent_space_transform(vec4(t, tangent.w), n, texture_base_color_transform, texture_matrix0));
     vary_sign = tangent.w;
     vary_normal = normalize(n);
 
     // Transform and pass tex coords
-    // *NOTE: KHR texture transform is ignored for now
-    vary_texcoord0.xy = texgen_object_pbr(vec4(texcoord0, 0, 1), texture_matrix0, object_plane_s, object_plane_t).xy;
-    
-    vec4 tc = vec4(texcoord1,0,1); // TODO: This is redundant. Better to just use position and ignore texcoord? (We still need to decide how to handle alpha ramp, though...)
-    vary_coords[0].xy = texgen_object_pbr(vec4(position.xy, 0, 1), texture_matrix0, object_plane_s, object_plane_t).xy;
-    vary_coords[0].zw = texgen_object_pbr(vec4(position.yz, 0, 1), texture_matrix0, object_plane_s, object_plane_t).xy;
-    vary_coords[1].xy = texgen_object_pbr(vec4(position.zx, 0, 1), texture_matrix0, object_plane_s, object_plane_t).xy;
+    // *HACK: texture_base_color_transform is used for all of these here, but
+    // the KHR texture transform spec requires handling texture transforms
+    // separately for each individual texture.
+#if TERRAIN_PLANAR_TEXTURE_SAMPLE_COUNT == 3
+    // xy
+    vary_coords[0].xy = texture_transform(position.xy, texture_base_color_transform, texture_matrix0);
+    // yz
+    vary_coords[0].zw = texture_transform(position.yz, texture_base_color_transform, texture_matrix0);
+    // (-x)z
+    vary_coords[1].xy = texture_transform(position.xz * vec2(-1, 1), texture_base_color_transform, texture_matrix0);
+#elif TERRAIN_PLANAR_TEXTURE_SAMPLE_COUNT == 1
+    vary_texcoord0.xy = texture_transform(texcoord0, texture_base_color_transform, texture_matrix0);
+#endif
     
+    vec4 tc = vec4(texcoord1,0,1);
     vary_texcoord0.zw = tc.xy;
     vary_texcoord1.xy = tc.xy-vec2(2.0, 0.0);
     vary_texcoord1.zw = tc.xy-vec2(1.0, 0.0);
diff --git a/indra/newview/app_settings/shaders/class1/deferred/textureUtilV.glsl b/indra/newview/app_settings/shaders/class1/deferred/textureUtilV.glsl
index 636dfed4ba..732333311c 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/textureUtilV.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/textureUtilV.glsl
@@ -65,7 +65,7 @@ vec2 texture_transform(vec2 vertex_texcoord, vec4[2] khr_gltf_transform, mat4 sl
     // Apply texture animation first to avoid shearing and other artifacts
     texcoord = (sl_animation_transform * vec4(texcoord, 0, 1)).xy;
     // Convert to left-handed coordinate system. The offset of 1 is necessary
-    // for rotations to be applied correctly.
+    // for rotation and scale to be applied correctly.
     texcoord.y = 1.0 - texcoord.y;
     texcoord = khr_texture_transform(texcoord, khr_gltf_transform[0].xy, khr_gltf_transform[0].z, khr_gltf_transform[1].xy);
     // Convert back to right-handed coordinate system