Fix potential undefined behavior when converting to and from glm types from LLVector3/4 and fall back mul_mat4_vec3 to scalar implementation to attempt crash mitigation (#3339)

9 files changed, 30 insertions, 34 deletions

diff --git a/indra/newview/gltfscenemanager.cpp b/indra/newview/gltfscenemanager.cpp
index ed66753267..2d6d9a153e 100644
--- a/indra/newview/gltfscenemanager.cpp
+++ b/indra/newview/gltfscenemanager.cpp
@@ -975,9 +975,9 @@ void renderAssetDebug(LLViewerObject* obj, Asset* asset)
 
     LLVector4a t;
     agent_to_asset.affineTransform(gDebugRaycastStart, t);
-    start = glm::make_vec4(t.getF32ptr());
+    start = vec4(t);
     agent_to_asset.affineTransform(gDebugRaycastEnd, t);
-    end = glm::make_vec4(t.getF32ptr());
+    end = vec4(t);
 
     start.w = end.w = 1.0;
 
diff --git a/indra/newview/llgltfmaterialpreviewmgr.cpp b/indra/newview/llgltfmaterialpreviewmgr.cpp
index cf6b08797d..da1f1a466f 100644
--- a/indra/newview/llgltfmaterialpreviewmgr.cpp
+++ b/indra/newview/llgltfmaterialpreviewmgr.cpp
@@ -472,9 +472,9 @@ bool LLGLTFPreviewTexture::render()
 
     gPipeline.setupHWLights();
     glm::mat4 mat = get_current_modelview();
-    glm::vec4 transformed_light_dir = glm::make_vec4(light_dir.mV);
+    glm::vec4 transformed_light_dir(light_dir);
     transformed_light_dir = mat * transformed_light_dir;
-    SetTemporarily<LLVector4> force_sun_direction_high_graphics(&gPipeline.mTransformedSunDir, LLVector4(glm::value_ptr(transformed_light_dir)));
+    SetTemporarily<LLVector4> force_sun_direction_high_graphics(&gPipeline.mTransformedSunDir, LLVector4(transformed_light_dir));
     // Override lights to ensure the sun is always shining from a certain direction (low graphics)
     // See also force_sun_direction_high_graphics and fixup_shader_constants
     {
diff --git a/indra/newview/llhudrender.cpp b/indra/newview/llhudrender.cpp
index 135fba7897..6850e57b94 100644
--- a/indra/newview/llhudrender.cpp
+++ b/indra/newview/llhudrender.cpp
@@ -106,7 +106,7 @@ void hud_render_text(const LLWString &wstr, const LLVector3 &pos_agent,
     LLRect world_view_rect = gViewerWindow->getWorldViewRectRaw();
     glm::ivec4 viewport(world_view_rect.mLeft, world_view_rect.mBottom, world_view_rect.getWidth(), world_view_rect.getHeight());
 
-    glm::vec3 win_coord = glm::project(glm::make_vec3(render_pos.mV), get_current_modelview(), get_current_projection(), viewport);
+    glm::vec3 win_coord = glm::project(glm::vec3(render_pos), get_current_modelview(), get_current_projection(), viewport);
 
     //fonts all render orthographically, set up projection``
     gGL.matrixMode(LLRender::MM_PROJECTION);
diff --git a/indra/newview/llpanelprimmediacontrols.cpp b/indra/newview/llpanelprimmediacontrols.cpp
index 4db0a5b59d..b8c12ce0b9 100644
--- a/indra/newview/llpanelprimmediacontrols.cpp
+++ b/indra/newview/llpanelprimmediacontrols.cpp
@@ -660,11 +660,11 @@ void LLPanelPrimMediaControls::updateShape()
         for(; vert_it != vert_end; ++vert_it)
         {
             // project silhouette vertices into screen space
-            glm::vec3 screen_vert(glm::make_vec3(vert_it->mV));
+            glm::vec3 screen_vert(*vert_it);
             screen_vert = mul_mat4_vec3(mat, screen_vert);
 
             // add to screenspace bounding box
-            update_min_max(min, max, LLVector3(glm::value_ptr(screen_vert)));
+            update_min_max(min, max, LLVector3(screen_vert));
         }
 
         // convert screenspace bbox to pixels (in screen coords)
diff --git a/indra/newview/llreflectionmap.cpp b/indra/newview/llreflectionmap.cpp
index 1196d138e9..f3adb52d5e 100644
--- a/indra/newview/llreflectionmap.cpp
+++ b/indra/newview/llreflectionmap.cpp
@@ -259,7 +259,7 @@ bool LLReflectionMap::getBox(LLMatrix4& box)
             glm::mat4 mv(get_current_modelview());
             LLVector3 s = mViewerObject->getScale().scaledVec(LLVector3(0.5f, 0.5f, 0.5f));
             mRadius = s.magVec();
-            glm::mat4 scale = glm::scale(glm::make_vec3(s.mV));
+            glm::mat4 scale = glm::scale(glm::vec3(s));
             if (mViewerObject->mDrawable != nullptr)
             {
                 // object to agent space (no scale)
diff --git a/indra/newview/llsettingsvo.cpp b/indra/newview/llsettingsvo.cpp
index 6f9d4a24bc..62df6cd275 100644
--- a/indra/newview/llsettingsvo.cpp
+++ b/indra/newview/llsettingsvo.cpp
@@ -1093,8 +1093,8 @@ void LLSettingsVOWater::applySpecial(void *ptarget, bool force)
 
         LLVector4 waterPlane(enorm.x, enorm.y, enorm.z, -glm::dot(ep, enorm));
 
-        norm = glm::make_vec3(gPipeline.mHeroProbeManager.mMirrorNormal.mV);
-        p    = glm::make_vec3(gPipeline.mHeroProbeManager.mMirrorPosition.mV);
+        norm = glm::vec3(gPipeline.mHeroProbeManager.mMirrorNormal);
+        p    = glm::vec3(gPipeline.mHeroProbeManager.mMirrorPosition);
         enorm = mul_mat4_vec3(invtrans, norm);
         enorm = glm::normalize(enorm);
         ep = mul_mat4_vec3(mat, p);
diff --git a/indra/newview/llviewercamera.cpp b/indra/newview/llviewercamera.cpp
index aa43b2dbad..7d777162ed 100644
--- a/indra/newview/llviewercamera.cpp
+++ b/indra/newview/llviewercamera.cpp
@@ -420,7 +420,7 @@ bool LLViewerCamera::projectPosAgentToScreen(const LLVector3 &pos_agent, LLCoord
 
     LLRect world_view_rect = gViewerWindow->getWorldViewRectRaw();
     glm::ivec4 viewport(world_view_rect.mLeft, world_view_rect.mBottom, world_view_rect.getWidth(), world_view_rect.getHeight());
-    glm::vec3 win_coord = glm::project(glm::make_vec3(pos_agent.mV), get_current_modelview(), get_current_projection(), viewport);
+    glm::vec3 win_coord = glm::project(glm::vec3(pos_agent), get_current_modelview(), get_current_projection(), viewport);
 
     {
         // convert screen coordinates to virtual UI coordinates
@@ -514,7 +514,7 @@ bool LLViewerCamera::projectPosAgentToScreenEdge(const LLVector3 &pos_agent,
     LLRect world_view_rect = gViewerWindow->getWorldViewRectRaw();
 
     glm::ivec4 viewport(world_view_rect.mLeft, world_view_rect.mBottom, world_view_rect.getWidth(), world_view_rect.getHeight());
-    glm::vec3 win_coord = glm::project(glm::make_vec3(pos_agent.mV), get_current_modelview(), get_current_projection(), viewport);
+    glm::vec3 win_coord = glm::project(glm::vec3(pos_agent), get_current_modelview(), get_current_projection(), viewport);
 
     {
         win_coord.x /= gViewerWindow->getDisplayScale().mV[VX];
diff --git a/indra/newview/llvoavatar.cpp b/indra/newview/llvoavatar.cpp
index 7bc9d06f9a..7bf9c88b99 100644
--- a/indra/newview/llvoavatar.cpp
+++ b/indra/newview/llvoavatar.cpp
@@ -1928,8 +1928,8 @@ bool LLVOAvatar::lineSegmentIntersect(const LLVector4a& start, const LLVector4a&
             glm::mat4 inverse = glm::inverse(mat);
             glm::mat4 norm_mat = glm::transpose(inverse);
 
-            glm::vec3 p1(glm::make_vec3(start.getF32ptr()));
-            glm::vec3 p2(glm::make_vec3(end.getF32ptr()));
+            glm::vec3 p1(start);
+            glm::vec3 p2(end);
 
             p1 = mul_mat4_vec3(inverse, p1);
             p2 = mul_mat4_vec3(inverse, p2);
@@ -1937,12 +1937,12 @@ bool LLVOAvatar::lineSegmentIntersect(const LLVector4a& start, const LLVector4a&
             LLVector3 position;
             LLVector3 norm;
 
-            if (linesegment_sphere(LLVector3(glm::value_ptr(p1)), LLVector3(glm::value_ptr(p2)), LLVector3(0,0,0), 1.f, position, norm))
+            if (linesegment_sphere(LLVector3(p1), LLVector3(p2), LLVector3(0,0,0), 1.f, position, norm))
             {
-                glm::vec3 res_pos(glm::make_vec3(position.mV));
+                glm::vec3 res_pos(position);
                 res_pos = mul_mat4_vec3(mat, res_pos);
 
-                 glm::vec3 res_norm(glm::make_vec3(norm.mV));
+                 glm::vec3 res_norm(norm);
                 res_norm = glm::normalize(res_norm);
                 res_norm = glm::mat3(norm_mat) * res_norm;
 
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index c6e6c454de..f91b054dd7 100644
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -8396,13 +8396,13 @@ void LLPipeline::renderDeferredLighting()
 
         setupHWLights();  // to set mSun/MoonDir;
 
-        glm::vec4 tc(glm::make_vec4(mSunDir.mV));
+        glm::vec4 tc(mSunDir);
         tc = mat * tc;
-        mTransformedSunDir.set(glm::value_ptr(tc));
+        mTransformedSunDir.set(tc);
 
-        glm::vec4 tc_moon(glm::make_vec4(mMoonDir.mV));
+        glm::vec4 tc_moon(mMoonDir);
         tc_moon = mat * tc_moon;
-        mTransformedMoonDir.set(glm::value_ptr(tc_moon));
+        mTransformedMoonDir.set(tc_moon);
 
         if ((RenderDeferredSSAO && !gCubeSnapshot) || RenderShadowDetail > 0)
         {
@@ -8655,7 +8655,7 @@ void LLPipeline::renderDeferredLighting()
                             continue;
                         }
 
-                        glm::vec3 tc(glm::make_vec3(c));
+                        glm::vec3 tc(center);
                         tc = mul_mat4_vec3(mat, tc);
 
                         fullscreen_lights.push_back(LLVector4(tc.x, tc.y, tc.z, s));
@@ -8762,13 +8762,12 @@ void LLPipeline::renderDeferredLighting()
                     LLDrawable* drawablep = *iter;
                     LLVOVolume* volume = drawablep->getVOVolume();
                     LLVector3   center = drawablep->getPositionAgent();
-                    F32* c = center.mV;
                     F32         light_size_final = volume->getLightRadius() * 1.5f;
                     F32         light_falloff_final = volume->getLightFalloff(DEFERRED_LIGHT_FALLOFF);
 
                     sVisibleLightCount++;
 
-                    glm::vec3 tc(glm::make_vec3(c));
+                    glm::vec3 tc(center);
                     tc = mul_mat4_vec3(mat, tc);
 
                     setupSpotLight(gDeferredMultiSpotLightProgram, drawablep);
@@ -9903,10 +9902,7 @@ void LLPipeline::generateSunShadow(LLCamera& camera)
     LLVector3 lightDir = -caster_dir;
     lightDir.normVec();
 
-    glm::vec3 light_dir(glm::make_vec3(lightDir.mV));
-
     //create light space camera matrix
-
     LLVector3 at = lightDir;
 
     LLVector3 up = camera.getAtAxis();
@@ -9958,9 +9954,9 @@ void LLPipeline::generateSunShadow(LLCamera& camera)
         //get good split distances for frustum
         for (U32 i = 0; i < fp.size(); ++i)
         {
-            glm::vec3 v(glm::make_vec3(fp[i].mV));
+            glm::vec3 v(fp[i]);
             v = mul_mat4_vec3(saved_view, v);
-            fp[i].setVec(glm::value_ptr(v));
+            fp[i] = LLVector3(v);
         }
 
         min = fp[0];
@@ -10109,9 +10105,9 @@ void LLPipeline::generateSunShadow(LLCamera& camera)
 
             for (U32 i = 0; i < fp.size(); i++)
             {
-                glm::vec3 p = glm::make_vec3(fp[i].mV);
+                glm::vec3 p(fp[i]);
                 p = mul_mat4_vec3(view[j], p);
-                wpf.push_back(LLVector3(glm::value_ptr(p)));
+                wpf.push_back(LLVector3(p));
             }
 
             min = wpf[0];
@@ -10312,19 +10308,19 @@ void LLPipeline::generateSunShadow(LLCamera& camera)
                         view[j] = glm::inverse(view[j]);
                         //llassert(origin.isFinite());
 
-                        glm::vec3 origin_agent(glm::make_vec3(origin.mV));
+                        glm::vec3 origin_agent(origin);
 
                         //translate view to origin
                         origin_agent = mul_mat4_vec3(view[j], origin_agent);
 
-                        eye = LLVector3(glm::value_ptr(origin_agent));
+                        eye = LLVector3(origin_agent);
                         //llassert(eye.isFinite());
                         if (!hasRenderDebugMask(LLPipeline::RENDER_DEBUG_SHADOW_FRUSTA) && !gCubeSnapshot)
                         {
                             mShadowFrustOrigin[j] = eye;
                         }
 
-                        view[j] = look(LLVector3(glm::value_ptr(origin_agent)), lightDir, -up);
+                        view[j] = look(LLVector3(origin_agent), lightDir, -up);
 
                         F32 fx = 1.f/tanf(fovx);
                         F32 fz = 1.f/tanf(fovz);