Fake anti-aliasing for deferred rendering as an alternative to real anti-aliasing.

6 files changed, 89 insertions, 135 deletions

diff --git a/indra/newview/app_settings/shaders/class1/deferred/postDeferredF.glsl b/indra/newview/app_settings/shaders/class1/deferred/postDeferredF.glsl
index 7e41b07f8a..03c2e63fb1 100644
--- a/indra/newview/app_settings/shaders/class1/deferred/postDeferredF.glsl
+++ b/indra/newview/app_settings/shaders/class1/deferred/postDeferredF.glsl
@@ -10,50 +10,77 @@
 #extension GL_ARB_texture_rectangle : enable
 
 uniform sampler2DRect diffuseRect;
-uniform sampler2DRect localLightMap;
-uniform sampler2DRect sunLightMap;
-uniform sampler2DRect giLightMap;
-uniform sampler2D	  luminanceMap;
-uniform sampler2DRect lightMap;
+uniform sampler2DRect edgeMap;
+uniform sampler2DRect depthMap;
+uniform sampler2DRect normalMap;
+uniform sampler2D bloomMap;
 
-uniform vec3 lum_quad;
-uniform float lum_lod;
-uniform vec4 ambient;
-
-uniform vec3 gi_quad;
+uniform float depth_cutoff;
+uniform float norm_cutoff;
 
+uniform mat4 inv_proj;
 uniform vec2 screen_res;
+
 varying vec2 vary_fragcoord;
 
+float getDepth(vec2 pos_screen)
+{
+	float z = texture2DRect(depthMap, pos_screen.xy).a;
+	z = z*2.0-1.0;
+	vec4 ndc = vec4(0.0, 0.0, z, 1.0);
+	vec4 p = inv_proj*ndc;
+	return p.z/p.w;
+}
+
 void main() 
 {
+	vec3 norm = texture2DRect(normalMap, vary_fragcoord.xy).xyz;
+	norm = vec3((norm.xy-0.5)*2.0,norm.z); // unpack norm
+	float depth = getDepth(vary_fragcoord.xy);
+	
 	vec2 tc = vary_fragcoord.xy;
-	vec3 lum = texture2DLod(luminanceMap, tc/screen_res, lum_lod).rgb;
-	float luminance = lum.r;
-	luminance = luminance*lum_quad.y+lum_quad.z;
-
-	vec4 diff = texture2DRect(diffuseRect, vary_fragcoord.xy);
-
-	float ambocc = texture2DRect(lightMap, vary_fragcoord.xy).g;
-			
-	vec3 gi_col = texture2DRect(giLightMap, vary_fragcoord.xy).rgb;
-	gi_col = gi_col*gi_col*gi_quad.x + gi_col*gi_quad.y+gi_quad.z*ambocc*ambient.rgb;
-	gi_col *= diff;
 	
-	vec4 sun_col =	texture2DRect(sunLightMap, vary_fragcoord.xy);
+	float sc = 0.75;
+	
+	vec2 de;
+	de.x = (depth-getDepth(tc+vec2(sc, sc))) + (depth-getDepth(tc+vec2(-sc, -sc)));
+	de.y = (depth-getDepth(tc+vec2(-sc, sc))) + (depth-getDepth(tc+vec2(sc, -sc)));
+	de /= depth;
+	de *= de;
+	de = step(depth_cutoff, de);
+	
+	vec2 ne;
+	vec3 nexnorm = texture2DRect(normalMap, tc+vec2(-sc,-sc)).rgb;
+	nexnorm = vec3((nexnorm.xy-0.5)*2.0,nexnorm.z); // unpack norm
+	ne.x = dot(nexnorm, norm);
+	vec3 neynorm = texture2DRect(normalMap, tc+vec2(sc,sc)).rgb;
+	neynorm = vec3((neynorm.xy-0.5)*2.0,neynorm.z); // unpack norm
+	ne.y = dot(neynorm, norm);
+	
+	ne = 1.0-ne;
+	
+	ne = step(norm_cutoff, ne);
+	
+	float edge_weight = clamp(dot(de,de)+dot(ne,ne), 0.0, 1.0);
+	//edge_weight *= 0.0;
+	
+	vec4 bloom = texture2D(bloomMap, vary_fragcoord.xy/screen_res);
+	vec4 diff = texture2DRect(diffuseRect, vary_fragcoord.xy);
+	diff += texture2DRect(diffuseRect, vary_fragcoord.xy+vec2(1,1))*edge_weight;
+	diff += texture2DRect(diffuseRect, vary_fragcoord.xy+vec2(-1,-1))*edge_weight;
+	diff += texture2DRect(diffuseRect, vary_fragcoord.xy+vec2(-1,1))*edge_weight;
+	diff += texture2DRect(diffuseRect, vary_fragcoord.xy+vec2(1,-1))*edge_weight;
+	diff += texture2DRect(diffuseRect, vary_fragcoord.xy+vec2(-1,0))*edge_weight;
+	diff += texture2DRect(diffuseRect, vary_fragcoord.xy+vec2(1,0))*edge_weight;
+	diff += texture2DRect(diffuseRect, vary_fragcoord.xy+vec2(0,1))*edge_weight;
+	diff += texture2DRect(diffuseRect, vary_fragcoord.xy+vec2(0,-1))*edge_weight;
 	
-	vec3 local_col = texture2DRect(localLightMap, vary_fragcoord.xy).rgb;
-		
-		
-	sun_col *= 1.0/min(luminance, 1.0);
-	gi_col *= 1.0/luminance;
-		
-	vec3 col = sun_col.rgb+gi_col+local_col;
+	diff /= 1.0+edge_weight*8.0;
 	
-	gl_FragColor.rgb = col.rgb;
-	col.rgb = max(col.rgb-vec3(1.0,1.0,1.0), vec3(0.0, 0.0, 0.0)); 
+	vec4 blur = texture2DRect(edgeMap, vary_fragcoord.xy);
 	
-	gl_FragColor.a = 0.0; // max(dot(col.rgb,col.rgb)*lum_quad.x, sun_col.a);
+	//gl_FragColor = vec4(edge_weight,edge_weight,edge_weight, 1.0);
+	gl_FragColor = diff + bloom;
+	//gl_FragColor.r = edge_weight;
 	
-	//gl_FragColor.rgb = vec3(lum_lod);
 }
diff --git a/indra/newview/app_settings/shaders/class2/deferred/edgeF.glsl b/indra/newview/app_settings/shaders/class2/deferred/edgeF.glsl
index c9bee13b48..3155f3f929 100644
--- a/indra/newview/app_settings/shaders/class2/deferred/edgeF.glsl
+++ b/indra/newview/app_settings/shaders/class2/deferred/edgeF.glsl
@@ -12,8 +12,6 @@
 uniform sampler2DRect depthMap;
 uniform sampler2DRect normalMap;
 
-uniform float gi_dist_cutoff;
-
 varying vec2 vary_fragcoord;
 
 uniform float depth_cutoff;
diff --git a/indra/newview/app_settings/shaders/class2/deferred/postDeferredF.glsl b/indra/newview/app_settings/shaders/class2/deferred/postDeferredF.glsl
deleted file mode 100644
index 3da50eb52a..0000000000
--- a/indra/newview/app_settings/shaders/class2/deferred/postDeferredF.glsl
+++ /dev/null
@@ -1,61 +0,0 @@
-/** 
- * @file postDeferredF.glsl
- *
- * $LicenseInfo:firstyear=2007&license=viewerlgpl$
- * $/LicenseInfo$
- */
- 
-#version 120
-
-uniform sampler2DRect diffuseRect;
-uniform sampler2DRect localLightMap;
-uniform sampler2DRect sunLightMap;
-uniform sampler2DRect giLightMap;
-uniform sampler2D	  luminanceMap;
-uniform sampler2DRect lightMap;
-
-uniform vec3 gi_lum_quad;
-uniform vec3 sun_lum_quad;
-uniform vec3 lum_quad;
-uniform float lum_lod;
-uniform vec4 ambient;
-
-uniform vec3 gi_quad;
-
-uniform vec2 screen_res;
-varying vec2 vary_fragcoord;
-
-void main() 
-{
-	vec2 tc = vary_fragcoord.xy;
-	vec3 lcol = texture2DLod(luminanceMap, tc/screen_res, lum_lod).rgb;
-
-	float lum = sqrt(lcol.r)*lum_quad.x+lcol.r*lcol.r*lum_quad.y+lcol.r*lum_quad.z;
-	
-	vec4 diff = texture2DRect(diffuseRect, vary_fragcoord.xy);
-
-	float ambocc = texture2DRect(lightMap, vary_fragcoord.xy).g;
-			
-	vec3 gi_col = texture2DRect(giLightMap, vary_fragcoord.xy).rgb;
-	gi_col = gi_col*gi_col*gi_quad.x + gi_col*gi_quad.y+gi_quad.z*ambocc*ambient.rgb;
-	gi_col *= diff;
-	
-	vec4 sun_col =	texture2DRect(sunLightMap, vary_fragcoord.xy);
-	
-	vec3 local_col = texture2DRect(localLightMap, vary_fragcoord.xy).rgb;
-		
-
-	float sun_lum = 1.0-lum;
-	sun_lum = sun_lum*sun_lum*sun_lum_quad.x + sun_lum*sun_lum_quad.y+sun_lum_quad.z;
-		
-	float gi_lum = lum;
-	gi_lum = gi_lum*gi_lum*gi_lum_quad.x+gi_lum*gi_lum_quad.y+gi_lum_quad.z;
-	gi_col *= 1.0/gi_lum;
-		
-	vec3 col = sun_col.rgb*(1.0+max(sun_lum,0.0))+gi_col+local_col;
-	
-	gl_FragColor.rgb = col.rgb;
-	gl_FragColor.a = max(sun_lum*min(sun_col.r+sun_col.g+sun_col.b, 1.0), sun_col.a);
-	
-	//gl_FragColor.rgb = texture2DRect(giLightMap, vary_fragcoord.xy).rgb;
-}
diff --git a/indra/newview/app_settings/shaders/class2/deferred/postDeferredV.glsl b/indra/newview/app_settings/shaders/class2/deferred/postDeferredV.glsl
deleted file mode 100644
index 12983baa94..0000000000
--- a/indra/newview/app_settings/shaders/class2/deferred/postDeferredV.glsl
+++ /dev/null
@@ -1,19 +0,0 @@
-/** 
- * @file postDeferredV.glsl
- *
- * $LicenseInfo:firstyear=2007&license=viewerlgpl$
- * $/LicenseInfo$
- */
- 
-#version 120
-
-varying vec2 vary_fragcoord;
-uniform vec2 screen_res;
-
-void main()
-{
-	//transform vertex
-	gl_Position = ftransform(); 
-	vec4 pos = gl_ModelViewProjectionMatrix * gl_Vertex;
-	vary_fragcoord = (pos.xy*0.5+0.5)*screen_res;
-}
diff --git a/indra/newview/llviewershadermgr.cpp b/indra/newview/llviewershadermgr.cpp
index 6231822fe7..a16257940e 100644
--- a/indra/newview/llviewershadermgr.cpp
+++ b/indra/newview/llviewershadermgr.cpp
@@ -1225,6 +1225,16 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 		success = gDeferredAvatarAlphaProgram.createShader(&mAvatarAttribs, &mAvatarUniforms);
 	}
 
+	if (success)
+	{
+		gDeferredPostProgram.mName = "Deferred Post Shader";
+		gDeferredPostProgram.mShaderFiles.clear();
+		gDeferredPostProgram.mShaderFiles.push_back(make_pair("deferred/postDeferredV.glsl", GL_VERTEX_SHADER_ARB));
+		gDeferredPostProgram.mShaderFiles.push_back(make_pair("deferred/postDeferredF.glsl", GL_FRAGMENT_SHADER_ARB));
+		gDeferredPostProgram.mShaderLevel = mVertexShaderLevel[SHADER_DEFERRED];
+		success = gDeferredPostProgram.createShader(NULL, NULL);
+	}
+
 	if (mVertexShaderLevel[SHADER_DEFERRED] > 1)
 	{
 		if (success)
@@ -1240,15 +1250,7 @@ BOOL LLViewerShaderMgr::loadShadersDeferred()
 
 	if (mVertexShaderLevel[SHADER_DEFERRED] > 2)
 	{
-		if (success)
-		{
-			gDeferredPostProgram.mName = "Deferred Post Shader";
-			gDeferredPostProgram.mShaderFiles.clear();
-			gDeferredPostProgram.mShaderFiles.push_back(make_pair("deferred/postDeferredV.glsl", GL_VERTEX_SHADER_ARB));
-			gDeferredPostProgram.mShaderFiles.push_back(make_pair("deferred/postDeferredF.glsl", GL_FRAGMENT_SHADER_ARB));
-			gDeferredPostProgram.mShaderLevel = mVertexShaderLevel[SHADER_DEFERRED];
-			success = gDeferredPostProgram.createShader(NULL, NULL);
-		}
+		
 
 		if (success)
 		{
diff --git a/indra/newview/pipeline.cpp b/indra/newview/pipeline.cpp
index d1a4575880..4007d9d8f3 100644
--- a/indra/newview/pipeline.cpp
+++ b/indra/newview/pipeline.cpp
@@ -693,8 +693,9 @@ void LLPipeline::allocateScreenBuffer(U32 resX, U32 resY)
 	}
 	
 
-	if (LLRenderTarget::sUseFBO && gGLManager.mHasFramebufferMultisample && samples > 1)
-	{
+	if (LLRenderTarget::sUseFBO && !sRenderDeferred && gGLManager.mHasFramebufferMultisample && samples > 1)
+	{ // DON'T use multisample buffers when rendering deferred -- multisampling doesn't play nice with deferred rendering
+		//so a post-effect smooths edges in screen space
 		mSampleBuffer.allocate(resX,resY,GL_RGBA,TRUE,TRUE,LLTexUnit::TT_RECT_TEXTURE,FALSE,samples);
 		if (LLPipeline::sRenderDeferred)
 		{
@@ -6245,12 +6246,18 @@ void LLPipeline::renderBloom(BOOL for_snapshot, F32 zoom_factor, int subfield)
 	
 	LLVertexBuffer::unbind();
 
-	if (LLPipeline::sRenderDeferred && LLViewerShaderMgr::instance()->getVertexShaderLevel(LLViewerShaderMgr::SHADER_DEFERRED) > 2)
+	if (LLPipeline::sRenderDeferred)
 	{
+		LLGLSLShader* shader = &gDeferredPostProgram;
+		if (LLViewerShaderMgr::instance()->getVertexShaderLevel(LLViewerShaderMgr::SHADER_DEFERRED) > 2)
+		{
+			shader = &gDeferredGIFinalProgram;
+		}
+
 		LLGLDisable blend(GL_BLEND);
-		bindDeferredShader(gDeferredGIFinalProgram);
+		bindDeferredShader(*shader);
 
-		S32 channel = gDeferredGIFinalProgram.enableTexture(LLViewerShaderMgr::DEFERRED_DIFFUSE, LLTexUnit::TT_RECT_TEXTURE);
+		S32 channel = shader->enableTexture(LLViewerShaderMgr::DEFERRED_DIFFUSE, LLTexUnit::TT_RECT_TEXTURE);
 		if (channel > -1)
 		{
 			mScreen.bindTexture(0, channel);
@@ -6268,7 +6275,7 @@ void LLPipeline::renderBloom(BOOL for_snapshot, F32 zoom_factor, int subfield)
 		
 		gGL.end();
 
-		unbindDeferredShader(gDeferredGIFinalProgram);
+		unbindDeferredShader(*shader);
 	}
 	else
 	{
@@ -6329,12 +6336,12 @@ void LLPipeline::renderBloom(BOOL for_snapshot, F32 zoom_factor, int subfield)
 
 		gGL.getTexUnit(0)->activate();
 		gGL.getTexUnit(0)->setTextureBlendType(LLTexUnit::TB_MULT);
+	}
 
-		if (LLRenderTarget::sUseFBO)
-		{ //copy depth buffer from mScreen to framebuffer
-			LLRenderTarget::copyContentsToFramebuffer(mScreen, 0, 0, mScreen.getWidth(), mScreen.getHeight(), 
-				0, 0, mScreen.getWidth(), mScreen.getHeight(), GL_DEPTH_BUFFER_BIT, GL_NEAREST);
-		}
+	if (LLRenderTarget::sUseFBO)
+	{ //copy depth buffer from mScreen to framebuffer
+		LLRenderTarget::copyContentsToFramebuffer(mScreen, 0, 0, mScreen.getWidth(), mScreen.getHeight(), 
+			0, 0, mScreen.getWidth(), mScreen.getHeight(), GL_DEPTH_BUFFER_BIT, GL_NEAREST);
 	}
 	
 	gGL.setSceneBlendType(LLRender::BT_ALPHA);