From 24a075388127dbb5c625c120e77519c0f81f2cee Mon Sep 17 00:00:00 2001
From: AronKovacs <s.aron.kovacs@gmail.com>
Date: Sat, 19 Dec 2020 21:59:56 +0100
Subject: [PATCH] fix fog for outlines

---
 src/mol-canvas3d/passes/postprocessing.ts | 3 +++
 src/mol-gl/shader/postprocessing.frag.ts  | 7 +++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/mol-canvas3d/passes/postprocessing.ts b/src/mol-canvas3d/passes/postprocessing.ts
index bca5b8422..d90958235 100644
--- a/src/mol-canvas3d/passes/postprocessing.ts
+++ b/src/mol-canvas3d/passes/postprocessing.ts
@@ -183,6 +183,7 @@ const PostprocessingSchema = {
     uTexSize: UniformSpec('v2'),
 
     dOrthographic: DefineSpec('number'),
+    uInvProjection: UniformSpec('m4'),
     uNear: UniformSpec('f'),
     uFar: UniformSpec('f'),
     uFogNear: UniformSpec('f'),
@@ -211,6 +212,7 @@ function getPostprocessingRenderable(ctx: WebGLContext, colorTexture: Texture, d
         uTexSize: ValueCell.create(Vec2.create(colorTexture.getWidth(), colorTexture.getHeight())),
 
         dOrthographic: ValueCell.create(0),
+        uInvProjection: ValueCell.create(Mat4.identity()),
         uNear: ValueCell.create(1),
         uFar: ValueCell.create(10000),
         uFogNear: ValueCell.create(10000),
@@ -408,6 +410,7 @@ export class PostprocessingPass {
             ValueCell.updateIfChanged(this.outlinesRenderable.values.uFar, camera.far);
             ValueCell.updateIfChanged(this.outlinesRenderable.values.uMaxPossibleViewZDiff, maxPossibleViewZDiff);
 
+            ValueCell.updateIfChanged(this.renderable.values.uInvProjection, invProjection);
             ValueCell.updateIfChanged(this.renderable.values.uMaxPossibleViewZDiff, maxPossibleViewZDiff);
             let fogColor = Vec3();
             Color.toVec3Normalized(fogColor, backgroundColor);
diff --git a/src/mol-gl/shader/postprocessing.frag.ts b/src/mol-gl/shader/postprocessing.frag.ts
index a3a30836a..ecc9e5153 100644
--- a/src/mol-gl/shader/postprocessing.frag.ts
+++ b/src/mol-gl/shader/postprocessing.frag.ts
@@ -16,6 +16,7 @@ uniform sampler2D tDepth;
 uniform sampler2D tOutlines;
 uniform vec2 uTexSize;
 
+uniform mat4 uInvProjection;
 uniform float uNear;
 uniform float uFar;
 uniform float uFogNear;
@@ -66,7 +67,7 @@ float getOutline(const in vec2 coords, out float closestTexel) {
     float selfViewZ = isBackground(selfDepth) ? backgroundViewZ : getViewZ(getDepth(coords));
 
     float outline = 1.0;
-    closestTexel = 1.0;
+    closestTexel = backgroundViewZ;
     for (float y = -uOutlineScale; y <= uOutlineScale; y++) {
         for (float x = -uOutlineScale; x <= uOutlineScale; x++) {
             if (x * x + y * y > uOutlineScale * uOutlineScale) {
@@ -79,7 +80,9 @@ float getOutline(const in vec2 coords, out float closestTexel) {
             float sampleOutline = sampleOutlineCombined.r;
             float sampleOutlineDepth = unpackRGToUnitInterval(sampleOutlineCombined.gb);
 
-            if (sampleOutline == 0.0 && sampleOutlineDepth < closestTexel && abs(selfViewZ - sampleOutlineDepth) > uMaxPossibleViewZDiff) {
+            float sampleOutlineViewDirLength = length(screenSpaceToViewSpace(vec3(sampleCoords, sampleOutlineDepth), uInvProjection));
+
+            if (sampleOutline == 0.0 && sampleOutlineViewDirLength < closestTexel && abs(selfViewZ - sampleOutlineDepth) > uMaxPossibleViewZDiff) {
                 outline = 0.0;
                 closestTexel = sampleOutlineDepth;
             }
-- 
GitLab