fix fog for outlines

src/mol-canvas3d/passes/postprocessing.ts

@@ -183,6 +183,7 @@ const PostprocessingSchema = {
     uTexSize: UniformSpec('v2'),
 
     dOrthographic: DefineSpec('number'),
+    uInvProjection: UniformSpec('m4'),
     uNear: UniformSpec('f'),
     uFar: UniformSpec('f'),
     uFogNear: UniformSpec('f'),
@@ -211,6 +212,7 @@ function getPostprocessingRenderable(ctx: WebGLContext, colorTexture: Texture, d
         uTexSize: ValueCell.create(Vec2.create(colorTexture.getWidth(), colorTexture.getHeight())),
 
         dOrthographic: ValueCell.create(0),
+        uInvProjection: ValueCell.create(Mat4.identity()),
         uNear: ValueCell.create(1),
         uFar: ValueCell.create(10000),
         uFogNear: ValueCell.create(10000),
@@ -408,6 +410,7 @@ export class PostprocessingPass {
             ValueCell.updateIfChanged(this.outlinesRenderable.values.uFar, camera.far);
             ValueCell.updateIfChanged(this.outlinesRenderable.values.uMaxPossibleViewZDiff, maxPossibleViewZDiff);
 
+            ValueCell.updateIfChanged(this.renderable.values.uInvProjection, invProjection);
             ValueCell.updateIfChanged(this.renderable.values.uMaxPossibleViewZDiff, maxPossibleViewZDiff);
             let fogColor = Vec3();
             Color.toVec3Normalized(fogColor, backgroundColor);

src/mol-gl/shader/postprocessing.frag.ts

@@ -16,6 +16,7 @@ uniform sampler2D tDepth;
 uniform sampler2D tOutlines;
 uniform vec2 uTexSize;
 
+uniform mat4 uInvProjection;
 uniform float uNear;
 uniform float uFar;
 uniform float uFogNear;
@@ -66,7 +67,7 @@ float getOutline(const in vec2 coords, out float closestTexel) {
     float selfViewZ = isBackground(selfDepth) ? backgroundViewZ : getViewZ(getDepth(coords));
 
     float outline = 1.0;
-    closestTexel = 1.0;
+    closestTexel = backgroundViewZ;
     for (float y = -uOutlineScale; y <= uOutlineScale; y++) {
         for (float x = -uOutlineScale; x <= uOutlineScale; x++) {
             if (x * x + y * y > uOutlineScale * uOutlineScale) {
@@ -79,7 +80,9 @@ float getOutline(const in vec2 coords, out float closestTexel) {
             float sampleOutline = sampleOutlineCombined.r;
             float sampleOutlineDepth = unpackRGToUnitInterval(sampleOutlineCombined.gb);
 
-            if (sampleOutline == 0.0 && sampleOutlineDepth < closestTexel && abs(selfViewZ - sampleOutlineDepth) > uMaxPossibleViewZDiff) {
+            float sampleOutlineViewDirLength = length(screenSpaceToViewSpace(vec3(sampleCoords, sampleOutlineDepth), uInvProjection));
+
+            if (sampleOutline == 0.0 && sampleOutlineViewDirLength < closestTexel && abs(selfViewZ - sampleOutlineDepth) > uMaxPossibleViewZDiff) {
                 outline = 0.0;
                 closestTexel = sampleOutlineDepth;
             }