From bbf96567b175fe955ed6fadc49ab56e69a2d7f0b Mon Sep 17 00:00:00 2001
From: Alexander Rose <alexander.rose@weirdbyte.de>
Date: Sun, 29 Aug 2021 22:14:11 -0700
Subject: [PATCH] handle clipping/fog for marking edges

---
 src/mol-gl/shader/chunks/assign-material-color.glsl.ts | 6 +++++-
 src/mol-gl/shader/marking/edge.frag.ts                 | 3 ++-
 src/mol-gl/shader/marking/overlay.frag.ts              | 2 +-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/mol-gl/shader/chunks/assign-material-color.glsl.ts b/src/mol-gl/shader/chunks/assign-material-color.glsl.ts
index 55b3c4344..c147051ec 100644
--- a/src/mol-gl/shader/chunks/assign-material-color.glsl.ts
+++ b/src/mol-gl/shader/chunks/assign-material-color.glsl.ts
@@ -45,7 +45,11 @@ export const assign_material_color = `
         depthTest = (fragmentDepth >= getDepth(gl_FragCoord.xy / uDrawingBufferSize)) ? 1.0 : 0.0;
     }
     bool isHighlight = intMod(marker, 2.0) > 0.1;
-    vec4 material = vec4(0.0, depthTest, isHighlight ? 1.0 : 0.0, 1.0);
+    float viewZ = depthToViewZ(uIsOrtho, fragmentDepth, uNear, uFar);
+    float fogFactor = smoothstep(uFogNear, uFogFar, abs(viewZ));
+    if (fogFactor == 1.0)
+        discard;
+    vec4 material = vec4(0.0, depthTest, isHighlight ? 1.0 : 0.0, 1.0 - fogFactor);
 #endif
 
 // apply screendoor transparency
diff --git a/src/mol-gl/shader/marking/edge.frag.ts b/src/mol-gl/shader/marking/edge.frag.ts
index 475db2b0d..637b8533b 100644
--- a/src/mol-gl/shader/marking/edge.frag.ts
+++ b/src/mol-gl/shader/marking/edge.frag.ts
@@ -23,6 +23,7 @@ void main() {
     float visibility = min(a1, a2) > 0.001 ? 1.0 : 0.0;
     float mask = c0.r;
     float marker = min(c1.b, min(c2.b, min(c3.b, c4.b)));
-    gl_FragColor = vec4(visibility, mask, marker, 1.0);
+    float fogAlpha = min(c1.a, min(c2.a, min(c3.a, c4.a)));
+    gl_FragColor = vec4(visibility, mask, marker, fogAlpha);
 }
 `;
\ No newline at end of file
diff --git a/src/mol-gl/shader/marking/overlay.frag.ts b/src/mol-gl/shader/marking/overlay.frag.ts
index 2c8450ee9..5ba04ce00 100644
--- a/src/mol-gl/shader/marking/overlay.frag.ts
+++ b/src/mol-gl/shader/marking/overlay.frag.ts
@@ -15,7 +15,7 @@ void main() {
     if (edgeValue.a > 0.0) {
         vec3 edgeColor = edgeValue.b == 1.0 ? uHighlightEdgeColor : uSelectEdgeColor;
         gl_FragColor.rgb = edgeValue.g > 0.0 ? edgeColor : edgeColor * uInnerEdgeFactor;
-        gl_FragColor.a = edgeValue.r == 1.0 ? uGhostEdgeStrength : 1.0;
+        gl_FragColor.a = (edgeValue.r == 1.0 ? uGhostEdgeStrength : 1.0) * edgeValue.a;
     } else {
         gl_FragColor = vec4(0.0);
     }
-- 
GitLab