handle clipping/fog for marking edges

src/mol-gl/shader/chunks/assign-material-color.glsl.ts

@@ -45,7 +45,11 @@ export const assign_material_color = `
         depthTest = (fragmentDepth >= getDepth(gl_FragCoord.xy / uDrawingBufferSize)) ? 1.0 : 0.0;
     }
     bool isHighlight = intMod(marker, 2.0) > 0.1;
-    vec4 material = vec4(0.0, depthTest, isHighlight ? 1.0 : 0.0, 1.0);
+    float viewZ = depthToViewZ(uIsOrtho, fragmentDepth, uNear, uFar);
+    float fogFactor = smoothstep(uFogNear, uFogFar, abs(viewZ));
+    if (fogFactor == 1.0)
+        discard;
+    vec4 material = vec4(0.0, depthTest, isHighlight ? 1.0 : 0.0, 1.0 - fogFactor);
 #endif
 
 // apply screendoor transparency

src/mol-gl/shader/marking/edge.frag.ts

@@ -23,6 +23,7 @@ void main() {
     float visibility = min(a1, a2) > 0.001 ? 1.0 : 0.0;
     float mask = c0.r;
     float marker = min(c1.b, min(c2.b, min(c3.b, c4.b)));
-    gl_FragColor = vec4(visibility, mask, marker, 1.0);
+    float fogAlpha = min(c1.a, min(c2.a, min(c3.a, c4.a)));
+    gl_FragColor = vec4(visibility, mask, marker, fogAlpha);
 }
 `;
\ No newline at end of file

src/mol-gl/shader/marking/overlay.frag.ts

@@ -15,7 +15,7 @@ void main() {
     if (edgeValue.a > 0.0) {
         vec3 edgeColor = edgeValue.b == 1.0 ? uHighlightEdgeColor : uSelectEdgeColor;
         gl_FragColor.rgb = edgeValue.g > 0.0 ? edgeColor : edgeColor * uInnerEdgeFactor;
-        gl_FragColor.a = edgeValue.r == 1.0 ? uGhostEdgeStrength : 1.0;
+        gl_FragColor.a = (edgeValue.r == 1.0 ? uGhostEdgeStrength : 1.0) * edgeValue.a;
     } else {
         gl_FragColor = vec4(0.0);
     }