fix outline artefact with opaque behind transparent

CHANGELOG.md

@@ -7,6 +7,7 @@ Note that since we don't clearly distinguish between a public and private interf
 ## [Unreleased]
 
 - Remove pca transform from components ui focus (too distracting)
+- Fix artefacts with opaque outlines behind transparent objects
 
 ## [v3.31.1] - 2023-02-05
 

src/mol-gl/shader/outlines.frag.ts

 /**
- * Copyright (c) 2019-2022 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2019-2023 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Áron Samuel Kovács <aron.kovacs@mail.muni.cz>
- @author Alexander Rose <alexander.rose@weirdbyte.de>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
 export const outlines_frag = `
@@ -63,6 +63,7 @@ void main(void) {
 
     float outline = 1.0;
     float bestDepth = 1.0;
+    float transparentFlag = 0.0;
 
     for (int y = -1; y <= 1; y++) {
         for (int x = -1; x <= 1; x++) {
@@ -82,11 +83,12 @@ void main(void) {
                 if (abs(selfViewZTransparent - sampleViewZTransparent) > uMaxPossibleViewZDiff && selfDepthTransparent > sampleDepthTransparent && sampleDepthTransparent <= bestDepth) {
                     outline = 0.0;
                     bestDepth = sampleDepthTransparent;
+                    transparentFlag = 1.0;
                 }
             }
         }
     }
 
-    gl_FragColor = vec4(outline, packUnitIntervalToRG(bestDepth), 0.0);
+    gl_FragColor = vec4(outline, packUnitIntervalToRG(bestDepth), transparentFlag);
 }
 `;
\ No newline at end of file

src/mol-gl/shader/postprocessing.frag.ts

@@ -74,8 +74,10 @@ float getOutline(const in vec2 coords, const in float opaqueDepth, out float clo
     float backgroundViewZ = uFar + 3.0 * uMaxPossibleViewZDiff;
     vec2 invTexSize = 1.0 / uTexSize;
 
-    float selfDepth = min(opaqueDepth, getDepthTransparent(coords));
-    float selfViewZ = isBackground(selfDepth) ? backgroundViewZ : getViewZ(selfDepth);
+    float transparentDepth = getDepthTransparent(coords);
+    float opaqueSelfViewZ = isBackground(opaqueDepth) ? backgroundViewZ : getViewZ(opaqueDepth);
+    float transparentSelfViewZ = isBackground(transparentDepth) ? backgroundViewZ : getViewZ(transparentDepth);
+    float selfDepth = min(opaqueDepth, transparentDepth);
     float pixelSize = getPixelSize(coords, selfDepth);
 
     float outline = 1.0;
@@ -93,6 +95,7 @@ float getOutline(const in vec2 coords, const in float opaqueDepth, out float clo
             float sampleOutlineDepth = unpackRGToUnitInterval(sampleOutlineCombined.gb);
             float sampleOutlineViewZ = isBackground(sampleOutlineDepth) ? backgroundViewZ : getViewZ(sampleOutlineDepth);
 
+            float selfViewZ = sampleOutlineCombined.a == 0.0 ? opaqueSelfViewZ : transparentSelfViewZ;
             if (sampleOutline == 0.0 && sampleOutlineDepth < closestTexel && abs(selfViewZ - sampleOutlineViewZ) > uMaxPossibleViewZDiff + (pixelSize * outlineDistanceFactor)) {
                 outline = 0.0;
                 closestTexel = sampleOutlineDepth;