From 25ab0d7799e3105411638a0474ccb2459f49d967 Mon Sep 17 00:00:00 2001
From: Alexander Rose <alexander.rose@weirdbyte.de>
Date: Sat, 11 Feb 2023 10:49:43 -0800
Subject: [PATCH] fix outline artefact with opaque behind transparent

---
 CHANGELOG.md                             | 1 +
 src/mol-gl/shader/outlines.frag.ts       | 8 +++++---
 src/mol-gl/shader/postprocessing.frag.ts | 7 +++++--
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9e8339369..707052713 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,7 @@ Note that since we don't clearly distinguish between a public and private interf
 ## [Unreleased]
 
 - Remove pca transform from components ui focus (too distracting)
+- Fix artefacts with opaque outlines behind transparent objects
 
 ## [v3.31.1] - 2023-02-05
 
diff --git a/src/mol-gl/shader/outlines.frag.ts b/src/mol-gl/shader/outlines.frag.ts
index 6c55edc06..443843945 100644
--- a/src/mol-gl/shader/outlines.frag.ts
+++ b/src/mol-gl/shader/outlines.frag.ts
@@ -1,8 +1,8 @@
 /**
- * Copyright (c) 2019-2022 mol* contributors, licensed under MIT, See LICENSE file for more info.
+ * Copyright (c) 2019-2023 mol* contributors, licensed under MIT, See LICENSE file for more info.
  *
  * @author Áron Samuel Kovács <aron.kovacs@mail.muni.cz>
- @author Alexander Rose <alexander.rose@weirdbyte.de>
+ * @author Alexander Rose <alexander.rose@weirdbyte.de>
  */
 
 export const outlines_frag = `
@@ -63,6 +63,7 @@ void main(void) {
 
     float outline = 1.0;
     float bestDepth = 1.0;
+    float transparentFlag = 0.0;
 
     for (int y = -1; y <= 1; y++) {
         for (int x = -1; x <= 1; x++) {
@@ -82,11 +83,12 @@ void main(void) {
                 if (abs(selfViewZTransparent - sampleViewZTransparent) > uMaxPossibleViewZDiff && selfDepthTransparent > sampleDepthTransparent && sampleDepthTransparent <= bestDepth) {
                     outline = 0.0;
                     bestDepth = sampleDepthTransparent;
+                    transparentFlag = 1.0;
                 }
             }
         }
     }
 
-    gl_FragColor = vec4(outline, packUnitIntervalToRG(bestDepth), 0.0);
+    gl_FragColor = vec4(outline, packUnitIntervalToRG(bestDepth), transparentFlag);
 }
 `;
\ No newline at end of file
diff --git a/src/mol-gl/shader/postprocessing.frag.ts b/src/mol-gl/shader/postprocessing.frag.ts
index d22deb145..49b335fc1 100644
--- a/src/mol-gl/shader/postprocessing.frag.ts
+++ b/src/mol-gl/shader/postprocessing.frag.ts
@@ -74,8 +74,10 @@ float getOutline(const in vec2 coords, const in float opaqueDepth, out float clo
     float backgroundViewZ = uFar + 3.0 * uMaxPossibleViewZDiff;
     vec2 invTexSize = 1.0 / uTexSize;
 
-    float selfDepth = min(opaqueDepth, getDepthTransparent(coords));
-    float selfViewZ = isBackground(selfDepth) ? backgroundViewZ : getViewZ(selfDepth);
+    float transparentDepth = getDepthTransparent(coords);
+    float opaqueSelfViewZ = isBackground(opaqueDepth) ? backgroundViewZ : getViewZ(opaqueDepth);
+    float transparentSelfViewZ = isBackground(transparentDepth) ? backgroundViewZ : getViewZ(transparentDepth);
+    float selfDepth = min(opaqueDepth, transparentDepth);
     float pixelSize = getPixelSize(coords, selfDepth);
 
     float outline = 1.0;
@@ -93,6 +95,7 @@ float getOutline(const in vec2 coords, const in float opaqueDepth, out float clo
             float sampleOutlineDepth = unpackRGToUnitInterval(sampleOutlineCombined.gb);
             float sampleOutlineViewZ = isBackground(sampleOutlineDepth) ? backgroundViewZ : getViewZ(sampleOutlineDepth);
 
+            float selfViewZ = sampleOutlineCombined.a == 0.0 ? opaqueSelfViewZ : transparentSelfViewZ;
             if (sampleOutline == 0.0 && sampleOutlineDepth < closestTexel && abs(selfViewZ - sampleOutlineViewZ) > uMaxPossibleViewZDiff + (pixelSize * outlineDistanceFactor)) {
                 outline = 0.0;
                 closestTexel = sampleOutlineDepth;
-- 
GitLab