From 1c17277f0358706852a84c8e47da1f925a39867c Mon Sep 17 00:00:00 2001 From: Alexander Rose <alexander.rose@weirdbyte.de> Date: Sat, 31 Oct 2020 11:34:44 -0700 Subject: [PATCH] picking improvements - get 3d position from depth - option to render an object only in color pass --- src/mol-canvas3d/canvas3d.ts | 10 ++--- .../helper/bounding-sphere-helper.ts | 2 +- src/mol-canvas3d/helper/interaction-events.ts | 20 +++++----- src/mol-canvas3d/passes/pick.ts | 40 ++++++++++++++++--- src/mol-geo/geometry/base.ts | 1 + src/mol-geo/geometry/picking.ts | 5 --- src/mol-gl/_spec/renderer.spec.ts | 1 + src/mol-gl/renderable.ts | 1 + src/mol-gl/renderer.ts | 4 +- src/mol-plugin-state/manager/interactivity.ts | 6 +-- src/mol-repr/representation.ts | 6 ++- src/mol-repr/shape/representation.ts | 1 + src/mol-repr/structure/complex-visual.ts | 3 ++ src/mol-repr/structure/units-visual.ts | 3 ++ src/mol-repr/visual.ts | 5 +++ src/mol-repr/volume/representation.ts | 3 ++ src/tests/browser/render-shape.ts | 2 +- src/tests/browser/render-structure.ts | 2 +- 18 files changed, 82 insertions(+), 33 deletions(-) diff --git a/src/mol-canvas3d/canvas3d.ts b/src/mol-canvas3d/canvas3d.ts index 325eb6a90..f484230e7 100644 --- a/src/mol-canvas3d/canvas3d.ts +++ b/src/mol-canvas3d/canvas3d.ts @@ -27,7 +27,7 @@ import { Canvas3dInteractionHelper } from './helper/interaction-events'; import { PostprocessingParams, PostprocessingPass } from './passes/postprocessing'; import { MultiSampleParams, MultiSamplePass } from './passes/multi-sample'; import { DrawPass } from './passes/draw'; -import { PickPass } from './passes/pick'; +import { PickData, PickPass } from './passes/pick'; import { ImagePass, ImageProps } from './passes/image'; import { Sphere3D } from '../mol-math/geometry'; import { isDebugMode } from '../mol-util/debug'; @@ -100,7 +100,7 @@ interface Canvas3D { requestDraw(force?: boolean): void animate(): void pause(): void - identify(x: number, y: number): PickingId | undefined + identify(x: number, y: number): PickData | undefined mark(loci: Representation.Loci, action: MarkerAction): void getLoci(pickingId: PickingId | undefined): Representation.Loci @@ -132,9 +132,9 @@ const cancelAnimationFrame = typeof window !== 'undefined' : (handle: number) => clearImmediate(handle as unknown as NodeJS.Immediate); namespace Canvas3D { - export interface HoverEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys } + export interface HoverEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys, page?: Vec2, position?: Vec3 } export interface DragEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys, pageStart: Vec2, pageEnd: Vec2 } - export interface ClickEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys } + export interface ClickEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys, position?: Vec3 } export function fromCanvas(canvas: HTMLCanvasElement, props: PartialCanvas3DProps = {}, attribs: Partial<{ antialias: boolean, pixelScale: number }> = {}) { const gl = getGLContext(canvas, { @@ -346,7 +346,7 @@ namespace Canvas3D { animationFrameHandle = 0; } - function identify(x: number, y: number): PickingId | undefined { + function identify(x: number, y: number): PickData | undefined { return webgl.isContextLost ? undefined : pickPass.identify(x, y); } diff --git a/src/mol-canvas3d/helper/bounding-sphere-helper.ts b/src/mol-canvas3d/helper/bounding-sphere-helper.ts index 1b60e779c..163edef3b 100644 --- a/src/mol-canvas3d/helper/bounding-sphere-helper.ts +++ b/src/mol-canvas3d/helper/bounding-sphere-helper.ts @@ -160,5 +160,5 @@ const instanceMaterialId = getNextMaterialId(); function createBoundingSphereRenderObject(mesh: Mesh, color: Color, materialId: number, transform?: TransformData) { const values = Mesh.Utils.createValuesSimple(mesh, { alpha: 0.1, doubleSided: false }, color, 1, transform); - return createRenderObject('mesh', values, { visible: true, alphaFactor: 1, pickable: false, opaque: false, writeDepth: false }, materialId); + return createRenderObject('mesh', values, { visible: true, alphaFactor: 1, pickable: false, colorOnly: false, opaque: false, writeDepth: false }, materialId); } \ No newline at end of file diff --git a/src/mol-canvas3d/helper/interaction-events.ts b/src/mol-canvas3d/helper/interaction-events.ts index 5cde1ea86..53847848e 100644 --- a/src/mol-canvas3d/helper/interaction-events.ts +++ b/src/mol-canvas3d/helper/interaction-events.ts @@ -9,7 +9,7 @@ import { PickingId } from '../../mol-geo/geometry/picking'; import { Representation } from '../../mol-repr/representation'; import InputObserver, { ModifiersKeys, ButtonsType } from '../../mol-util/input/input-observer'; import { RxEventHelper } from '../../mol-util/rx-event-helper'; -import { Vec2 } from '../../mol-math/linear-algebra'; +import { Vec2, Vec3 } from '../../mol-math/linear-algebra'; import { Camera } from '../camera'; type Canvas3D = import('../canvas3d').Canvas3D @@ -34,6 +34,7 @@ export class Canvas3dInteractionHelper { private endY = -1; private id: PickingId | undefined = void 0; + private position: Vec3 | undefined = void 0; private currentIdentifyT = 0; private isInteracting = false; @@ -61,14 +62,16 @@ export class Canvas3dInteractionHelper { } if (xyChanged) { - this.id = this.canvasIdentify(this.endX, this.endY); + const pickData = this.canvasIdentify(this.endX, this.endY); + this.id = pickData?.id; + this.position = pickData?.position; this.startX = this.endX; this.startY = this.endY; } if (e === InputEvent.Click) { const loci = this.getLoci(this.id); - this.events.click.next({ current: loci, buttons: this.buttons, button: this.button, modifiers: this.modifiers }); + this.events.click.next({ current: loci, buttons: this.buttons, button: this.button, modifiers: this.modifiers, position: this.position }); this.prevLoci = loci; return; } @@ -78,11 +81,8 @@ export class Canvas3dInteractionHelper { } const loci = this.getLoci(this.id); - // only broadcast the latest hover - if (!Representation.Loci.areEqual(this.prevLoci, loci)) { - this.events.hover.next({ current: loci, buttons: this.buttons, button: this.button, modifiers: this.modifiers }); - this.prevLoci = loci; - } + this.events.hover.next({ current: loci, buttons: this.buttons, button: this.button, modifiers: this.modifiers, page: Vec2.create(this.endX, this.endY), position: this.position }); + this.prevLoci = loci; } tick(t: number) { @@ -129,9 +129,9 @@ export class Canvas3dInteractionHelper { } private modify(modifiers: ModifiersKeys) { - if (Representation.Loci.isEmpty(this.prevLoci) || ModifiersKeys.areEqual(modifiers, this.modifiers)) return; + if (ModifiersKeys.areEqual(modifiers, this.modifiers)) return; this.modifiers = modifiers; - this.events.hover.next({ current: this.prevLoci, buttons: this.buttons, button: this.button, modifiers: this.modifiers }); + this.events.hover.next({ current: this.prevLoci, buttons: this.buttons, button: this.button, modifiers: this.modifiers, page: Vec2.create(this.endX, this.endY), position: this.position }); } private outsideViewport(x: number, y: number) { diff --git a/src/mol-canvas3d/passes/pick.ts b/src/mol-canvas3d/passes/pick.ts index 004d003e1..7de7688e0 100644 --- a/src/mol-canvas3d/passes/pick.ts +++ b/src/mol-canvas3d/passes/pick.ts @@ -10,26 +10,32 @@ import Scene from '../../mol-gl/scene'; import { WebGLContext } from '../../mol-gl/webgl/context'; import { GraphicsRenderVariant } from '../../mol-gl/webgl/render-item'; import { RenderTarget } from '../../mol-gl/webgl/render-target'; -import { decodeFloatRGB } from '../../mol-util/float-packing'; +import { Vec3 } from '../../mol-math/linear-algebra'; +import { decodeFloatRGB, unpackRGBAToDepth } from '../../mol-util/float-packing'; import { Camera, ICamera } from '../camera'; import { StereoCamera } from '../camera/stereo'; +import { cameraUnproject } from '../camera/util'; import { HandleHelper } from '../helper/handle-helper'; import { DrawPass } from './draw'; const NullId = Math.pow(2, 24) - 2; +export type PickData = { id: PickingId, position: Vec3 } + export class PickPass { pickDirty = true objectPickTarget: RenderTarget instancePickTarget: RenderTarget groupPickTarget: RenderTarget + depthPickTarget: RenderTarget isStereo = false private objectBuffer: Uint8Array private instanceBuffer: Uint8Array private groupBuffer: Uint8Array + private depthBuffer: Uint8Array private pickScale: number private pickWidth: number @@ -43,6 +49,7 @@ export class PickPass { this.objectPickTarget = webgl.createRenderTarget(this.pickWidth, this.pickHeight); this.instancePickTarget = webgl.createRenderTarget(this.pickWidth, this.pickHeight); this.groupPickTarget = webgl.createRenderTarget(this.pickWidth, this.pickHeight); + this.depthPickTarget = webgl.createRenderTarget(this.pickWidth, this.pickHeight); this.setupBuffers(); } @@ -53,6 +60,7 @@ export class PickPass { this.objectBuffer = new Uint8Array(bufferSize); this.instanceBuffer = new Uint8Array(bufferSize); this.groupBuffer = new Uint8Array(bufferSize); + this.depthBuffer = new Uint8Array(bufferSize); } } @@ -68,6 +76,7 @@ export class PickPass { this.objectPickTarget.setSize(this.pickWidth, this.pickHeight); this.instancePickTarget.setSize(this.pickWidth, this.pickHeight); this.groupPickTarget.setSize(this.pickWidth, this.pickHeight); + this.depthPickTarget.setSize(this.pickWidth, this.pickHeight); this.setupBuffers(); } @@ -107,6 +116,9 @@ export class PickPass { this.groupPickTarget.bind(); this.renderVariant('pickGroup'); + this.depthPickTarget.bind(); + this.renderVariant('depth'); + this.pickDirty = false; } @@ -121,14 +133,27 @@ export class PickPass { this.groupPickTarget.bind(); webgl.readPixels(0, 0, this.pickWidth, this.pickHeight, this.groupBuffer); + + this.depthPickTarget.bind(); + webgl.readPixels(0, 0, this.pickWidth, this.pickHeight, this.depthBuffer); + } + + private getBufferIdx(x: number, y: number): number { + return (y * this.pickWidth + x) * 4; + } + + private getDepth(x: number, y: number): number { + const idx = this.getBufferIdx(x, y); + const b = this.depthBuffer; + return unpackRGBAToDepth(b[idx], b[idx + 1], b[idx + 2], b[idx + 3]); } private getId(x: number, y: number, buffer: Uint8Array) { - const idx = (y * this.pickWidth + x) * 4; + const idx = this.getBufferIdx(x, y); return decodeFloatRGB(buffer[idx], buffer[idx + 1], buffer[idx + 2]); } - identify(x: number, y: number): PickingId | undefined { + identify(x: number, y: number): PickData | undefined { const { webgl, pickScale, camera: { viewport } } = this; if (webgl.isContextLost) return; @@ -168,7 +193,12 @@ export class PickPass { const groupId = this.getId(xp, yp, this.groupBuffer); // console.log('groupId', groupId); if (groupId === -1 || groupId === NullId) return; - // console.log({ objectId, instanceId, groupId }); - return { objectId, instanceId, groupId }; + + const z = this.getDepth(xp, yp); + const position = Vec3.create(x, gl.drawingBufferHeight - y, z); + cameraUnproject(position, position, viewport, this.camera.inverseProjectionView); + + // console.log({ { objectId, instanceId, groupId }, position} ); + return { id: { objectId, instanceId, groupId }, position }; } } \ No newline at end of file diff --git a/src/mol-geo/geometry/base.ts b/src/mol-geo/geometry/base.ts index 3d2b13791..272add646 100644 --- a/src/mol-geo/geometry/base.ts +++ b/src/mol-geo/geometry/base.ts @@ -78,6 +78,7 @@ export namespace BaseGeometry { visible: true, alphaFactor: 1, pickable: true, + colorOnly: false, opaque, writeDepth: opaque, }; diff --git a/src/mol-geo/geometry/picking.ts b/src/mol-geo/geometry/picking.ts index 8ae9b7510..bf7e9a6cd 100644 --- a/src/mol-geo/geometry/picking.ts +++ b/src/mol-geo/geometry/picking.ts @@ -15,8 +15,3 @@ export namespace PickingId { return a.objectId === b.objectId && a.instanceId === b.instanceId && a.groupId === b.groupId; } } - -export interface PickingInfo { - label: string - data?: any -} \ No newline at end of file diff --git a/src/mol-gl/_spec/renderer.spec.ts b/src/mol-gl/_spec/renderer.spec.ts index 5c3eb8fdf..7c4872bd3 100644 --- a/src/mol-gl/_spec/renderer.spec.ts +++ b/src/mol-gl/_spec/renderer.spec.ts @@ -92,6 +92,7 @@ function createPoints() { visible: true, alphaFactor: 1, pickable: true, + colorOnly: false, opaque: true, writeDepth: true }; diff --git a/src/mol-gl/renderable.ts b/src/mol-gl/renderable.ts index 5f94a8575..51f087b79 100644 --- a/src/mol-gl/renderable.ts +++ b/src/mol-gl/renderable.ts @@ -17,6 +17,7 @@ export type RenderableState = { visible: boolean alphaFactor: number pickable: boolean + colorOnly: boolean opaque: boolean writeDepth: boolean, } diff --git a/src/mol-gl/renderer.ts b/src/mol-gl/renderer.ts index dedea00b3..3273c4382 100644 --- a/src/mol-gl/renderer.ts +++ b/src/mol-gl/renderer.ts @@ -368,7 +368,9 @@ namespace Renderer { } } else { // picking & depth for (let i = 0, il = renderables.length; i < il; ++i) { - renderObject(renderables[i], variant, depthTexture); + if (!renderables[i].state.colorOnly) { + renderObject(renderables[i], variant, depthTexture); + } } } diff --git a/src/mol-plugin-state/manager/interactivity.ts b/src/mol-plugin-state/manager/interactivity.ts index 12b6a6deb..87fe84af5 100644 --- a/src/mol-plugin-state/manager/interactivity.ts +++ b/src/mol-plugin-state/manager/interactivity.ts @@ -15,7 +15,7 @@ import { shallowEqual } from '../../mol-util/object'; import { ParamDefinition as PD } from '../../mol-util/param-definition'; import { StatefulPluginComponent } from '../component'; import { StructureSelectionManager } from './structure/selection'; -import { Vec2 } from '../../mol-math/linear-algebra'; +import { Vec2, Vec3 } from '../../mol-math/linear-algebra'; export { InteractivityManager }; @@ -70,9 +70,9 @@ namespace InteractivityManager { export type Params = typeof Params export type Props = PD.Values<Params> - export interface HoverEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys } + export interface HoverEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys, page?: Vec2, position?: Vec3 } export interface DragEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys, pageStart: Vec2, pageEnd: Vec2 } - export interface ClickEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys } + export interface ClickEvent { current: Representation.Loci, buttons: ButtonsType, button: ButtonsType.Flag, modifiers: ModifiersKeys, position?: Vec3 } export type LociMarkProvider = (loci: Representation.Loci, action: MarkerAction) => void diff --git a/src/mol-repr/representation.ts b/src/mol-repr/representation.ts index b9e0f19a7..5c3e5b9e5 100644 --- a/src/mol-repr/representation.ts +++ b/src/mol-repr/representation.ts @@ -174,6 +174,8 @@ namespace Representation { alphaFactor: number /** Controls if the representation's renderobjects are pickable or not */ pickable: boolean + /** Controls if the representation's renderobjects is rendered in color pass (i.e., not pick and depth) or not */ + colorOnly: boolean /** Overpaint applied to the representation's renderobjects */ overpaint: Overpaint /** Per group transparency applied to the representation's renderobjects */ @@ -188,12 +190,13 @@ namespace Representation { markerActions: MarkerActions } export function createState(): State { - return { visible: true, alphaFactor: 1, pickable: true, syncManually: false, transform: Mat4.identity(), overpaint: Overpaint.Empty, transparency: Transparency.Empty, clipping: Clipping.Empty, markerActions: MarkerActions.All }; + return { visible: true, alphaFactor: 1, pickable: true, colorOnly: false, syncManually: false, transform: Mat4.identity(), overpaint: Overpaint.Empty, transparency: Transparency.Empty, clipping: Clipping.Empty, markerActions: MarkerActions.All }; } export function updateState(state: State, update: Partial<State>) { if (update.visible !== undefined) state.visible = update.visible; if (update.alphaFactor !== undefined) state.alphaFactor = update.alphaFactor; if (update.pickable !== undefined) state.pickable = update.pickable; + if (update.colorOnly !== undefined) state.colorOnly = update.colorOnly; if (update.overpaint !== undefined) state.overpaint = update.overpaint; if (update.transparency !== undefined) state.transparency = update.transparency; if (update.clipping !== undefined) state.clipping = update.clipping; @@ -363,6 +366,7 @@ namespace Representation { if (state.visible !== undefined) Visual.setVisibility(renderObject, state.visible); if (state.alphaFactor !== undefined) Visual.setAlphaFactor(renderObject, state.alphaFactor); if (state.pickable !== undefined) Visual.setPickable(renderObject, state.pickable); + if (state.colorOnly !== undefined) Visual.setColorOnly(renderObject, state.colorOnly); if (state.overpaint !== undefined) { // TODO } diff --git a/src/mol-repr/shape/representation.ts b/src/mol-repr/shape/representation.ts index fa3da89a3..3ee72b35a 100644 --- a/src/mol-repr/shape/representation.ts +++ b/src/mol-repr/shape/representation.ts @@ -203,6 +203,7 @@ export function ShapeRepresentation<D, G extends Geometry, P extends Geometry.Pa if (state.visible !== undefined) Visual.setVisibility(_renderObject, state.visible); if (state.alphaFactor !== undefined) Visual.setAlphaFactor(_renderObject, state.alphaFactor); if (state.pickable !== undefined) Visual.setPickable(_renderObject, state.pickable); + if (state.colorOnly !== undefined) Visual.setColorOnly(_renderObject, state.colorOnly); if (state.overpaint !== undefined) { Visual.setOverpaint(_renderObject, state.overpaint, lociApply, true); } diff --git a/src/mol-repr/structure/complex-visual.ts b/src/mol-repr/structure/complex-visual.ts index 170e2347f..6b0b6a8b9 100644 --- a/src/mol-repr/structure/complex-visual.ts +++ b/src/mol-repr/structure/complex-visual.ts @@ -217,6 +217,9 @@ export function ComplexVisual<G extends Geometry, P extends StructureParams & Ge setPickable(pickable: boolean) { Visual.setPickable(renderObject, pickable); }, + setColorOnly(colorOnly: boolean) { + Visual.setColorOnly(renderObject, colorOnly); + }, setTransform(matrix?: Mat4, instanceMatrices?: Float32Array | null) { Visual.setTransform(renderObject, matrix, instanceMatrices); }, diff --git a/src/mol-repr/structure/units-visual.ts b/src/mol-repr/structure/units-visual.ts index 8f928c99b..aa1cb4e1a 100644 --- a/src/mol-repr/structure/units-visual.ts +++ b/src/mol-repr/structure/units-visual.ts @@ -274,6 +274,9 @@ export function UnitsVisual<G extends Geometry, P extends StructureParams & Geom setPickable(pickable: boolean) { Visual.setPickable(renderObject, pickable); }, + setColorOnly(colorOnly: boolean) { + Visual.setColorOnly(renderObject, colorOnly); + }, setTransform(matrix?: Mat4, instanceMatrices?: Float32Array | null) { Visual.setTransform(renderObject, matrix, instanceMatrices); }, diff --git a/src/mol-repr/visual.ts b/src/mol-repr/visual.ts index b999b848c..b163b9f3b 100644 --- a/src/mol-repr/visual.ts +++ b/src/mol-repr/visual.ts @@ -41,6 +41,7 @@ interface Visual<D, P extends PD.Params> { setVisibility: (visible: boolean) => void setAlphaFactor: (alphaFactor: number) => void setPickable: (pickable: boolean) => void + setColorOnly: (colorOnly: boolean) => void setTransform: (matrix?: Mat4, instanceMatrices?: Float32Array | null) => void setOverpaint: (overpaint: Overpaint) => void setTransparency: (transparency: Transparency) => void @@ -62,6 +63,10 @@ namespace Visual { if (renderObject) renderObject.state.pickable = pickable; } + export function setColorOnly(renderObject: GraphicsRenderObject | undefined, colorOnly: boolean) { + if (renderObject) renderObject.state.colorOnly = colorOnly; + } + export function mark(renderObject: GraphicsRenderObject | undefined, loci: Loci, action: MarkerAction, lociApply: LociApply) { if (!renderObject) return false; diff --git a/src/mol-repr/volume/representation.ts b/src/mol-repr/volume/representation.ts index 5bb1ffc32..48544c3b3 100644 --- a/src/mol-repr/volume/representation.ts +++ b/src/mol-repr/volume/representation.ts @@ -190,6 +190,9 @@ export function VolumeVisual<G extends Geometry, P extends VolumeParams & Geomet setPickable(pickable: boolean) { Visual.setPickable(renderObject, pickable); }, + setColorOnly(colorOnly: boolean) { + Visual.setColorOnly(renderObject, colorOnly); + }, setTransform(matrix?: Mat4, instanceMatrices?: Float32Array | null) { Visual.setTransform(renderObject, matrix, instanceMatrices); }, diff --git a/src/tests/browser/render-shape.ts b/src/tests/browser/render-shape.ts index 856293a4a..d34274456 100644 --- a/src/tests/browser/render-shape.ts +++ b/src/tests/browser/render-shape.ts @@ -41,7 +41,7 @@ let prevReprLoci = Representation.Loci.Empty; const canvas3d = Canvas3D.fromCanvas(canvas); canvas3d.animate(); canvas3d.input.move.subscribe(({x, y}) => { - const pickingId = canvas3d.identify(x, y); + const pickingId = canvas3d.identify(x, y)?.id; let label = ''; if (pickingId) { const reprLoci = canvas3d.getLoci(pickingId); diff --git a/src/tests/browser/render-structure.ts b/src/tests/browser/render-structure.ts index 8e155735c..9bd232b08 100644 --- a/src/tests/browser/render-structure.ts +++ b/src/tests/browser/render-structure.ts @@ -51,7 +51,7 @@ parent.appendChild(info); let prevReprLoci = Representation.Loci.Empty; canvas3d.input.move.pipe(throttleTime(100)).subscribe(({x, y}) => { - const pickingId = canvas3d.identify(x, y); + const pickingId = canvas3d.identify(x, y)?.id; let label = ''; if (pickingId) { const reprLoci = canvas3d.getLoci(pickingId); -- GitLab