AllenInstitute · froyo-np · Aug 30, 2024 · Aug 30, 2024 · Aug 31, 2024 · Sep 3, 2024
@@ -0,0 +1,13 @@
+<!doctype html>
+<html>
+    <body>
+        <canvas
+            id="glCanvas"
+            style="top: 0; left: 0%; width: 100%; height: 100%; position: absolute"
+        >
+    </body>
+</html>
+<script
+    type="module"
+    src="./src/constellation/constellation.ts"
+></script>
@@ -6,6 +6,7 @@
         <ul>
             <li><a href="/dzi">Deep Zoom Image</a><br /></li>
             <li><a href="/layers">Layers</a><br /></li>
+            <li><a href="/constellation">Constellation</a><br /></li>
         </ul>
     </body>
 </html>
@@ -39,6 +39,8 @@
         "@types/react": "^18.3.0",
         "@types/react-dom": "^18.3.0",
         "@vitejs/plugin-react-swc": "^3.5.0",
+        "gqty": "^3.2.1",
+        "graphql": "^16.9.0",
         "typescript": "^5.3.3",
         "vite": "^5.3.5"
     },

@@ -9,6 +9,30 @@
 5. `pnpm run dev`
 6. navigate to the running app (default `localhost://5173`). you can click a link to a specific example, or just use the address bar (`localhost://5173/{path_to_desired_example}`)
 
+## Constellation Example
+
+### Why?
+
+A POC for investigating the use of constellation plots for examining the cell type taxonomy as a graph, in which edges represent strong, asymetric similarity relationships between various taxons.
+Most of the data / technique in this vis comes from a paper (todo: find the paper and give proper attribution).
+
+Nodes in the graph are sized by the number of cells in that node. Nodes positions are the centroid of their constituent cells in the UMAP.
+Note that when zooming in, a "Big dot" aka Node aka Taxon is rendered simply by filling the appropriate radius with the cells that comprise that taxon. This fill is essentially random. Note also that "little" dot (aka cell) sizes are adaptive according to local density - this is an effort to ensure that no cell is ever fully occluded by any other; differences in little-dot sizes have no meaning in this plot.
+Edges are colored by class - each end of an edge is the color of the class of the node at the opposite end. (pink) green-->fade to--->pink (green)
+The weight of an edge at its ends indicates the strength of the connection in that direction. See the paper for the specifics of how edges are calculated.
+Edge curvature is purely aesthetic.
+
+This POC is very much a rough draft from a "useable software" perspective! It will flicker wildly when loading, which can take some time. It has no UI, just hotkeys:
+
+a: enable/disable animation. initially its disabled.
+0-4: transition to the corresponding layer: 0=Class, 1=SubClass, ... 4 = Umap cells
+c: cycle colorBy - dots will be colored by one of the taxonomy layers, starting with Class.
+o: decrease the ID used for filtering a cluster (turn that cluster white) by 1
+p: increase the ID used for filtering a cluster by 1
+scroll-wheel: zoom in/out
+click-drag: pan the view.
+hover: highlight the connections leaving or entering a taxon.
+
 ## DZI Example
 
 ### Why?

@@ -63,7 +63,7 @@ function toReglBuffer(c: ColumnData, regl: REGL.Regl) {
         data: regl.buffer(c),
     } as const;
 }
-function fetchAndUpload(
+export function fetchAndUpload(
     settings: { dataset: Dataset; regl: REGL.Regl },
     node: ColumnarNode<vec2>,
     req: ColumnRequest,

@@ -135,14 +135,14 @@ function convertTree2D(
         children:
             n.children !== undefined && n.children.length > 0
                 ? n.children.map((c) =>
-                      convertTree2D(
-                          c,
-                          getChildBoundsUsingPotreeIndexing(bounds, getRelativeIndex(safeName, sanitizeName(c.file))),
-                          depth + 1,
-                          metadataPath,
-                          genePath
-                      )
-                  )
+                    convertTree2D(
+                        c,
+                        getChildBoundsUsingPotreeIndexing(bounds, getRelativeIndex(safeName, sanitizeName(c.file))),
+                        depth + 1,
+                        metadataPath,
+                        genePath
+                    )
+                )
                 : [],
     };
 }
@@ -199,6 +199,7 @@ function loadSlideViewDataset(metadata: SlideColumnarMetadata, _datasetUrl: stri
         visualizationReferenceId,
     };
 }
+export type ScatterplotDataset = ReturnType<typeof loadDataset>;
 export type SlideViewDataset = ReturnType<typeof loadSlideViewDataset>;
 
 export function loadDataset(metadata: ColumnarMetadata, datasetUrl: string) {
@@ -230,7 +231,7 @@ export function loadDataset(metadata: ColumnarMetadata, datasetUrl: string) {
     };
 }
 
-type MetadataColumn = {
+export type MetadataColumn = {
     type: 'METADATA';
     name: string;
 };

@@ -0,0 +1,176 @@
+
+import {
+    type Renderer,
+    type ReglCacheEntry,
+    type CachedTexture,
+    buildAsyncRenderer,
+    type CachedVertexBuffer,
+} from '@alleninstitute/vis-scatterbrain';
+// lets see if we can use the simple renderer in a more complex case - constellations have multiple passes and pre-allocated (weird) data
+// also, they're animated, and not really spatially indexed...
+
+import type REGL from "regl";
+import { buildConstellationBuffers, exampleTaxonomy, type TaxonomyFeatures } from './loader';
+import { fetchColumn, type ColumnRequest, type ColumnarTree, type ColumnarNode, type ScatterplotDataset, type ColumnData } from '~/common/loaders/scatterplot/scatterbrain-loader';
+import { Box2D, Vec2, type box2D, type vec2 } from '@alleninstitute/vis-geometry';
+import { buildTaxonomyRenderer, type TaxonomyGpuBuffers, type TaxonomyRenderSettings, } from './taxonomy-renderer';
+import { getVisibleItems } from '~/common/loaders/scatterplot/data';
+import { buildEdgeRenderer } from './edge-renderer';
+
+type Constellation = ScatterplotDataset & TaxonomyFeatures
+type ConstellationChunk = ColumnarTree<vec2>
+
+export type ConstellationRenderSettings = Omit<TaxonomyRenderSettings, 'taxonomyPositions' | 'taxonomySize' | 'target'>
+
+
+function isPrepared(cacheData: Record<string, ReglCacheEntry | undefined>): cacheData is TaxonomyGpuBuffers {
+    return 'position' in cacheData &&
+        'Class' in cacheData &&
+        'SubClass' in cacheData &&
+        'SuperType' in cacheData &&
+        'Cluster' in cacheData &&
+        cacheData['position']?.type === 'buffer' &&
+        cacheData['Class']?.type === 'buffer' &&
+        cacheData['SubClass']?.type === 'buffer' &&
+        cacheData['SuperType']?.type === 'buffer' &&
+        cacheData['Cluster']?.type === 'buffer';
+
+}
+
+export async function buildConstellationRenderer(regl: REGL.Regl) {
+
+    const { texture, edgesByLevel, size } = await buildConstellationBuffers(exampleTaxonomy);
+    const taxonomyData = regl.texture({ data: texture, width: size[0], height: size[1], format: 'rgba', type: 'float' })
+    const edgeBuffers = edgesByLevel.map((lvl) => {
+        if (lvl) {
+            return { start: regl.buffer(lvl.start), end: regl.buffer(lvl.end), pStart: regl.buffer(lvl.pStart), pEnd: regl.buffer(lvl.pEnd), count: lvl.count }
+        }
+        return null;
+    });
+    // we just created some static resources that our renderer will use over and over.
+    // when we delete that renderer, we should clean up!
+    const destroy = () => {
+        taxonomyData.destroy();
+        edgeBuffers.map(layer => {
+            layer?.pEnd.destroy();
+            layer?.pStart.destroy();
+            layer?.start.destroy();
+            layer?.end.destroy();
+        });
+    }
+    const dotRenderCmd = buildTaxonomyRenderer(regl);
+    const edgeRenderCmd = buildEdgeRenderer(regl);
+    const dotRenderer: Renderer<Constellation, ConstellationChunk, ConstellationRenderSettings, TaxonomyGpuBuffers> = {
+        destroy,
+        cacheKey: (item, rqKey, dataset, settings) => cacheKey(item, rqKey, settings),
+        fetchItemContent: (item, dataset, settings, signal) => {
+            const { Class, SubClass, SuperType, Cluster } = dataset;
+            const fetchSettings = { dataset, regl };
+            const position = () =>
+                fetchAndUpload(fetchSettings, item.content, { type: 'METADATA', name: dataset.spatialColumn }, signal);
+            const cls = () => fetchAndUpload(fetchSettings, item.content, Class, signal);
+            const sub = () => fetchAndUpload(fetchSettings, item.content, SubClass, signal);
+            const spr = () => fetchAndUpload(fetchSettings, item.content, SuperType, signal);
+            const clstr = () => fetchAndUpload(fetchSettings, item.content, Cluster, signal);
+            return {
+                position,
+                Class: cls,
+                SubClass: sub,
+                SuperType: spr,
+                Cluster: clstr,
+            } as const;
+        },
+        getVisibleItems: (dataset, settings) => {
+            // because we move points around with our taxonomy shader, we cant rely on the positions in the quad-tree to 
+            // let us cut down the points we request... for now just get all of them!
+            const { view, screen } = settings.camera;
+            const unitsPerPixel = Vec2.div(Box2D.size(view), screen);
+            return getVisibleItems(dataset, dataset.bounds, 200 * unitsPerPixel[0])
+        },
+        isPrepared,
+        renderItem: (target, item, dataset, settings, gpuData) => {
+            dotRenderCmd(item, { ...settings, target, taxonomyPositions: taxonomyData, taxonomySize: size, }, gpuData);
+            // is here the right place to render the edges? maybe!
+        },
+    }
+
+    const edgeRenderer = (props: {
+        anmParam: number,
+        anmGoal: number,
+        view: box2D,
+        focus: vec2
+        target: REGL.Framebuffer2D | null,
+    }) => {
+        const { anmGoal, view, anmParam, target, focus } = props;
+        const stable = anmGoal == anmParam;
+        const animationDirection = stable ? (n: number) => n - Math.floor(n) : (n: number) => 1.0 - (n - Math.floor(n))
+        const edges = edgeBuffers[Math.ceil(anmParam)];
+        const parentEdges = edgeBuffers[Math.floor(anmParam)]
+        if (edges) {
+            edgeRenderCmd({
+                anmParam: animationDirection(anmParam),
+                color: [1, 1, 1, 1], // TODO remove unused
+                start: edges.start,
+                end: edges.end,
+                focus,
+                instances: edges.count,
+                pStart: edges.pStart,
+                pEnd: edges.pEnd,
+                target,
+                taxonLayer: Math.ceil(anmParam),
+                taxonomyPositions: taxonomyData,
+                taxonomySize: size,
+                view: Box2D.toFlatArray(view)
+            })
+        }
+        if (!stable && parentEdges) {
+            edgeRenderCmd({
+                color: [0.4, 0.45, 0.5, 0.8],
+                anmParam: 1.0 - animationDirection(anmParam),
+                taxonomyPositions: taxonomyData,
+                taxonomySize: size,
+                start: parentEdges.start,
+                end: parentEdges.end,
+                pStart: parentEdges.pStart,
+                pEnd: parentEdges.pEnd,
+                instances: parentEdges.count,
+                target,
+                taxonLayer: Math.floor(anmParam),
+                focus,
+                view: Box2D.toFlatArray(view)
+            })
+        }
+    }
+
+    return {
+        dotRenderer,
+        edgeRenderer
+    }
+
+}
+export async function buildAsyncConstellationRenderer(regl: REGL.Regl) {
+    const { dotRenderer, edgeRenderer } = await buildConstellationRenderer(regl);
+    return { dotRenderer: buildAsyncRenderer(dotRenderer), edgeRenderer }
+}
+
+// WARNING: this cache key is ok because this is a demo - 
+// make sure to carefully consider cache behavior here when following this example
+const cacheKey = (item: ColumnarTree<vec2>, reqKey: string, settings: ConstellationRenderSettings) =>
+    `${reqKey}:${item.content.name}`;
+
+function toReglBuffer(c: ColumnData, regl: REGL.Regl): CachedVertexBuffer {
+    return {
+        buffer: regl.buffer(c),
+        bytes: c.data.byteLength,
+        type: 'buffer'
+    }
+}
+function fetchAndUpload(
+    settings: { dataset: ScatterplotDataset; regl: REGL.Regl },
+    node: ColumnarNode<vec2>,
+    req: ColumnRequest,
+    signal?: AbortSignal | undefined
+) {
+    const { dataset, regl } = settings;
+    return fetchColumn(node, dataset, req, signal).then((cd) => toReglBuffer(cd, regl));
+}