Prepare a real Pancreas dataset for Cellucid

This notebook turns the checked-in pancreatic endocrinogenesis AnnData into a compact prepared dataset. It is intentionally small enough to run on a laptop and contains no private paths or hidden input files.

The example exports the real 2D UMAP, selected observation fields, the connectivity graph, and 128 highly variable genes. That keeps the tutorial quick while exercising the same prepare(...) contract used for larger datasets.

Use Prepare exports with quantization and compression for a detailed explanation of every export option.

import json
import shutil
import tempfile
from pathlib import Path

import anndata as ad
import numpy as np
from scipy import sparse

from cellucid import prepare, show

DATASET_RELATIVE_PATH = Path(
    "docs/user_guide/python_package/f_notebooks_tutorials/"
    "datasets/endocrinogenesis_day15.5_raw.h5ad"
)


def find_pancreas_h5ad(start: Path) -> Path:
    """Find the checked-in tutorial input from any directory inside the repo."""
    for base in (start.resolve(), *start.resolve().parents):
        candidate = base / DATASET_RELATIVE_PATH
        if candidate.is_file():
            return candidate
        candidate = base / "datasets" / DATASET_RELATIVE_PATH.name
        if candidate.is_file():
            return candidate
    raise FileNotFoundError(
        f"Could not find the checked-in Pancreas tutorial file {DATASET_RELATIVE_PATH}. "
        "Run this notebook from a clone of the cellucid-python repository."
    )


DATA_PATH = find_pancreas_h5ad(Path.cwd())
WORK_DIR = Path(tempfile.mkdtemp(prefix="cellucid-pancreas-tutorial-"))
EXPORT_DIR = WORK_DIR / "pancreas-day15-5"

print("Input:", DATA_PATH)
print("Temporary export:", EXPORT_DIR)

adata = ad.read_h5ad(DATA_PATH)

assert "X_umap_2d" in adata.obsm
assert "X_pca" in adata.obsm
assert "connectivities" in adata.obsp
assert "highly_variable_genes" in adata.var

highly_variable = adata.var["highly_variable_genes"].eq("True")
candidate_genes = adata.var_names[highly_variable]
candidate_expression = adata[:, candidate_genes].X
if sparse.issparse(candidate_expression):
    minima = candidate_expression.min(axis=0).toarray().ravel()
    maxima = candidate_expression.max(axis=0).toarray().ravel()
else:
    minima = np.asarray(candidate_expression).min(axis=0)
    maxima = np.asarray(candidate_expression).max(axis=0)
varying = np.isfinite(minima) & np.isfinite(maxima) & (maxima > minima)
genes = candidate_genes[varying][:128]
adata_export = adata[:, genes].copy()

print(adata)
print(f"Exporting {adata_export.n_obs:,} cells and {adata_export.n_vars:,} genes")

Build the prepared dataset

Prepared datasets split large arrays into independently cacheable, compressed resources. The browser can request the current embedding and coloring field without downloading every gene.

prepare(
    latent_space=adata.obsm["X_pca"],
    obs=adata.obs,
    obs_keys=["clusters", "clusters_coarse", "day", "phase", "palantir_pseudotime"],
    var=adata_export.var,
    gene_expression=adata_export.X,
    connectivities=adata.obsp["connectivities"],
    X_umap_2d=adata.obsm["X_umap_2d"],
    out_dir=EXPORT_DIR,
    dataset_name="Pancreatic endocrinogenesis (day 15.5)",
    dataset_id="pancreas-endocrinogenesis-day15-5-tutorial",
    dataset_description="A compact, real Pancreas export built by the Cellucid tutorial.",
    source_name="scVelo Pancreas vignette",
    source_url="https://scvelo.readthedocs.io/en/stable/vignettes/Pancreas.html",
    obs_categorical_dtype="uint16",
    obs_continuous_quantization=8,
    var_quantization=8,
    compression=6,
)

print("Prepared dataset written to", EXPORT_DIR)

identity = json.loads((EXPORT_DIR / "dataset_identity.json").read_text(encoding="utf-8"))
obs_manifest = json.loads((EXPORT_DIR / "obs_manifest.json").read_text(encoding="utf-8"))
var_manifest = json.loads((EXPORT_DIR / "var_manifest.json").read_text(encoding="utf-8"))

assert identity["id"] == "pancreas-endocrinogenesis-day15-5-tutorial"
assert (EXPORT_DIR / "points_2d.bin.gz").is_file()
assert (EXPORT_DIR / "connectivity_manifest.json").is_file()

obs_fields = [
    field[0]
    for group in ("_categoricalFields", "_continuousFields")
    for field in obs_manifest[group]
]

print("Embedding dimensions:", identity["embeddings"]["available_dimensions"])
print("Observation fields:", sorted(obs_fields))
print("Exported genes:", len(var_manifest["fields"]))

Open the prepared result

The prepared viewer uses the same UI as direct AnnData mode, but its resources are optimized for selective network loading.

viewer = show(EXPORT_DIR, height=650)
viewer

Clean up

Run this after you finish exploring. To keep the export, copy EXPORT_DIR elsewhere before cleanup.

viewer.stop()
shutil.rmtree(WORK_DIR)
print("Temporary export removed.")