From 7bc9c4f004855fc0d1b080011ca6179c74d2600a Mon Sep 17 00:00:00 2001 From: dburkhardt Date: Tue, 28 Nov 2023 15:46:19 +0000 Subject: [PATCH 01/14] Generalize dendrogram to multiple axes, fix #1549 --- ...rogram_key_added.py => test_dendrogram.py} | 5 + scanpy/tools/_dendrogram.py | 124 ++++++++++++------ 2 files changed, 88 insertions(+), 41 deletions(-) rename scanpy/tests/{test_dendrogram_key_added.py => test_dendrogram.py} (77%) diff --git a/scanpy/tests/test_dendrogram_key_added.py b/scanpy/tests/test_dendrogram.py similarity index 77% rename from scanpy/tests/test_dendrogram_key_added.py rename to scanpy/tests/test_dendrogram.py index 586612962a..23fbd69867 100644 --- a/scanpy/tests/test_dendrogram_key_added.py +++ b/scanpy/tests/test_dendrogram.py @@ -22,3 +22,8 @@ def test_dendrogram_key_added(groupby, key_added): if key_added is None: key_added = dendrogram_key assert key_added in adata.uns + +def test_dendrogram_var_names(): + adata = pbmc68k_reduced() + sc.tl.dendrogram(adata, var_names=adata.var_names[:10], axis=1) + assert len(adata.uns["dendrogram_var"]["categories_ordered"]) == 10 diff --git a/scanpy/tools/_dendrogram.py b/scanpy/tools/_dendrogram.py index ccae494748..de0f9092fd 100644 --- a/scanpy/tools/_dendrogram.py +++ b/scanpy/tools/_dendrogram.py @@ -9,11 +9,14 @@ import pandas as pd from pandas.api.types import CategoricalDtype +from anndata._core.merge import _resolve_dim + from .. import logging as logg from .._utils import _doc_params from ..neighbors._doc import doc_n_pcs, doc_use_rep from ._utils import _choose_representation + if TYPE_CHECKING: from collections.abc import Sequence @@ -23,7 +26,9 @@ @_doc_params(n_pcs=doc_n_pcs, use_rep=doc_use_rep) def dendrogram( adata: AnnData, - groupby: str | Sequence[str], + groupby: str | Sequence[str] | None = None, + dim: Literal["obs", "var"] | None = None, + axis: Literal[0, 1] | None = None, n_pcs: int | None = None, use_rep: str | None = None, var_names: Sequence[str] | None = None, @@ -60,6 +65,12 @@ def dendrogram( ---------- adata Annotated data matrix + groupby + Optional, the obs column(s) to use to group observations. Default is None. + dim + Dimension (obs or var) along which to calculate the dedrogram. Pass either `dim` or `axis` but not both. + axis + Axis (0 or 1) along which to calculate the dendrogram. Pass either `dim` or `axis` but not both. {n_pcs} {use_rep} var_names @@ -103,49 +114,77 @@ def dendrogram( >>> markers = ['C1QA', 'PSAP', 'CD79A', 'CD79B', 'CST3', 'LYZ'] >>> sc.pl.dotplot(adata, markers, groupby='bulk_labels', dendrogram=True) """ - if isinstance(groupby, str): - # if not a list, turn into a list - groupby = [groupby] - for group in groupby: - if group not in adata.obs_keys(): - raise ValueError( - "groupby has to be a valid observation. " - f"Given value: {group}, valid observations: {adata.obs_keys()}" - ) - if not isinstance(adata.obs[group].dtype, CategoricalDtype): - raise ValueError( - "groupby has to be a categorical observation. " - f"Given value: {group}, Column type: {adata.obs[group].dtype}" + if dim is None and axis is None: + dim = "obs" + axis, dim = _resolve_dim(axis=axis, dim=dim) + alt_axis, alt_dim = _resolve_dim(axis=1-axis) + + if dim == "obs": + dim_names = adata.obs_names + metadata = adata.obs + metadata_keys = adata.obs_keys() + else: + metadata = adata.var + metadata_keys = adata.var_keys() + + if groupby is not None: + if isinstance(groupby, str): + # if not a list, turn into a list + groupby = [groupby] + for group in groupby: + if group not in adata.obs_keys(): + raise ValueError( + "groupby has to be a valid observation. " + f"Given value: {group}, valid observations: {adata.obs_keys()}" + ) + if not isinstance(adata.obs[group].dtype, CategoricalDtype): + raise ValueError( + "groupby has to be a categorical observation. " + f"Given value: {group}, Column type: {adata.obs[group].dtype}" + ) + + if var_names is None: + rep_df = pd.DataFrame( + _choose_representation(adata, use_rep=use_rep, n_pcs=n_pcs) ) - - if var_names is None: - rep_df = pd.DataFrame( - _choose_representation(adata, use_rep=use_rep, n_pcs=n_pcs) - ) - categorical = adata.obs[groupby[0]] - if len(groupby) > 1: - for group in groupby[1:]: - # create new category by merging the given groupby categories - categorical = ( - categorical.astype(str) + "_" + adata.obs[group].astype(str) - ).astype("category") - categorical.name = "_".join(groupby) - - rep_df.set_index(categorical, inplace=True) - categories = rep_df.index.categories + categorical = adata.obs[groupby[0]] + if len(groupby) > 1: + for group in groupby[1:]: + # create new category by merging the given groupby categories + categorical = ( + categorical.astype(str) + "_" + adata.obs[group].astype(str) + ).astype("category") + categorical.name = "_".join(groupby) + + rep_df.set_index(categorical, inplace=True) + categories = rep_df.index.categories + else: + from scanpy.plotting._anndata import _prepare_dataframe + + categories, rep_df = _prepare_dataframe(adata, var_names, groupby, use_raw) + + # aggregate values within categories using 'mean' + rep_df = rep_df.groupby(level=0).mean() else: - gene_names = adata.raw.var_names if use_raw else adata.var_names - from ..plotting._anndata import _prepare_dataframe - - categories, rep_df = _prepare_dataframe(adata, gene_names, groupby, use_raw) - - # aggregate values within categories using 'mean' - mean_df = rep_df.groupby(level=0).mean() - + if var_names is None: + rep_df = pd.DataFrame( + _choose_representation(adata, use_rep=use_rep, n_pcs=n_pcs) + ) + + else: + rep_df = pd.DataFrame( + _choose_representation(adata[:, var_names], use_rep=use_rep, n_pcs=n_pcs) + ) + categories = rep_df.axes[axis] + import scipy.cluster.hierarchy as sch from scipy.spatial import distance - corr_matrix = mean_df.T.corr(method=cor_method) + + if axis == 0: + corr_matrix = rep_df.T.corr(method=cor_method) + else: + corr_matrix = rep_df.corr(method=cor_method) corr_condensed = distance.squareform(1 - corr_matrix) z_var = sch.linkage( corr_condensed, method=linkage_method, optimal_ordering=optimal_ordering @@ -166,8 +205,11 @@ def dendrogram( if inplace: if key_added is None: - key_added = f'dendrogram_{"_".join(groupby)}' + if groupby is None: + key_added = f"dendrogram_{dim}" + else: + key_added = f'dendrogram_{"_".join(groupby)}' logg.info(f"Storing dendrogram info using `.uns[{key_added!r}]`") adata.uns[key_added] = dat else: - return dat + return dat \ No newline at end of file From d0f189479ee082622045b6921e6b8ecd39940028 Mon Sep 17 00:00:00 2001 From: dburkhardt Date: Tue, 28 Nov 2023 15:47:23 +0000 Subject: [PATCH 02/14] Add dendrogram test for no groupby on obs --- scanpy/tests/test_dendrogram.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scanpy/tests/test_dendrogram.py b/scanpy/tests/test_dendrogram.py index 23fbd69867..437afcbbc3 100644 --- a/scanpy/tests/test_dendrogram.py +++ b/scanpy/tests/test_dendrogram.py @@ -9,13 +9,15 @@ key = "test" -@pytest.mark.parametrize("groupby", ["bulk_labels", ["bulk_labels", "phase"]]) +@pytest.mark.parametrize("groupby", [None, "bulk_labels", ["bulk_labels", "phase"]]) @pytest.mark.parametrize("key_added", [None, "custom_key"]) def test_dendrogram_key_added(groupby, key_added): adata = pbmc68k_reduced() sc.tl.dendrogram(adata, groupby=groupby, key_added=key_added, use_rep="X_pca") if isinstance(groupby, list): dendrogram_key = f'dendrogram_{"_".join(groupby)}' + elif groupby is None: + dendrogram_key = "dendrogram_obs" else: dendrogram_key = f"dendrogram_{groupby}" From 6b446032fc6343e9c474c86ccef2337b557fd541 Mon Sep 17 00:00:00 2001 From: dburkhardt Date: Tue, 28 Nov 2023 16:36:47 +0000 Subject: [PATCH 03/14] add release notes --- scanpy/tools/_dendrogram.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scanpy/tools/_dendrogram.py b/scanpy/tools/_dendrogram.py index de0f9092fd..95eed74aa4 100644 --- a/scanpy/tools/_dendrogram.py +++ b/scanpy/tools/_dendrogram.py @@ -16,7 +16,6 @@ from ..neighbors._doc import doc_n_pcs, doc_use_rep from ._utils import _choose_representation - if TYPE_CHECKING: from collections.abc import Sequence From 660853d9e5b7635fbdb503640a9716564106cf8c Mon Sep 17 00:00:00 2001 From: dburkhardt Date: Tue, 28 Nov 2023 16:46:57 +0000 Subject: [PATCH 04/14] lint --- docs/release-notes/1.10.0.md | 2 ++ scanpy/tests/test_dendrogram.py | 1 + scanpy/tools/_dendrogram.py | 36 +++++++++++++-------------------- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/docs/release-notes/1.10.0.md b/docs/release-notes/1.10.0.md index 856a66b233..46e2d36e7f 100644 --- a/docs/release-notes/1.10.0.md +++ b/docs/release-notes/1.10.0.md @@ -13,6 +13,7 @@ * Enhanced dask support for some internal utilities, paving the way for more extensive dask support {pr}`2696` {smaller}`P Angerer` * {func}`scanpy.pp.pca`, {func}`scanpy.pp.scale`, {func}`scanpy.pl.embedding`, and {func}`scanpy.experimental.pp.normalize_pearson_residuals_pca` now support a `mask` parameter {pr}`2272` {smaller}`C Bright, T Marcella, & P Angerer` +* {func}`scanpy.tl.dendrogram` now accepts and axis or dim argument to enable dendrograms on obs or var {pr}`2770` {smaller}`D Burkhardt` ```{rubric} Docs ``` @@ -26,6 +27,7 @@ * Fix {func}`~scanpy.pp.normalize_total` {pr}`2466` {smaller}`P Angerer` * Fix testing package build {pr}`2468` {smaller}`P Angerer` * Fix setting `sc.settings.verbosity` in some cases {pr}`2605` {smaller}`P Angerer` +* Fix `sc.tl.dendrogram` ignoring passed var_names ```{rubric} Ecosystem ``` diff --git a/scanpy/tests/test_dendrogram.py b/scanpy/tests/test_dendrogram.py index 437afcbbc3..c06d63aabc 100644 --- a/scanpy/tests/test_dendrogram.py +++ b/scanpy/tests/test_dendrogram.py @@ -25,6 +25,7 @@ def test_dendrogram_key_added(groupby, key_added): key_added = dendrogram_key assert key_added in adata.uns + def test_dendrogram_var_names(): adata = pbmc68k_reduced() sc.tl.dendrogram(adata, var_names=adata.var_names[:10], axis=1) diff --git a/scanpy/tools/_dendrogram.py b/scanpy/tools/_dendrogram.py index 95eed74aa4..6c4cb81b47 100644 --- a/scanpy/tools/_dendrogram.py +++ b/scanpy/tools/_dendrogram.py @@ -4,12 +4,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal import pandas as pd -from pandas.api.types import CategoricalDtype - from anndata._core.merge import _resolve_dim +from pandas.api.types import CategoricalDtype from .. import logging as logg from .._utils import _doc_params @@ -116,16 +115,8 @@ def dendrogram( if dim is None and axis is None: dim = "obs" axis, dim = _resolve_dim(axis=axis, dim=dim) - alt_axis, alt_dim = _resolve_dim(axis=1-axis) - - if dim == "obs": - dim_names = adata.obs_names - metadata = adata.obs - metadata_keys = adata.obs_keys() - else: - metadata = adata.var - metadata_keys = adata.var_keys() - + alt_axis, alt_dim = _resolve_dim(axis=1 - axis) + if groupby is not None: if isinstance(groupby, str): # if not a list, turn into a list @@ -141,7 +132,7 @@ def dendrogram( "groupby has to be a categorical observation. " f"Given value: {group}, Column type: {adata.obs[group].dtype}" ) - + if var_names is None: rep_df = pd.DataFrame( _choose_representation(adata, use_rep=use_rep, n_pcs=n_pcs) @@ -154,14 +145,14 @@ def dendrogram( categorical.astype(str) + "_" + adata.obs[group].astype(str) ).astype("category") categorical.name = "_".join(groupby) - + rep_df.set_index(categorical, inplace=True) categories = rep_df.index.categories else: from scanpy.plotting._anndata import _prepare_dataframe - + categories, rep_df = _prepare_dataframe(adata, var_names, groupby, use_raw) - + # aggregate values within categories using 'mean' rep_df = rep_df.groupby(level=0).mean() else: @@ -169,17 +160,18 @@ def dendrogram( rep_df = pd.DataFrame( _choose_representation(adata, use_rep=use_rep, n_pcs=n_pcs) ) - + else: rep_df = pd.DataFrame( - _choose_representation(adata[:, var_names], use_rep=use_rep, n_pcs=n_pcs) + _choose_representation( + adata[:, var_names], use_rep=use_rep, n_pcs=n_pcs + ) ) categories = rep_df.axes[axis] - + import scipy.cluster.hierarchy as sch from scipy.spatial import distance - if axis == 0: corr_matrix = rep_df.T.corr(method=cor_method) else: @@ -211,4 +203,4 @@ def dendrogram( logg.info(f"Storing dendrogram info using `.uns[{key_added!r}]`") adata.uns[key_added] = dat else: - return dat \ No newline at end of file + return dat From 042d13555759c9c0b4d4df9d53985a527fe124c9 Mon Sep 17 00:00:00 2001 From: dburkhardt Date: Tue, 28 Nov 2023 16:50:04 +0000 Subject: [PATCH 05/14] update release notes --- docs/release-notes/1.10.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes/1.10.0.md b/docs/release-notes/1.10.0.md index 46e2d36e7f..572a6b943a 100644 --- a/docs/release-notes/1.10.0.md +++ b/docs/release-notes/1.10.0.md @@ -13,7 +13,7 @@ * Enhanced dask support for some internal utilities, paving the way for more extensive dask support {pr}`2696` {smaller}`P Angerer` * {func}`scanpy.pp.pca`, {func}`scanpy.pp.scale`, {func}`scanpy.pl.embedding`, and {func}`scanpy.experimental.pp.normalize_pearson_residuals_pca` now support a `mask` parameter {pr}`2272` {smaller}`C Bright, T Marcella, & P Angerer` -* {func}`scanpy.tl.dendrogram` now accepts and axis or dim argument to enable dendrograms on obs or var {pr}`2770` {smaller}`D Burkhardt` +* {func}`scanpy.tl.dendrogram` now accepts and axis or dim argument to enable dendrograms on obs or var {pr}`2771` {smaller}`D Burkhardt` ```{rubric} Docs ``` From e36ee273adccb97023baf1089527f88caeb8381f Mon Sep 17 00:00:00 2001 From: Philipp A Date: Mon, 15 Jan 2024 12:24:47 +0100 Subject: [PATCH 06/14] Update scanpy/tools/_dendrogram.py Co-authored-by: Ilan Gold --- scanpy/tools/_dendrogram.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/scanpy/tools/_dendrogram.py b/scanpy/tools/_dendrogram.py index 6c4cb81b47..ef48934347 100644 --- a/scanpy/tools/_dendrogram.py +++ b/scanpy/tools/_dendrogram.py @@ -137,13 +137,10 @@ def dendrogram( rep_df = pd.DataFrame( _choose_representation(adata, use_rep=use_rep, n_pcs=n_pcs) ) - categorical = adata.obs[groupby[0]] - if len(groupby) > 1: - for group in groupby[1:]: - # create new category by merging the given groupby categories - categorical = ( - categorical.astype(str) + "_" + adata.obs[group].astype(str) - ).astype("category") + if len(groupby) == 1: + categorical = adata.obs[groupby[0]] + else: + categorical = adata.obs[groupby].apply("_".join, axis=1).astype("category") categorical.name = "_".join(groupby) rep_df.set_index(categorical, inplace=True) From d797d43f23b0dcb1824f7e5afc0cf71a0bb72bb3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 11:30:32 +0000 Subject: [PATCH 07/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scanpy/tools/_dendrogram.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scanpy/tools/_dendrogram.py b/scanpy/tools/_dendrogram.py index c941217fbe..78d3dcc923 100644 --- a/scanpy/tools/_dendrogram.py +++ b/scanpy/tools/_dendrogram.py @@ -153,7 +153,9 @@ def dendrogram( if len(groupby) == 1: categorical = adata.obs[groupby[0]] else: - categorical = adata.obs[groupby].apply("_".join, axis=1).astype("category") + categorical = ( + adata.obs[groupby].apply("_".join, axis=1).astype("category") + ) categorical.name = "_".join(groupby) rep_df.set_index(categorical, inplace=True) From fc88b9fb029996b1b0d94670b2b02629131c2682 Mon Sep 17 00:00:00 2001 From: Philipp A Date: Mon, 15 Jan 2024 12:31:43 +0100 Subject: [PATCH 08/14] remove unused var --- scanpy/tools/_dendrogram.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scanpy/tools/_dendrogram.py b/scanpy/tools/_dendrogram.py index 78d3dcc923..dcd5a1c15b 100644 --- a/scanpy/tools/_dendrogram.py +++ b/scanpy/tools/_dendrogram.py @@ -128,7 +128,6 @@ def dendrogram( if dim is None and axis is None: dim = "obs" axis, dim = _resolve_dim(axis=axis, dim=dim) - alt_axis, alt_dim = _resolve_dim(axis=1 - axis) if groupby is not None: if isinstance(groupby, str): From c34fd3a9186de990ca2c624cb162cec12354d362 Mon Sep 17 00:00:00 2001 From: Philipp A Date: Thu, 18 Jan 2024 15:21:07 +0100 Subject: [PATCH 09/14] adapt to #1244 --- scanpy/tools/_dendrogram.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/scanpy/tools/_dendrogram.py b/scanpy/tools/_dendrogram.py index dcd5a1c15b..b843bec048 100644 --- a/scanpy/tools/_dendrogram.py +++ b/scanpy/tools/_dendrogram.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Literal import pandas as pd -from anndata._core.merge import _resolve_dim +from anndata._core.merge import _resolve_axis from pandas.api.types import CategoricalDtype from .. import logging as logg @@ -38,8 +38,7 @@ def dendrogram( adata: AnnData, groupby: str | Sequence[str] | None = None, *, - dim: Literal["obs", "var"] | None = None, - axis: Literal[0, 1] | None = None, + axis: Literal["obs", 0, "var", 1] = "obs", n_pcs: int | None = None, use_rep: str | None = None, var_names: Sequence[str] | None = None, @@ -78,10 +77,8 @@ def dendrogram( Annotated data matrix groupby Optional, the obs column(s) to use to group observations. Default is None. - dim - Dimension (obs or var) along which to calculate the dedrogram. Pass either `dim` or `axis` but not both. axis - Axis (0 or 1) along which to calculate the dendrogram. Pass either `dim` or `axis` but not both. + Axis along which to calculate the dendrogram. {n_pcs} {use_rep} var_names @@ -125,9 +122,7 @@ def dendrogram( >>> markers = ['C1QA', 'PSAP', 'CD79A', 'CD79B', 'CST3', 'LYZ'] >>> sc.pl.dotplot(adata, markers, groupby='bulk_labels', dendrogram=True) """ - if dim is None and axis is None: - dim = "obs" - axis, dim = _resolve_dim(axis=axis, dim=dim) + axis, axis_name = _resolve_axis(axis) if groupby is not None: if isinstance(groupby, str): @@ -208,7 +203,7 @@ def dendrogram( if inplace: if key_added is None: if groupby is None: - key_added = f"dendrogram_{dim}" + key_added = f"dendrogram_{axis_name}" else: key_added = f'dendrogram_{"_".join(groupby)}' logg.info(f"Storing dendrogram info using `.uns[{key_added!r}]`") From 24acb72f28c429b703ae192443d5b93083319528 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 23 Jan 2024 15:15:14 +0100 Subject: [PATCH 10/14] add util --- scanpy/tests/test_dendrogram.py | 25 ++++++++++++++++--------- scanpy/tools/_dendrogram.py | 3 +-- scanpy/tools/_utils.py | 12 +++++++++++- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/scanpy/tests/test_dendrogram.py b/scanpy/tests/test_dendrogram.py index c06d63aabc..073ee4988b 100644 --- a/scanpy/tests/test_dendrogram.py +++ b/scanpy/tests/test_dendrogram.py @@ -9,17 +9,24 @@ key = "test" -@pytest.mark.parametrize("groupby", [None, "bulk_labels", ["bulk_labels", "phase"]]) -@pytest.mark.parametrize("key_added", [None, "custom_key"]) -def test_dendrogram_key_added(groupby, key_added): +@pytest.mark.parametrize( + ("groupby", "dendrogram_key"), + [ + pytest.param(None, "dendrogram_obs", id="obs"), + pytest.param("bulk_labels", "dendrogram_bulk_labels", id="bulk_labels"), + pytest.param( + ["bulk_labels", "phase"], + "dendrogram_bulk_labels_phase", + id="bulk_labels+phase", + ), + ], +) +@pytest.mark.parametrize( + "key_added", [None, "custom_key"], ids=["derived", "custom_key"] +) +def test_dendrogram_key_added(groupby, dendrogram_key, key_added): adata = pbmc68k_reduced() sc.tl.dendrogram(adata, groupby=groupby, key_added=key_added, use_rep="X_pca") - if isinstance(groupby, list): - dendrogram_key = f'dendrogram_{"_".join(groupby)}' - elif groupby is None: - dendrogram_key = "dendrogram_obs" - else: - dendrogram_key = f"dendrogram_{groupby}" if key_added is None: key_added = dendrogram_key diff --git a/scanpy/tools/_dendrogram.py b/scanpy/tools/_dendrogram.py index b843bec048..f9107202af 100644 --- a/scanpy/tools/_dendrogram.py +++ b/scanpy/tools/_dendrogram.py @@ -7,14 +7,13 @@ from typing import TYPE_CHECKING, Any, Literal import pandas as pd -from anndata._core.merge import _resolve_axis from pandas.api.types import CategoricalDtype from .. import logging as logg from .._compat import old_positionals from .._utils import _doc_params from ..neighbors._doc import doc_n_pcs, doc_use_rep -from ._utils import _choose_representation +from ._utils import _choose_representation, _resolve_axis if TYPE_CHECKING: from collections.abc import Sequence diff --git a/scanpy/tools/_utils.py b/scanpy/tools/_utils.py index 317948caf6..0c3e932d93 100644 --- a/scanpy/tools/_utils.py +++ b/scanpy/tools/_utils.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal import numpy as np @@ -129,3 +129,13 @@ def get_init_pos_from_paga( else: raise ValueError("Plot PAGA first, so that adata.uns['paga']" "with key 'pos'.") return init_pos + + +def _resolve_axis( + axis: Literal["obs", 0, "var", 1], +) -> tuple[Literal[0], Literal["obs"]] | tuple[Literal[1], Literal["var"]]: + if axis in {0, "obs"}: + return (0, "obs") + if axis in {1, "var"}: + return (1, "var") + raise ValueError(f"`axis` must be either 0, 1, 'obs', or 'var', was {axis}") From 4c9c2774ef04afbf36c993742b035ab3ad05071a Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 23 Jan 2024 15:22:55 +0100 Subject: [PATCH 11/14] Fix relnotes --- docs/release-notes/1.10.0.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/release-notes/1.10.0.md b/docs/release-notes/1.10.0.md index 4fa716f399..2430d5f3d6 100644 --- a/docs/release-notes/1.10.0.md +++ b/docs/release-notes/1.10.0.md @@ -11,7 +11,7 @@ now uses numba for variance computation {pr}`2612` {smaller}`S Dicks & P Angerer` * {func}`scanpy.external.pp.harmony_integrate` now runs with 64 bit floats improving reproducibility {pr}`2655` {smaller}`S Dicks` * Enhanced dask support for some internal utilities, paving the way for more extensive dask support {pr}`2696` {smaller}`P Angerer` -* {func}`scanpy.tl.dendrogram` now accepts and axis or dim argument to enable dendrograms on obs or var {pr}`2771` {smaller}`D Burkhardt` +* {func}`scanpy.tl.dendrogram` now accepts `axis` argument to enable dendrograms on `obs` or `var` {pr}`2771` {smaller}`D Burkhardt` * {func}`scanpy.pp.pca`, {func}`scanpy.pp.scale`, {func}`scanpy.pl.embedding`, and {func}`scanpy.experimental.pp.normalize_pearson_residuals_pca` now support a `mask` parameter {pr}`2272` {smaller}`C Bright, T Marcella, & P Angerer` * {func}`scanpy.tl.rank_genes_groups` no longer warns that it's default was changed from t-test_overestim_var to t-test {pr}`2798` {smaller}`L Heumos` @@ -29,7 +29,7 @@ * Fix {func}`~scanpy.pp.normalize_total` {pr}`2466` {smaller}`P Angerer` * Fix testing package build {pr}`2468` {smaller}`P Angerer` * Fix setting `sc.settings.verbosity` in some cases {pr}`2605` {smaller}`P Angerer` -* Fix `sc.tl.dendrogram` ignoring passed var_names {pr}`2771` {smaller}`D Burkhardt` +* Fix `sc.tl.dendrogram` ignoring passed `var_names` {pr}`2771` {smaller}`D Burkhardt` * Fix all remaining pandas warnings {pr}`2789` {smaller}`P Angerer` ```{rubric} Ecosystem From fa2a7815c9b2fd8644df023391fc2da907c7a7da Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 23 Jan 2024 15:32:54 +0100 Subject: [PATCH 12/14] extract --- scanpy/tools/_dendrogram.py | 127 ++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 57 deletions(-) diff --git a/scanpy/tools/_dendrogram.py b/scanpy/tools/_dendrogram.py index f9107202af..bc810f68f4 100644 --- a/scanpy/tools/_dendrogram.py +++ b/scanpy/tools/_dendrogram.py @@ -75,7 +75,7 @@ def dendrogram( adata Annotated data matrix groupby - Optional, the obs column(s) to use to group observations. Default is None. + The obs column(s) to use to group observations. Default is None. axis Axis along which to calculate the dendrogram. {n_pcs} @@ -127,60 +127,28 @@ def dendrogram( if isinstance(groupby, str): # if not a list, turn into a list groupby = [groupby] - for group in groupby: - if group not in adata.obs_keys(): - raise ValueError( - "groupby has to be a valid observation. " - f"Given value: {group}, valid observations: {adata.obs_keys()}" - ) - if not isinstance(adata.obs[group].dtype, CategoricalDtype): - raise ValueError( - "groupby has to be a categorical observation. " - f"Given value: {group}, Column type: {adata.obs[group].dtype}" - ) - - if var_names is None: - rep_df = pd.DataFrame( - _choose_representation(adata, use_rep=use_rep, n_pcs=n_pcs) - ) - if len(groupby) == 1: - categorical = adata.obs[groupby[0]] - else: - categorical = ( - adata.obs[groupby].apply("_".join, axis=1).astype("category") - ) - categorical.name = "_".join(groupby) - - rep_df.set_index(categorical, inplace=True) - categories = rep_df.index.categories - else: - from scanpy.plotting._anndata import _prepare_dataframe - - categories, rep_df = _prepare_dataframe(adata, var_names, groupby, use_raw) - - # aggregate values within categories using 'mean' - rep_df = rep_df.groupby(level=0, observed=True).mean() + rep_df, categories = _dendrogram_grouped( + adata, + groupby, + n_pcs=n_pcs, + use_rep=use_rep, + var_names=var_names, + use_raw=use_raw, + ) else: - if var_names is None: - rep_df = pd.DataFrame( - _choose_representation(adata, use_rep=use_rep, n_pcs=n_pcs) - ) - - else: - rep_df = pd.DataFrame( - _choose_representation( - adata[:, var_names], use_rep=use_rep, n_pcs=n_pcs - ) + rep_df = pd.DataFrame( + _choose_representation( + adata if var_names is None else adata[:, var_names], + use_rep=use_rep, + n_pcs=n_pcs, ) + ) categories = rep_df.axes[axis] import scipy.cluster.hierarchy as sch from scipy.spatial import distance - if axis == 0: - corr_matrix = rep_df.T.corr(method=cor_method) - else: - corr_matrix = rep_df.corr(method=cor_method) + corr_matrix = (rep_df.T if axis == 0 else rep_df).corr(method=cor_method) corr_condensed = distance.squareform(1 - corr_matrix) z_var = sch.linkage( corr_condensed, method=linkage_method, optimal_ordering=optimal_ordering @@ -199,13 +167,58 @@ def dendrogram( correlation_matrix=corr_matrix.values, ) - if inplace: - if key_added is None: - if groupby is None: - key_added = f"dendrogram_{axis_name}" - else: - key_added = f'dendrogram_{"_".join(groupby)}' - logg.info(f"Storing dendrogram info using `.uns[{key_added!r}]`") - adata.uns[key_added] = dat - else: + if not inplace: return dat + if key_added is None: + if groupby is None: + key_added = f"dendrogram_{axis_name}" + else: + key_added = f'dendrogram_{"_".join(groupby)}' + logg.info(f"Storing dendrogram info using `.uns[{key_added!r}]`") + adata.uns[key_added] = dat + return None + + +def _dendrogram_grouped( + adata: AnnData, + groupby: Sequence[str], + *, + n_pcs: int | None, + use_rep: str | None, + var_names: Sequence[str] | None, + use_raw: bool | None, +) -> tuple[pd.DataFrame, pd.Index[str]]: + for group in groupby: + if group not in adata.obs_keys(): + raise ValueError( + "groupby has to be a valid observation. " + f"Given value: {group}, valid observations: {adata.obs_keys()}" + ) + if not isinstance(adata.obs[group].dtype, CategoricalDtype): + raise ValueError( + "groupby has to be a categorical observation. " + f"Given value: {group}, Column type: {adata.obs[group].dtype}" + ) + + if var_names is None: + rep_df = pd.DataFrame( + _choose_representation(adata, use_rep=use_rep, n_pcs=n_pcs) + ) + if len(groupby) == 1: + categorical = adata.obs[groupby[0]] + else: + categorical = adata.obs[groupby].apply("_".join, axis=1).astype("category") + categorical.name = "_".join(groupby) + + rep_df.set_index(categorical, inplace=True) + categories = rep_df.index.categories + else: + from scanpy.plotting._anndata import _prepare_dataframe + + categories, rep_df = _prepare_dataframe( + adata, var_names, groupby, use_raw=use_raw + ) + + # aggregate values within categories using 'mean' + rep_df = rep_df.groupby(level=0, observed=True).mean() + return rep_df, categories From 01561a9effa2eefdc816687371359c30324ea689 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 6 Feb 2024 12:07:26 +0100 Subject: [PATCH 13/14] Use helper everywhere --- scanpy/plotting/_anndata.py | 60 ++++++++++++------------------ scanpy/plotting/_baseplot_class.py | 8 ++-- scanpy/tools/_dendrogram.py | 49 ++++++++++++++++++++---- 3 files changed, 70 insertions(+), 47 deletions(-) diff --git a/scanpy/plotting/_anndata.py b/scanpy/plotting/_anndata.py index c54ee5cc3b..0d5a63f2c7 100755 --- a/scanpy/plotting/_anndata.py +++ b/scanpy/plotting/_anndata.py @@ -21,6 +21,8 @@ from .._compat import old_positionals from .._settings import settings from .._utils import _check_use_raw, _doc_params, sanitize_anndata +from ..tools._dendrogram import _get_dendrogram_key +from ..tools._utils import _resolve_axis from . import _utils from ._docs import ( doc_common_plot_args, @@ -1737,8 +1739,9 @@ def tracksplot( @_doc_params(show_save_ax=doc_show_save_ax) def dendrogram( adata: AnnData, - groupby: str, + groupby: str | Sequence[str] | None = None, *, + axis: Literal["obs", 0, "var", 1] = "obs", dendrogram_key: str | None = None, orientation: Literal["top", "bottom", "left", "right"] = "top", remove_labels: bool = False, @@ -1757,6 +1760,8 @@ def dendrogram( Annotated data matrix. groupby Categorical data column used to create the dendrogram + axis + Axis for which to plot the dendrogram. dendrogram_key Key under with the dendrogram information was stored. By default the dendrogram information is stored under @@ -1787,10 +1792,12 @@ def dendrogram( """ if ax is None: _, ax = plt.subplots() + axis, axis_name = _resolve_axis(axis) _plot_dendrogram( ax, adata, groupby, + axis_name=axis_name, dendrogram_key=dendrogram_key, remove_labels=remove_labels, orientation=orientation, @@ -1866,7 +1873,9 @@ def correlation_matrix( >>> sc.pl.correlation_matrix(adata, 'bulk_labels') """ - dendrogram_key = _get_dendrogram_key(adata, dendrogram, groupby) + dendrogram_key = _get_dendrogram_key( + dendrogram, groupby, axis_name="obs", adata=adata + ) index = adata.uns[dendrogram_key]["categories_idx_ordered"] corr_matrix = adata.uns[dendrogram_key]["correlation_matrix"] @@ -2243,8 +2252,8 @@ def _plot_gene_groups_brackets( def _reorder_categories_after_dendrogram( adata: AnnData, - groupby, - dendrogram, + groupby: str | Sequence[str], + dendrogram: bool | str | None, *, var_names=None, var_group_labels=None, @@ -2269,12 +2278,14 @@ def _reorder_categories_after_dendrogram( 'var_group_labels', and 'var_group_positions' """ - key = _get_dendrogram_key(adata, dendrogram, groupby) + dendrogram_key = _get_dendrogram_key( + dendrogram, groupby, axis_name="obs", adata=adata + ) if isinstance(groupby, str): groupby = [groupby] - dendro_info = adata.uns[key] + dendro_info = adata.uns[dendrogram_key] if groupby != dendro_info["groupby"]: raise ValueError( "Incompatible observations. The precomputed dendrogram contains " @@ -2354,50 +2365,25 @@ def _format_first_three_categories(categories): return ", ".join(categories) -def _get_dendrogram_key(adata, dendrogram_key, groupby): - # the `dendrogram_key` can be a bool an NoneType or the name of the - # dendrogram key. By default the name of the dendrogram key is 'dendrogram' - if not isinstance(dendrogram_key, str): - if isinstance(groupby, str): - dendrogram_key = f"dendrogram_{groupby}" - elif isinstance(groupby, list): - dendrogram_key = f'dendrogram_{"_".join(groupby)}' - - if dendrogram_key not in adata.uns: - from ..tools._dendrogram import dendrogram - - logg.warning( - f"dendrogram data not found (using key={dendrogram_key}). " - "Running `sc.tl.dendrogram` with default parameters. For fine " - "tuning it is recommended to run `sc.tl.dendrogram` independently." - ) - dendrogram(adata, groupby, key_added=dendrogram_key) - - if "dendrogram_info" not in adata.uns[dendrogram_key]: - raise ValueError( - f"The given dendrogram key ({dendrogram_key!r}) does not contain " - "valid dendrogram information." - ) - - return dendrogram_key - - def _plot_dendrogram( dendro_ax: Axes, adata: AnnData, - groupby: str, + groupby: str | Sequence[str] | None, *, + axis_name: Literal["obs", "var"] = "obs", dendrogram_key: str | None = None, orientation: Literal["top", "bottom", "left", "right"] = "right", remove_labels: bool = True, ticks: Collection[float] | None = None, -): +) -> None: """\ Plots a dendrogram on the given ax using the precomputed dendrogram information stored in `.uns[dendrogram_key]` """ - dendrogram_key = _get_dendrogram_key(adata, dendrogram_key, groupby) + dendrogram_key = _get_dendrogram_key( + dendrogram_key, groupby, axis_name=axis_name, adata=adata + ) def translate_pos(pos_list, new_ticks, old_ticks): """\ diff --git a/scanpy/plotting/_baseplot_class.py b/scanpy/plotting/_baseplot_class.py index 22ff04913e..c204b46a8b 100644 --- a/scanpy/plotting/_baseplot_class.py +++ b/scanpy/plotting/_baseplot_class.py @@ -857,7 +857,7 @@ def savefig(self, filename: str, bbox_inches: str | None = "tight", **kwargs): self.make_figure() plt.savefig(filename, bbox_inches=bbox_inches, **kwargs) - def _reorder_categories_after_dendrogram(self, dendrogram) -> None: + def _reorder_categories_after_dendrogram(self, dendrogram_key: str | None) -> None: """\ Function used by plotting functions that need to reorder the the groupby observations based on the dendrogram results. @@ -883,9 +883,11 @@ def _format_first_three_categories(_categories): _categories = _categories[:3] + ["etc."] return ", ".join(_categories) - key = _get_dendrogram_key(self.adata, dendrogram, self.groupby) + dendrogram_key = _get_dendrogram_key( + dendrogram_key, self.groupby, axis_name="obs", adata=self.adata + ) - dendro_info = self.adata.uns[key] + dendro_info = self.adata.uns[dendrogram_key] if self.groupby != dendro_info["groupby"]: raise ValueError( "Incompatible observations. The precomputed dendrogram contains " diff --git a/scanpy/tools/_dendrogram.py b/scanpy/tools/_dendrogram.py index bc810f68f4..c2066e2065 100644 --- a/scanpy/tools/_dendrogram.py +++ b/scanpy/tools/_dendrogram.py @@ -4,6 +4,7 @@ from __future__ import annotations +from collections.abc import Sequence from typing import TYPE_CHECKING, Any, Literal import pandas as pd @@ -16,8 +17,6 @@ from ._utils import _choose_representation, _resolve_axis if TYPE_CHECKING: - from collections.abc import Sequence - from anndata import AnnData @@ -169,11 +168,7 @@ def dendrogram( if not inplace: return dat - if key_added is None: - if groupby is None: - key_added = f"dendrogram_{axis_name}" - else: - key_added = f'dendrogram_{"_".join(groupby)}' + key_added = _get_dendrogram_key(key_added, groupby, axis_name=axis_name) logg.info(f"Storing dendrogram info using `.uns[{key_added!r}]`") adata.uns[key_added] = dat return None @@ -222,3 +217,43 @@ def _dendrogram_grouped( # aggregate values within categories using 'mean' rep_df = rep_df.groupby(level=0, observed=True).mean() return rep_df, categories + + +def _get_dendrogram_key( + dendrogram_key: bool | str | None, + groupby: str | Sequence[str] | None, + *, + axis_name: Literal["obs", "var"], + adata: AnnData | None = None, +) -> str: + # the `dendrogram_key` can be a bool an NoneType or the name of the + # dendrogram key. By default the name of the dendrogram key is 'dendrogram' + if not isinstance(dendrogram_key, str): + if groupby is None: + dendrogram_key = f"dendrogram_{axis_name}" + elif isinstance(groupby, str): + dendrogram_key = f"dendrogram_{groupby}" + elif isinstance(groupby, Sequence): + dendrogram_key = f'dendrogram_{"_".join(groupby)}' + else: + msg = "Either `groupby` or `dendrogram_key` must be specified." + raise TypeError(msg) + + if adata is None: + return dendrogram_key + + if dendrogram_key not in adata.uns: + logg.warning( + f"dendrogram data not found (using key={dendrogram_key}). " + "Running `sc.tl.dendrogram` with default parameters. For fine " + "tuning it is recommended to run `sc.tl.dendrogram` independently." + ) + dendrogram(adata, groupby, key_added=dendrogram_key) + + if "dendrogram_info" not in adata.uns[dendrogram_key]: + raise ValueError( + f"The given dendrogram key ({dendrogram_key!r}) does not contain " + "valid dendrogram information." + ) + + return dendrogram_key From 521a590df85459a24987994bd36ee0ff13f2c3d4 Mon Sep 17 00:00:00 2001 From: "Philipp A." Date: Tue, 20 Feb 2024 09:01:49 +0100 Subject: [PATCH 14/14] error msg --- scanpy/tools/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanpy/tools/_utils.py b/scanpy/tools/_utils.py index 0c3e932d93..ffc2fca3cb 100644 --- a/scanpy/tools/_utils.py +++ b/scanpy/tools/_utils.py @@ -138,4 +138,4 @@ def _resolve_axis( return (0, "obs") if axis in {1, "var"}: return (1, "var") - raise ValueError(f"`axis` must be either 0, 1, 'obs', or 'var', was {axis}") + raise ValueError(f"`axis` must be either 0, 1, 'obs', or 'var', was {axis!r}")