diff --git a/deeplabcut/core/crossvalutils.py b/deeplabcut/core/crossvalutils.py index ca1447c61..8e8047f92 100644 --- a/deeplabcut/core/crossvalutils.py +++ b/deeplabcut/core/crossvalutils.py @@ -30,6 +30,8 @@ ) from deeplabcut.utils import auxfun_multianimal, auxiliaryfunctions +_trapz = getattr(np, "trapezoid", np.trapz) # NumPy 2.0+ compat; drop once NumPy 1 unsupported + def _set_up_evaluation(data): params = dict() @@ -84,7 +86,13 @@ def find_closest_neighbors(query: np.ndarray, ref: np.ndarray, k: int = 3) -> np return neighbors -def _calc_separability(vals_left, vals_right, n_bins=101, metric="jeffries", max_sensitivity=False): +def calc_separability( + vals_left: np.ndarray, + vals_right: np.ndarray, + n_bins: int = 101, + metric: str = "jeffries", + max_sensitivity: bool = False, +) -> tuple[float, float]: if metric not in ("jeffries", "auc"): raise ValueError("`metric` should be either 'jeffries' or 'auc'.") @@ -97,7 +105,7 @@ def _calc_separability(vals_left, vals_right, n_bins=101, metric="jeffries", max if metric == "jeffries": sep = np.sqrt(2 * (1 - np.sum(np.sqrt(hist_left * hist_right)))) # Jeffries-Matusita distance else: - sep = np.trapz(np.cumsum(hist_left), tpr) + sep = _trapz(np.cumsum(hist_left), tpr) if max_sensitivity: threshold = bins[max(1, np.argmax(tpr > 0))] else: @@ -344,7 +352,7 @@ def _get_n_best_paf_graphs( return ([existing_edges], dict(zip(existing_edges, [0] * len(existing_edges), strict=False))) scores, _ = zip( - *[_calc_separability(between_train[n], within_train[n], metric=metric) for n in existing_edges], strict=False + *[calc_separability(between_train[n], within_train[n], metric=metric) for n in existing_edges], strict=False ) # Find minimal skeleton diff --git a/deeplabcut/core/inferenceutils.py b/deeplabcut/core/inferenceutils.py index b31bd8581..f9f697ed2 100644 --- a/deeplabcut/core/inferenceutils.py +++ b/deeplabcut/core/inferenceutils.py @@ -1085,7 +1085,7 @@ def find_outlier_assemblies(dict_of_assemblies, criterion="area", qs=(5, 95)): tuples.append((frame_ind, getattr(assembly, criterion))) frame_inds, vals = zip(*tuples, strict=False) vals = np.asarray(vals) - lo, up = np.percentile(vals, qs, interpolation="nearest") + lo, up = np.percentile(vals, qs, method="nearest") inds = np.flatnonzero((vals < lo) | (vals > up)).tolist() return list(set(frame_inds[i] for i in inds)) diff --git a/deeplabcut/core/trackingutils.py b/deeplabcut/core/trackingutils.py index e897d1ae3..80084e488 100644 --- a/deeplabcut/core/trackingutils.py +++ b/deeplabcut/core/trackingutils.py @@ -539,7 +539,7 @@ def object_keypoint_similarity(x, y): xx = x[mask] yy = y[mask] dist = np.linalg.norm(xx - yy, axis=1) - scale = np.sqrt(np.product(np.ptp(yy, axis=0))) # square root of bounding box area + scale = np.sqrt(np.prod(np.ptp(yy, axis=0))) # square root of bounding box area oks = np.exp(-0.5 * (dist / (0.05 * scale)) ** 2) return np.mean(oks) diff --git a/deeplabcut/pose_estimation_pytorch/apis/prune_paf_graph.py b/deeplabcut/pose_estimation_pytorch/apis/prune_paf_graph.py index 0fb26e146..b4a5c6eda 100644 --- a/deeplabcut/pose_estimation_pytorch/apis/prune_paf_graph.py +++ b/deeplabcut/pose_estimation_pytorch/apis/prune_paf_graph.py @@ -23,7 +23,7 @@ import deeplabcut.pose_estimation_pytorch.data as data import deeplabcut.pose_estimation_pytorch.models.predictors as predictors import deeplabcut.utils.auxiliaryfunctions as auxiliaryfunctions -from deeplabcut.core.crossvalutils import find_closest_neighbors +from deeplabcut.core.crossvalutils import calc_separability, find_closest_neighbors from deeplabcut.pose_estimation_pytorch.models import PoseModel from deeplabcut.pose_estimation_pytorch.models.predictors.paf_predictor import Graph @@ -162,33 +162,6 @@ def benchmark_paf_graphs( return results -def _calc_separability( - vals_left: np.ndarray, - vals_right: np.ndarray, - n_bins: int = 101, - metric: str = "jeffries", - max_sensitivity: bool = False, -) -> tuple[float, float]: - if metric not in ("jeffries", "auc"): - raise ValueError("`metric` should be either 'jeffries' or 'auc'.") - - bins = np.linspace(0, 1, n_bins) - hist_left = np.histogram(vals_left, bins=bins)[0] - hist_left = hist_left / hist_left.sum() - hist_right = np.histogram(vals_right, bins=bins)[0] - hist_right = hist_right / hist_right.sum() - tpr = np.cumsum(hist_right) - if metric == "jeffries": - sep = np.sqrt(2 * (1 - np.sum(np.sqrt(hist_left * hist_right)))) # Jeffries-Matusita distance - else: - sep = np.trapz(np.cumsum(hist_left), tpr) - if max_sensitivity: - threshold = bins[max(1, np.argmax(tpr > 0))] - else: - threshold = bins[np.argmin(1 - np.cumsum(hist_left) + tpr)] - return sep, threshold - - @torch.no_grad() def compute_within_between_paf_costs( model: PoseModel, @@ -263,7 +236,7 @@ def get_n_best_paf_graphs( existing_edges = list(set(k for k, v in within_train.items() if v)) scores, _ = zip( - *[_calc_separability(between_train[n], within_train[n], metric=metric) for n in existing_edges], strict=False + *[calc_separability(between_train[n], within_train[n], metric=metric) for n in existing_edges], strict=False ) # Find minimal skeleton diff --git a/deeplabcut/pose_estimation_tensorflow/core/predict_multianimal.py b/deeplabcut/pose_estimation_tensorflow/core/predict_multianimal.py index dddc36899..47d7396b2 100644 --- a/deeplabcut/pose_estimation_tensorflow/core/predict_multianimal.py +++ b/deeplabcut/pose_estimation_tensorflow/core/predict_multianimal.py @@ -15,6 +15,8 @@ from scipy.ndimage import measurements from skimage.feature import peak_local_max +_trapz = getattr(np, "trapezoid", np.trapz) # NumPy 2.0+ compat; drop once NumPy 1 unsupported + def extract_cnn_output(outputs_np, cfg): """Extract locref, scmap and partaffinityfield from network.""" @@ -120,7 +122,7 @@ def compute_edge_costs( xy[..., 1], edge_inds.reshape((-1, 1)), ] - integ = np.trapz(y, xy[..., ::-1], axis=1) + integ = _trapz(y, xy[..., ::-1], axis=1) affinities = np.linalg.norm(integ, axis=1).astype(np.float32) # unit_vecs = vecs / lengths[:, np.newaxis] # affinities = np.squeeze(y @ np.expand_dims(unit_vecs, axis=2)).sum(axis=1) diff --git a/deeplabcut/refine_training_dataset/stitch.py b/deeplabcut/refine_training_dataset/stitch.py index a86c1d302..cbaccbc5c 100644 --- a/deeplabcut/refine_training_dataset/stitch.py +++ b/deeplabcut/refine_training_dataset/stitch.py @@ -22,7 +22,6 @@ import networkx as nx import numpy as np import pandas as pd -import scipy.linalg.interpolative as sli from networkx.algorithms.flow import preflow_push from scipy.linalg import hankel from scipy.spatial.distance import directed_hausdorff @@ -384,11 +383,11 @@ def estimate_rank(self, tol): 4/sqrt(3) """ mat = self.to_hankelet() - if np.any(mat): # check that the matrix contains non-zero entries + if np.any(mat): # nrows, ncols = mat.shape # beta = nrows / ncols # omega = 0.56 * beta ** 3 - 0.95 * beta ** 2 + 1.82 * beta + 1.43 - _, s, _ = sli.svd(mat, min(10, min(mat.shape))) + s = np.linalg.svd(mat, compute_uv=False)[:10] else: s = np.zeros(min(10, min(mat.shape))) diff --git a/deeplabcut/utils/plotting.py b/deeplabcut/utils/plotting.py index 48ae8715c..db1bfe269 100644 --- a/deeplabcut/utils/plotting.py +++ b/deeplabcut/utils/plotting.py @@ -464,7 +464,7 @@ def plot_edge_affinity_distributions( i1, i2 = graph[ind] w_tr = w_train[ind] b_tr = b_train[ind] - sep, _ = crossvalutils._calc_separability(b_tr, w_tr, metric="auc") + sep, _ = crossvalutils.calc_separability(b_tr, w_tr, metric="auc") axes[n].text( 0.5, 0.8, diff --git a/pyproject.toml b/pyproject.toml index efa7f7e70..9a1647e72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ "matplotlib>=3.3,<3.9,!=3.7,!=3.7.1", "networkx>=2.6", "numba>=0.54", - "numpy>=1.18.5,<2", + "numpy>=1.22.4,<2", "packaging>=26", # Migration to pandas 3.0 is tracked in https://github.com/DeepLabCut/DeepLabCut/issues/3362. "pandas[hdf5,performance]>=2.2,<3", diff --git a/tests/test_inferenceutils.py b/tests/test_inferenceutils.py index d91fb6c4a..59da7b69f 100644 --- a/tests/test_inferenceutils.py +++ b/tests/test_inferenceutils.py @@ -215,7 +215,15 @@ def test_assembler_with_unique_bodypart(real_assemblies_montblanc, test_data_dir assemblies_gt = np.concatenate( [ass.xy for assemblies in real_assemblies_montblanc[0].values() for ass in assemblies] ) - np.testing.assert_equal(assemblies, assemblies_gt) + + # Individuals are interchangeable in multi-animal assembly, so the order of + # equally-scored assemblies within a frame is arbitrary and depends on sort + # tie-breaking, which changed in NumPy 2.0. Compare the two point sets in a + # canonical, order-independent order (lexsort keeps NaN rows aligned). + def _canonical(a): + return a[np.lexsort((a[:, 1], a[:, 0]))] + + np.testing.assert_equal(_canonical(assemblies), _canonical(assemblies_gt)) def test_assembler_with_identity(tmpdir_factory, real_assemblies, test_data_dir): diff --git a/tests/test_stitcher.py b/tests/test_stitcher.py index 699ba96d7..5dbbb101e 100644 --- a/tests/test_stitcher.py +++ b/tests/test_stitcher.py @@ -261,3 +261,29 @@ def weight_func(t1, t2): assert all(0.998 <= track.likelihood <= 1 for track in stitcher.tracks) tracks = sorted(stitcher.tracks, key=lambda t: t.identity) assert all(track.identity == i for i, track in enumerate(tracks)) + + +_n_frames = 20 +_t = np.linspace(0, 4 * np.pi, _n_frames) + + +@pytest.mark.parametrize( + "signal, expected", + [ + (np.sin(_t), 2), + (np.sin(_t) + np.sin(2 * _t), 4), + ], +) +def test_estimate_rank(signal, expected): + """A Hankelet of a pure frequency signal has rank 2 per complex exponential + (two real dimensions); a second distinct frequency doubles it to rank 4. + """ + n_frames = len(signal) + data = np.zeros((n_frames, 2, 3)) + data[:, 0, 0] = signal + data[:, 0, 1] = signal + data[:, 1, 0] = signal + data[:, 1, 1] = signal + data[:, :, 2] = 1.0 + tracklet = Tracklet(data, np.arange(n_frames)) + assert tracklet.estimate_rank(tol=0.01) == expected