Detectors

Review of all the detectors implemented in ByoTrack ___________________________________________________

1. Wavelet Detector (Wavelet decomposition + noise filtering)

2. StarDist (Unet + StarConvex prior)

import cv2
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import torch

import byotrack
import byotrack.visualize

TEST = True  # Set to False to analyze a whole video

Load a video

video_path = "path/to/video.ext"

# Simply open a video
video = byotrack.Video(video_path)

fps = 20
# fps = video.reader.fps

# Note: video could also be a 4 dimensionnal numpy array

# A transform can be added to normalize and aggregate channels

transform_config = byotrack.VideoTransformConfig(aggregate=True, normalize=True, q_min=0.02, q_max=0.995, smooth_clip=1.0)
video.set_transform(transform_config)

# Show the min max value used to clip and normalize
print(video._normalizer.mini, video._normalizer.maxi)

[0.] [126.]

# Display the first frame

plt.figure(figsize=(24, 16), dpi=100)
plt.imshow(video[0])
plt.show()

WaveletDetector

from byotrack.implementation.detector.wavelet import WaveletDetector

WaveletDetector?

# Create the detector object with its hyper parameters.
# The most important ones being the scale (size of the spots) and k the threshold noise

detector = WaveletDetector(scale=1, k=3.0, min_area=3.0, batch_size=20, device=torch.device("cpu"))

# Set the hyperparameters manually on the video
# Use w/x to move backward/forward in the video
# Use c/v to update k (noise threshold)
# Use b/n to update the scale (expected size of the spots)
# You can also use the min_area to filter more or less spots given their area

K_SPEED = 0.01
scale = 1

i = 0
detector = WaveletDetector(scale=scale, k=3.0, min_area=3.0, device=torch.device("cpu"))

while True:
    frame = video[i]

    # Run detection on a single frame using detect
    detections = detector.detect(frame[None, ...])[0]
    mask = (detections.segmentation.numpy() != 0).astype(np.uint8) * 255

    image = np.concatenate((frame, np.zeros_like(frame), mask[..., None]), axis=-1)

    # Display the resulting frame
    cv2.imshow('Frame', image)
    cv2.setWindowTitle('Frame', f'Frame {i} / {len(video)} - scale={scale}, k={detector.k} - Num detections: {detections.length}')

    # Press Q on keyboard to  exit
    key = cv2.waitKey() & 0xFF

    if key == ord('q'):
        break

    if cv2.getWindowProperty("Frame", cv2.WND_PROP_VISIBLE) <1:
        break

    if key == ord("w"):
        i = (i - 1) % len(video)

    if key == ord("x"):
        i = (i + 1) % len(video)

    if key == ord("c"):
        detector.k = detector.k * (1 - K_SPEED)

    if key == ord("v"):
        detector.k = detector.k * (1 + K_SPEED)

    if key == ord("b"):
        scale = max(0, scale - 1)
        detector = WaveletDetector(scale, detector.k, detector.min_area, detector.device)


    if key == ord("n"):
        scale = min(4, scale + 1)
        detector = WaveletDetector(scale, detector.k, detector.min_area, detector.device)


cv2.destroyAllWindows()

# Run the detection process on the current video

if TEST:  # Use slicing on video to run detection only on a part of it
    detections_sequence = detector.run(video[:50])
else:
    detections_sequence = detector.run(video)

Detections (Wavelet): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:02<00:00, 17.99it/s]

StarDist

from byotrack.implementation.detector.stardist import StarDistDetector

StarDistDetector?

2023-09-02 16:28:06.147572: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2023-09-02 16:28:06.178625: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-09-02 16:28:06.674761: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT

# Create the detector object from a trained model. Note that the training should be done with
# the official implementation (https://github.com/stardist/stardist).
# Hyperparameters are set during the training phase (nms_treshold and prob_threshold)
# They can be changed manually (See next cells)

model_path = "path/to/trained/model/"
detector = StarDistDetector(model_path, batch_size=5)

stardist.py (41): Current implementation do not support batch size greater than 1 and will iterate image by image
2023-09-02 16:28:11.441460: I tensorflow/compiler/xla/stream_executor/cuda/cuda_gpu_executor.cc:996] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355
2023-09-02 16:28:11.441738: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1956] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...

Loading network weights from 'weights_best.h5'.
Loading thresholds from 'thresholds.json'.
Using default values: prob_thresh=0.710898, nms_thresh=0.3.

# Set the hyperparameters manually on the video
# Use w/x to move backward/forward in the video
# Use c/v to update prob_threshold (The most probable spots are kept)
# Use b/n to update the nms_threshold (Delete overlapping spots)
# You can also the min_area to filter more or less spots given their area

prob_speed = 0.1
nms_speed = 0.2

i = 0
detector = StarDistDetector(model_path, batch_size=5)

while True:
    frame = video[i]

    # Run detection on a single frame using detect
    detections = detector.detect(frame[None, ...])[0]
    mask = (detections.segmentation.numpy() != 0).astype(np.uint8) * 255

    image = np.concatenate((frame, np.zeros_like(frame), mask[..., None]), axis=-1)

    # Display the resulting frame
    cv2.imshow('Frame', image)
    cv2.setWindowTitle('Frame', f'Frame {i} / {len(video)} - prob={detector.prob_threshold}, nms={detector.nms_threshold} - Num detections: {detections.length}')

    # Press Q on keyboard to  exit
    key = cv2.waitKey() & 0xFF

    if key == ord('q'):
        break

    if cv2.getWindowProperty("Frame", cv2.WND_PROP_VISIBLE) <1:
        break

    if key == ord("w"):
        i = (i - 1) % len(video)

    if key == ord("x"):
        i = (i + 1) % len(video)

    if key == ord("c"):
        detector.prob_threshold = detector.prob_threshold * (1 - prob_speed)

    if key == ord("v"):
        detector.prob_threshold = detector.prob_threshold * (1 + prob_speed)

    if key == ord("b"):
        detector.nms_threshold = detector.nms_threshold * (1 - nms_speed)

    if key == ord("n"):
        detector.nms_threshold = detector.nms_threshold * (1 + nms_speed)


cv2.destroyAllWindows()

Loading network weights from 'weights_best.h5'.
Loading thresholds from 'thresholds.json'.
Using default values: prob_thresh=0.710898, nms_thresh=0.3.

# Run the detection process on the current video

if TEST:  # Use slicing on video to run detection only on a part of it
    detections_sequence = detector.run(video[:50])
else:
    detections_sequence = detector.run(video)

Detections (StarDist): 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:15<00:00,  3.16it/s]

Visualize the detections

# Display the first detections

segmentation = detections_sequence[0].segmentation.clone()
segmentation[segmentation!=0] += 50  # Improve visibility of firsts labels

plt.figure(figsize=(24, 16), dpi=100)
plt.imshow(segmentation)
plt.show()

# Display the detections with opencv
# Use w/x to move forward in time (or space to run/pause the video)
# Use v to switch on/off the display of video
# Use d to switch detection display mode (None, mask, segmentation)

vis = byotrack.visualize.InteractiveVisualizer(video, detections_sequence)
vis._display_detections = 1  # Mask

vis.run()