ByoTrack fundamental features

[1]:

import cv2
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import torch

import byotrack
import byotrack.visualize


TEST = True  # Set to False to analyze a whole video

Loading a video

[2]:

video_path = "path/to/video.ext"

# Simply open a video
video = byotrack.Video(video_path)

fps = 20
# fps = video.reader.fps

# Note: video could also be a 4 dimensionnal numpy array loaded manually

[3]:

# A transform can be added to normalize and aggregate channels

transform_config = byotrack.VideoTransformConfig(aggregate=True, normalize=True, q_min=0.01, q_max=0.995, smooth_clip=1.0)
video.set_transform(transform_config)

# Show the min max value used to clip and normalize
print(video._normalizer.mini, video._normalizer.maxi)

[401.] [843.]

[4]:

# Display the first frame

plt.figure(figsize=(24, 16), dpi=100)
plt.imshow(video[0])
plt.show()
../_images/run_examples_ByoTrack_fundamental_5_0.png 
[5]:

# Visualization
# Use w/x to move forward in time (or space to run/pause the video)

byotrack.visualize.InteractiveVisualizer(video).run()

Detections on a video: Example of WaveletDetector

[6]:

# Create the detector object with its hyper parameters
from byotrack.implementation.detector.wavelet import WaveletDetector

detector = WaveletDetector(scale=1, k=3.0, min_area=3.0, batch_size=20, device=torch.device("cpu"))

[7]:

# Run the detection process on the current video

if TEST:  # Use slicing on video to run detection only on a part of it
    detections_sequence = detector.run(video[:50])
else:
    detections_sequence = detector.run(video)

Detections (Wavelet): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:03<00:00, 14.83it/s]

[8]:

# Display the first detections

segmentation = detections_sequence[0].segmentation.clone()
segmentation[segmentation!=0] += 50  # Improve visibility of firsts labels

plt.figure(figsize=(24, 16), dpi=100)
plt.imshow(segmentation)
plt.show()
../_images/run_examples_ByoTrack_fundamental_10_0.png 
[9]:

# Display the detections with opencv
# Use w/x to move forward in time (or space to run/pause the video)
# Use v to switch on/off the display of the video
# Use d to switch detection display mode (None, mask, segmentation)

byotrack.visualize.InteractiveVisualizer(video, detections_sequence).run()

[10]:

# Set hyperparameters manually on the video
# Use w/x to move backward/forward in the video
# Use c/v to update k (the main hyperparameter)
# You can restard with another scale/min_area

K_SPEED = 0.01

i = 0
detector = WaveletDetector(scale=1, k=3.0, min_area=3.0, device=torch.device("cpu"))

while True:
    frame = video[i]
    detections = detector.detect(frame[None, ...])[0]
    mask = (detections.segmentation.numpy() != 0).astype(np.uint8) * 255

    # Display the resulting frame
    cv2.imshow('Frame', mask)
    cv2.setWindowTitle('Frame', f'Frame {i} / {len(video)} - k={detector.k} - Num detections: {detections.length}')

    # Press Q on keyboard to  exit
    key = cv2.waitKey() & 0xFF

    if key == ord('q'):
        break

    if cv2.getWindowProperty("Frame", cv2.WND_PROP_VISIBLE) <1:
        break

    if key == ord("w"):
        i = (i - 1) % len(video)

    if key == ord("x"):
        i = (i + 1) % len(video)

    if key == ord("c"):
        detector.k = detector.k * (1 - K_SPEED)

    if key == ord("v"):
        detector.k = detector.k * (1 + K_SPEED)


cv2.destroyAllWindows()

Tracks refinement: Example of Cleaner, followed by EMC2 Stitcher

[16]:

from byotrack.implementation.refiner.cleaner import Cleaner
from byotrack.implementation.refiner.stitching import EMC2Stitcher

[17]:

# Split tracks with consecutive dist > 3.5
# Drop tracks with length < 5

cleaner = Cleaner(min_length=5, max_dist=3.5)
tracks = cleaner.run(video, tracks)

Cleaning: Split 243 tracks and dropped 206 resulting ones
Cleaning: From 1397 to 1434 tracks

[18]:

# Visualize track lifetime

byotrack.visualize.display_lifetime(tracks)
../_images/run_examples_ByoTrack_fundamental_22_0.png 
[19]:

# Stitch tracks together in order to produce coherent track on all the video

stitcher = EMC2Stitcher(eta=5.0)  # Don't link tracks if they are too far (EMC dist > 5 (pixels))
tracks = stitcher.run(video, tracks)

Forward propagation: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 49/49 [00:00<00:00, 329.52it/s]
Backward propagation: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 49/49 [00:00<00:00, 499.85it/s]

Merging 1434 tracks into 1146 resulting tracks

[20]:

# Visualize track lifetime

byotrack.visualize.display_lifetime(tracks)
../_images/run_examples_ByoTrack_fundamental_24_0.png

End-to-end tracking: Example of MultiStepTracker

[21]:

from byotrack import MultiStepTracker

[22]:

# Create all the steps: Detector, Linker[, Refiner]
# Then the tracker

detector = WaveletDetector(scale=1, k=3, min_area=3.0, batch_size=20, device=torch.device("cpu"))
linker = IcyEMHTLinker(icy_path)

# Optional refiner
refiners = []
if True:
    refiners = [Cleaner(5, 3.5), EMC2Stitcher(eta=5.0)]

tracker = MultiStepTracker(detector, linker, refiners)

[23]:

if TEST:  # Use slicing on video to run tracker only on a part of it
    tracks = tracker.run(video[:50])
else:
    tracks = tracker.run(video)

Detections (Wavelet): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:03<00:00, 14.61it/s]

Calling Icy with: java -jar icy.jar -hl -x plugins.adufour.protocols.Protocols protocol=/home/rreme/workspace/pasteur/byotrack/byotrack/implementation/linker/icy_emht/emht_protocol.xml rois=/home/rreme/workspace/pasteur/byotrack/docs/source/run_examples/_tmp_rois.xml tracks=/home/rreme/workspace/pasteur/byotrack/docs/source/run_examples/_tmp_tracks.xml directed=0 multi=0
Initializing...

OpenJDK Runtime Environment 11.0.20+8-post-Ubuntu-1ubuntu120.04 (64 bit)
Running on Linux 5.15.0-79-generic (amd64)
Number of processors : 16
System total memory : 32.6 GB
System available memory : 9.0 GB
Max java memory : 8.2 GB
Image cache initialized (reserved memory = 3188 MB, disk cache location = '/tmp/icy_cache')
Headless mode.


WARNING: An illegal reflective access operation has occurred
WARNING: Illegal reflective access by icy.util.ReflectionUtil (file:/home/rreme/workspace/pasteur/icy_build/icy.jar) to field java.lang.ClassLoader.usr_paths
WARNING: Please consider reporting this to the maintainers of icy.util.ReflectionUtil
WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
WARNING: All illegal access operations will be denied in a future release

java.lang.UnsatisfiedLinkError: /home/rreme/workspace/pasteur/icy_build/lib/unix64/vtk/libvtkRenderingCoreJava.so: libjawt.so: cannot open shared object file: No such file or directory

Cannot load VTK library...

Icy Version 2.4.3.0 started !

Loading workflow...
Track extraction at frame 49
Exiting...EHCache disposed
Image cache shutdown..
 done
Cleaning: Split 243 tracks and dropped 206 resulting ones
Cleaning: From 1397 to 1434 tracks

Forward propagation: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 49/49 [00:00<00:00, 577.33it/s]
Backward propagation: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 49/49 [00:00<00:00, 563.04it/s]

Merging 1434 tracks into 1146 resulting tracks

[24]:

# Visualize track lifetime

byotrack.visualize.display_lifetime(tracks)
../_images/run_examples_ByoTrack_fundamental_29_0.png 
[25]:

# Display the tracks with opencv
# Use w/x to move forward in time (or space to run/pause the video)
# Use v (resp. t) to switch on/off the display of video (resp. tracks)
# Use d to switch detection display mode (None, mask, segmentation)

byotrack.visualize.InteractiveVisualizer(video, detections_sequence, tracks).run()

Postprocessing: Fill NaN with interpolated values

[26]:

from byotrack.implementation.refiner.interpolater import ForwardBackwardInterpolater

[27]:

# After EMC2 stitching, NaN values can be inside merged tracks.
# It can be filled with interpolation between known positions

# Note that you can add this refiner to your MultiStepTracker pipeline

method = "constant"  # tps / constant
full = False  # Extrapolate position of the tracks on the all frame range and not just for the track lifespan

tracks = ForwardBackwardInterpolater(method, full=False).run(video, tracks)

Forward propagation: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 49/49 [00:00<00:00, 14793.13it/s]
Backward propagation: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 49/49 [00:00<00:00, 15634.91it/s]

[28]:

# Visualize track lifetime

byotrack.visualize.display_lifetime(tracks)
../_images/run_examples_ByoTrack_fundamental_34_0.png

Load or save tracks to files

[29]:

# Save tracks in ByoTrack format (compressed in a torch tensor)

byotrack.Track.save(tracks, "tracks.pth")

# Can be reload with
tracks = byotrack.Track.load("tracks.pth")

[30]:

# We also provide IO with Icy software

from byotrack import icy


icy.save_tracks(tracks, "tracks.xml")  # Note that holes should should be filled first with the ForwardBackwardInterpolater

# You can (re)load tracks from icy with
tracks = icy.load_tracks("tracks.xml")