ByoTrack fundamental features

[1]:

import cv2
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import torch

import byotrack
import byotrack.visualize

Loading a video

[2]:

video_path = "path/to/video.ext"

# Simply open a video
video = byotrack.Video(video_path)

fps = 20
# fps = video.reader.fps

# Note: video could also be a 4 dimensionnal numpy array loaded manually

[3]:

TEST = True  # Set to False to analyze a whole video

if TEST:
    video = video[:50]  # Temporal slicing to analyze only the first 50 frames

[4]:

# A transform can be added to normalize and aggregate channels

transform_config = byotrack.VideoTransformConfig(aggregate=True, normalize=True, q_min=0.01, q_max=0.995, smooth_clip=1.0)
video.set_transform(transform_config)

# Show the min max value used to clip and normalize
print(video._normalizer.mini, video._normalizer.maxi)

[401.] [843.]

[5]:

# Display the first frame

plt.figure(figsize=(24, 16), dpi=100)
plt.imshow(video[0])
plt.show()
../_images/run_examples_ByoTrack_fundamental_6_0.png 
[6]:

# Visualization
# Use w/x to move forward in time (or space to run/pause the video)

byotrack.visualize.InteractiveVisualizer(video).run()

Detections on a video: Example of WaveletDetector

[7]:

# Create the detector object with its hyper parameters
from byotrack.implementation.detector.wavelet import WaveletDetector

detector = WaveletDetector(scale=1, k=3.0, min_area=3.0, batch_size=20, device=torch.device("cpu"))

[8]:

# Run the detection process on the current video

detections_sequence = detector.run(video)

[9]:

# Display the first detections

segmentation = detections_sequence[0].segmentation.clone()
segmentation[segmentation!=0] += 50  # Improve visibility of firsts labels

plt.figure(figsize=(24, 16), dpi=100)
plt.imshow(segmentation)
plt.show()
../_images/run_examples_ByoTrack_fundamental_11_0.png 
[10]:

# Display the detections with opencv
# Use w/x to move forward in time (or space to run/pause the video)
# Use v to switch on/off the display of the video
# Use d to switch detection display mode (None, mask, segmentation)

byotrack.visualize.InteractiveVisualizer(video, detections_sequence).run()

[11]:

# Set hyperparameters manually on the video
# Use w/x to move backward/forward in the video
# Use c/v to update k (the main hyperparameter)
# You can restard with another scale/min_area

K_SPEED = 0.01

i = 0
detector = WaveletDetector(scale=1, k=3.0, min_area=3.0, device=torch.device("cpu"))

while True:
    frame = video[i]
    detections = detector.detect(frame[None, ...])[0]
    mask = (detections.segmentation.numpy() != 0).astype(np.uint8) * 255

    # Display the resulting frame
    cv2.imshow('Frame', mask)
    cv2.setWindowTitle('Frame', f'Frame {i} / {len(video)} - k={detector.k} - Num detections: {detections.length}')

    # Press Q on keyboard to  exit
    key = cv2.waitKey() & 0xFF

    if key == ord('q'):
        break

    if cv2.getWindowProperty("Frame", cv2.WND_PROP_VISIBLE) <1:
        break

    if key == ord("w"):
        i = (i - 1) % len(video)

    if key == ord("x"):
        i = (i + 1) % len(video)

    if key == ord("c"):
        detector.k = detector.k * (1 - K_SPEED)

    if key == ord("v"):
        detector.k = detector.k * (1 + K_SPEED)


cv2.destroyAllWindows()

Tracks refinement: Example of Cleaner, followed by EMC2 Stitcher

[17]:

from byotrack.implementation.refiner.cleaner import Cleaner
from byotrack.implementation.refiner.stitching import EMC2Stitcher

[18]:

# Split tracks with consecutive dist > 3.5
# Drop tracks with length < 5

cleaner = Cleaner(min_length=5, max_dist=3.5)
tracks = cleaner.run(video, tracks)

Cleaning: Split 1859 tracks and dropped 1471 resulting ones
Cleaning: From 1226 to 1614 tracks

[19]:

# Visualize track lifetime

byotrack.visualize.display_lifetime(tracks)
../_images/run_examples_ByoTrack_fundamental_23_0.png 
[20]:

# Stitch tracks together in order to produce coherent track on all the video

stitcher = EMC2Stitcher(eta=5.0)  # Don't link tracks if they are too far (EMC dist > 5 (pixels))
tracks = stitcher.run(video, tracks)

Merging 1614 tracks into 1133 resulting tracks

[21]:

# Visualize track lifetime

byotrack.visualize.display_lifetime(tracks)
../_images/run_examples_ByoTrack_fundamental_25_0.png

End-to-end tracking: Example of MultiStepTracker

[22]:

from byotrack import MultiStepTracker

[23]:

# Create all the steps: Detector, Linker[, Refiner]
# Then the tracker

detector = WaveletDetector(scale=1, k=3, min_area=3.0, batch_size=20, device=torch.device("cpu"))
linker = IcyEMHTLinker(icy_path, EMHTParameters(gate_factor=4.0, motion=motion, tree_depth=2))

# Optional refiner
refiners = []
if True:
    refiners = [Cleaner(5, 3.5), EMC2Stitcher(eta=5.0)]

tracker = MultiStepTracker(detector, linker, refiners)

[24]:

tracks = tracker.run(video)

Calling Icy with: java -jar icy.jar -hl -x plugins.adufour.protocols.Protocols protocol=/home/rreme/workspace/pasteur/byotrack/byotrack/implementation/linker/icy_emht/emht_protocol_with_full_specs.xml rois=/home/rreme/workspace/pasteur/byotrack/docs/source/run_examples/_tmp_rois.xml parameters=/home/rreme/workspace/pasteur/byotrack/docs/source/run_examples/_tmp_parameters.xml tracks=/home/rreme/workspace/pasteur/byotrack/docs/source/run_examples/_tmp_tracks.xml
[DEBUG] 2024-06-18 17:17:45 - Initializing...

JarResources.loadJar(/home/rreme/workspace/pasteur/icy-app-v3/plugins/nchenouard/particletracking/sequenceGenerator/._tracking-benchmark-generator-2.0.0.jar) error:
java.util.zip.ZipException: zip END header not found


[INFO] 2024-06-18 17:17:45 - Java(TM) SE Runtime Environment 21.0.1+12-LTS-29 (64 bit)
[INFO] 2024-06-18 17:17:45 - Running on Linux 5.15.0-107-generic (amd64)
[INFO] 2024-06-18 17:17:45 - System total memory: 32.6 GB
[INFO] 2024-06-18 17:17:45 - System available memory: 5.5 GB
[INFO] 2024-06-18 17:17:45 - Max Java memory: 8.2 GB
[INFO] 2024-06-18 17:17:46 - Headless mode.
[INFO] 2024-06-18 17:17:46 - Icy v3.0.0a started.
[DEBUG] 2024-06-18 17:17:46 - Magic name is Spot Tracking
[DEBUG] 2024-06-18 17:17:46 - Magic icon is /plugins/nchenouard/particletracking/simplifiedMHT/detectionIcon.png
[DEBUG] 2024-06-18 17:17:46 - Magic name is EzPlug tutorial
[DEBUG] 2024-06-18 17:17:46 - Magic icon is /plugins/adufour/ezplug/ezplug.png
Loading workflow...

Error(s) while loading protocol:
Variable 'useLPSolver' not found in block 'Spot tracking do tracking'. It may have been removed or renamed.
--

Converted from 38830 ROI(s)
Non binaryaction at frame 2
Non binaryaction at frame 3
Non binaryaction at frame 5
Non binaryaction at frame 9
Non binaryaction at frame 12
Non binaryaction at frame 15
Non binaryaction at frame 20
Non binaryaction at frame 22
Non binaryaction at frame 26
Non binaryaction at frame 27
Non binaryaction at frame 29
Non binary
Non binaryaction at frame 32
Non binaryaction at frame 34
Non binary
Non binary
Non binaryaction at frame 40
Non binaryaction at frame 42
Non binaryaction at frame 43
Non binary
Non binaryaction at frame 46
Non binaryaction at frame 47
Non binary
Non binaryaction at frame 49
Non binary
Non binary

[INFO] 2024-06-18 17:18:13 - Exiting...
[INFO] 2024-06-18 17:18:13 - Done.
Cleaning: Split 1859 tracks and dropped 1471 resulting ones
Cleaning: From 1226 to 1614 tracks

Merging 1614 tracks into 1133 resulting tracks

[25]:

# Visualize track lifetime

byotrack.visualize.display_lifetime(tracks)
../_images/run_examples_ByoTrack_fundamental_30_0.png 
[26]:

# Project tracks onto a single image and color by track

# Create a list of colors for each track (if more tracks than colors, it will cycle through it)

hsv = mpl.colormaps["hsv"]
colors = [tuple(int(c * 255) for c in hsv(k / 199)[:3]) for k in range(200)]

visu = byotrack.visualize.temporal_projection(
    byotrack.Track.tensorize(tracks),
    colors=colors,
)

plt.figure(figsize=(24, 16))
plt.imshow(visu)
plt.show()
../_images/run_examples_ByoTrack_fundamental_31_0.png 
[27]:

# Display the tracks with opencv
# Use w/x to move forward in time (or space to run/pause the video)
# Use v (resp. t) to switch on/off the display of video (resp. tracks)
# Use d to switch detection display mode (None, mask, segmentation)

byotrack.visualize.InteractiveVisualizer(video, detections_sequence, tracks).run()

Postprocessing: Fill NaN with interpolated values

[28]:

from byotrack.implementation.refiner.interpolater import ForwardBackwardInterpolater

[29]:

# After EMC2 stitching, NaN values can be inside merged tracks.
# It can be filled with interpolation between known positions

# Note that you can add this refiner to your MultiStepTracker pipeline

method = "tps"  # tps / constant / flow (You need to provided a valid byotrack.OpticalFlow then)
full = False  # Extrapolate position of the tracks on the all frame range and not just for the track lifespan

tracks = ForwardBackwardInterpolater(method, full).run(video, tracks)

[30]:

# Visualize track lifetime

byotrack.visualize.display_lifetime(tracks)
../_images/run_examples_ByoTrack_fundamental_36_0.png

Load or save tracks to files

[31]:

# Save tracks in ByoTrack format (compressed in a torch tensor)

byotrack.Track.save(tracks, "tracks.pt")

# Can be reload with
tracks = byotrack.Track.load("tracks.pt")

[32]:

# We also provide IO with Icy software

from byotrack import icy


icy.save_tracks(tracks, "tracks.xml")  # Note that holes should should be filled first with the ForwardBackwardInterpolater

# You can (re)load tracks from icy with
tracks = icy.load_tracks("tracks.xml")