ByoTrack fundamental features
[1]:
import cv2
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import torch
import byotrack
import byotrack.visualize
TEST = True # Set to False to analyze a whole video
Loading a video
[2]:
video_path = "path/to/video.ext"
# Simply open a video
video = byotrack.Video(video_path)
fps = 20
# fps = video.reader.fps
# Note: video could also be a 4 dimensionnal numpy array loaded manually
[3]:
# A transform can be added to normalize and aggregate channels
transform_config = byotrack.VideoTransformConfig(aggregate=True, normalize=True, q_min=0.01, q_max=0.995, smooth_clip=1.0)
video.set_transform(transform_config)
# Show the min max value used to clip and normalize
print(video._normalizer.mini, video._normalizer.maxi)
[401.] [843.]
[4]:
# Display the first frame
plt.figure(figsize=(24, 16), dpi=100)
plt.imshow(video[0])
plt.show()
[5]:
# Visualization
# Use w/x to move forward in time (or space to run/pause the video)
byotrack.visualize.InteractiveVisualizer(video).run()
Detections on a video: Example of WaveletDetector
[6]:
# Create the detector object with its hyper parameters
from byotrack.implementation.detector.wavelet import WaveletDetector
detector = WaveletDetector(scale=1, k=3.0, min_area=3.0, batch_size=20, device=torch.device("cpu"))
[7]:
# Run the detection process on the current video
if TEST: # Use slicing on video to run detection only on a part of it
detections_sequence = detector.run(video[:50])
else:
detections_sequence = detector.run(video)
Detections (Wavelet): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:03<00:00, 14.83it/s]
[8]:
# Display the first detections
segmentation = detections_sequence[0].segmentation.clone()
segmentation[segmentation!=0] += 50 # Improve visibility of firsts labels
plt.figure(figsize=(24, 16), dpi=100)
plt.imshow(segmentation)
plt.show()
[9]:
# Display the detections with opencv
# Use w/x to move forward in time (or space to run/pause the video)
# Use v to display the video
# Use d to switch detection display mode (None, mask, segmentation)
vis = byotrack.visualize.InteractiveVisualizer(video, detections_sequence)
vis._display_detections = 1 # Mask
vis._display_video = False
vis.run()
[10]:
# Set hyperparameters manually on the video
# Use w/x to move backward/forward in the video
# Use c/v to update k (the main hyperparameter)
# You can restard with another scale/min_area
K_SPEED = 0.01
i = 0
detector = WaveletDetector(scale=1, k=3.0, min_area=3.0, device=torch.device("cpu"))
while True:
frame = video[i]
detections = detector.detect(frame[None, ...])[0]
mask = (detections.segmentation.numpy() != 0).astype(np.uint8) * 255
# Display the resulting frame
cv2.imshow('Frame', mask)
cv2.setWindowTitle('Frame', f'Frame {i} / {len(video)} - k={detector.k} - Num detections: {detections.length}')
# Press Q on keyboard to exit
key = cv2.waitKey() & 0xFF
if key == ord('q'):
break
if cv2.getWindowProperty("Frame", cv2.WND_PROP_VISIBLE) <1:
break
if key == ord("w"):
i = (i - 1) % len(video)
if key == ord("x"):
i = (i + 1) % len(video)
if key == ord("c"):
detector.k = detector.k * (1 - K_SPEED)
if key == ord("v"):
detector.k = detector.k * (1 + K_SPEED)
cv2.destroyAllWindows()
Link detections: Example of IcyEMHTLinker
[11]:
from byotrack.implementation.linker.icy_emht import IcyEMHTLinker
# Create the linker object with icy path
# This Linker requires to install Icy software first
icy_path = "path/to/icy_dir/"
linker = IcyEMHTLinker(icy_path)
[12]:
# You can set the expected motion of particles
linker.motion = linker.Motion.BROWNIAN # Already by default
[13]:
# Run the linker given a video (Unused) and detections
if TEST: # Use slicing to run linker only on a part of the data
tracks = linker.run(video, detections_sequence[:50])
else:
tracks = linker.run(video, detections_sequence)
Calling Icy with: java -jar icy.jar -hl -x plugins.adufour.protocols.Protocols protocol=/home/rreme/workspace/pasteur/byotrack/byotrack/implementation/linker/icy_emht/emht_protocol.xml rois=/home/rreme/workspace/pasteur/byotrack/docs/source/run_examples/_tmp_rois.xml tracks=/home/rreme/workspace/pasteur/byotrack/docs/source/run_examples/_tmp_tracks.xml directed=0 multi=0
Initializing...
OpenJDK Runtime Environment 11.0.20+8-post-Ubuntu-1ubuntu120.04 (64 bit)
Running on Linux 5.15.0-79-generic (amd64)
Number of processors : 16
System total memory : 32.6 GB
System available memory : 9.4 GB
Max java memory : 8.2 GB
Image cache initialized (reserved memory = 3188 MB, disk cache location = '/tmp/icy_cache')
Headless mode.
java.lang.UnsatisfiedLinkError: /home/rreme/workspace/pasteur/icy_build/lib/unix64/vtk/libvtkRenderingCoreJava.so: libjawt.so: cannot open shared object file: No such file or directory
Cannot load VTK library...
Icy Version 2.4.3.0 started !
WARNING: An illegal reflective access operation has occurred
WARNING: Illegal reflective access by icy.util.ReflectionUtil (file:/home/rreme/workspace/pasteur/icy_build/icy.jar) to field java.lang.ClassLoader.usr_paths
WARNING: Please consider reporting this to the maintainers of icy.util.ReflectionUtil
WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
WARNING: All illegal access operations will be denied in a future release
Loading workflow...
Track extraction at frame 49
Exiting...EHCache disposed
Image cache shutdown..
done
[14]:
# Visualize track lifetime
byotrack.visualize.display_lifetime(tracks)
[15]:
# Display the detections with opencv
# Use w/x to move forward in time (or space to run/pause the video)
# Use v (resp. t) to switch on/off the display of video (resp. tracks)
# Use d to switch detection display mode (None, mask, segmentation)
vis = byotrack.visualize.InteractiveVisualizer(video, detections_sequence, tracks)
vis._display_tracks = 1
vis.run()
Tracks refinement: Example of Cleaner, followed by EMC2 Stitcher
[16]:
from byotrack.implementation.refiner.cleaner import Cleaner
from byotrack.implementation.refiner.stitching import EMC2Stitcher
[17]:
# Split tracks with consecutive dist > 3.5
# Drop tracks with length < 5
cleaner = Cleaner(min_length=5, max_dist=3.5)
tracks = cleaner.run(video, tracks)
Cleaning: Split 243 tracks and dropped 206 resulting ones
Cleaning: From 1397 to 1434 tracks
[18]:
# Visualize track lifetime
byotrack.visualize.display_lifetime(tracks)
[19]:
# Stitch tracks together in order to produce coherent track on all the video
stitcher = EMC2Stitcher(eta=5.0) # Don't link tracks if they are too far (EMC dist > 5 (pixels))
tracks = stitcher.run(video, tracks)
Forward propagation: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 49/49 [00:00<00:00, 329.52it/s]
Backward propagation: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 49/49 [00:00<00:00, 499.85it/s]
Merging 1434 tracks into 1146 resulting tracks
[20]:
# Visualize track lifetime
byotrack.visualize.display_lifetime(tracks)
End-to-end tracking: Example of MultiStepTracker
[21]:
from byotrack import MultiStepTracker
[22]:
# Create all the steps: Detector, Linker[, Refiner]
# Then the tracker
detector = WaveletDetector(scale=1, k=3, min_area=3.0, batch_size=20, device=torch.device("cpu"))
linker = IcyEMHTLinker(icy_path)
# Optional refiner
refiners = []
if True:
refiners = [Cleaner(5, 3.5), EMC2Stitcher(eta=5.0)]
tracker = MultiStepTracker(detector, linker, refiners)
[23]:
if TEST: # Use slicing on video to run tracker only on a part of it
tracks = tracker.run(video[:50])
else:
tracks = tracker.run(video)
Detections (Wavelet): 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:03<00:00, 14.61it/s]
Calling Icy with: java -jar icy.jar -hl -x plugins.adufour.protocols.Protocols protocol=/home/rreme/workspace/pasteur/byotrack/byotrack/implementation/linker/icy_emht/emht_protocol.xml rois=/home/rreme/workspace/pasteur/byotrack/docs/source/run_examples/_tmp_rois.xml tracks=/home/rreme/workspace/pasteur/byotrack/docs/source/run_examples/_tmp_tracks.xml directed=0 multi=0
Initializing...
OpenJDK Runtime Environment 11.0.20+8-post-Ubuntu-1ubuntu120.04 (64 bit)
Running on Linux 5.15.0-79-generic (amd64)
Number of processors : 16
System total memory : 32.6 GB
System available memory : 9.0 GB
Max java memory : 8.2 GB
Image cache initialized (reserved memory = 3188 MB, disk cache location = '/tmp/icy_cache')
Headless mode.
WARNING: An illegal reflective access operation has occurred
WARNING: Illegal reflective access by icy.util.ReflectionUtil (file:/home/rreme/workspace/pasteur/icy_build/icy.jar) to field java.lang.ClassLoader.usr_paths
WARNING: Please consider reporting this to the maintainers of icy.util.ReflectionUtil
WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
WARNING: All illegal access operations will be denied in a future release
java.lang.UnsatisfiedLinkError: /home/rreme/workspace/pasteur/icy_build/lib/unix64/vtk/libvtkRenderingCoreJava.so: libjawt.so: cannot open shared object file: No such file or directory
Cannot load VTK library...
Icy Version 2.4.3.0 started !
Loading workflow...
Track extraction at frame 49
Exiting...EHCache disposed
Image cache shutdown..
done
Cleaning: Split 243 tracks and dropped 206 resulting ones
Cleaning: From 1397 to 1434 tracks
Forward propagation: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 49/49 [00:00<00:00, 577.33it/s]
Backward propagation: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 49/49 [00:00<00:00, 563.04it/s]
Merging 1434 tracks into 1146 resulting tracks
[24]:
# Visualize track lifetime
byotrack.visualize.display_lifetime(tracks)
[25]:
# Display the detections with opencv
# Use w/x to move forward in time (or space to run/pause the video)
# Use v (resp. t) to switch on/off the display of video (resp. tracks)
# Use d to switch detection display mode (None, mask, segmentation)
vis = byotrack.visualize.InteractiveVisualizer(video, detections_sequence, tracks)
vis._display_tracks = 1
vis.run()
Postprocessing: Fill NaN with interpolated values
[26]:
from byotrack.implementation.refiner.interpolater import ForwardBackwardInterpolater
[27]:
# After EMC2 stitching, NaN values can be inside merged tracks.
# It can be filled with interpolation between known positions
# Note that you can add this refiner to your MultiStepTracker pipeline
method = "constant" # tps / constant
full = False # Extrapolate position of the tracks on the all frame range and not just for the track lifespan
tracks = ForwardBackwardInterpolater(method, full=False).run(video, tracks)
Forward propagation: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 49/49 [00:00<00:00, 14793.13it/s]
Backward propagation: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 49/49 [00:00<00:00, 15634.91it/s]
[28]:
# Visualize track lifetime
byotrack.visualize.display_lifetime(tracks)
Load or save tracks to files
[29]:
# Save tracks in ByoTrack format (compressed in a torch tensor)
byotrack.Track.save(tracks, "tracks.pth")
# Can be reload with
tracks = byotrack.Track.load("tracks.pth")
[30]:
# We also provide IO with Icy software
from byotrack import icy
icy.save_tracks(tracks, "tracks.xml") # Note that holes should should be filled first with the ForwardBackwardInterpolater
# You can (re)load tracks from icy with
tracks = icy.load_tracks("tracks.xml")