Spartan/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2024-11-27 03:10:34 -06:00
Code Issues Packages Projects Releases Wiki Activity

split into separate processes

Browse Source
This commit is contained in:
Blake Blackshear 2020-02-15 21:07:54 -06:00
parent ffa9534549
commit 569e07949f
10 changed files with 1234 additions and 560 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Dockerfile
View File

@ -26,10 +26,11 @@ RUN apt -qq update && apt -qq install --no-install-recommends -y \
scipy \
&& python3.7 -m pip install -U \
SharedArray \
# Flask \
# paho-mqtt \
# PyYAML \
# matplotlib \
Flask \
paho-mqtt \
PyYAML \
matplotlib \
pyarrow \
&& echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" > /etc/apt/sources.list.d/coral-edgetpu.list \
&& wget -q -O - https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
&& apt -qq update \

87
detect_objects.py
View File

@ -2,13 +2,16 @@ import cv2
import time
import queue
import yaml
import multiprocessing as mp
import subprocess as sp
import numpy as np
from flask import Flask, Response, make_response, jsonify
import paho.mqtt.client as mqtt
from frigate.video import Camera
from frigate.object_detection import PreppedQueueProcessor
from frigate.video import track_camera
from frigate.object_processing import TrackedObjectProcessor
from frigate.util import EventsPerSecond
from frigate.edgetpu import EdgeTPUProcess
with open('/config/config.yml') as f:
CONFIG = yaml.safe_load(f)
@ -38,8 +41,7 @@ FFMPEG_DEFAULT_CONFIG = {
'-stimeout', '5000000',
'-use_wallclock_as_timestamps', '1']),
'output_args': FFMPEG_CONFIG.get('output_args',
['-vf', 'mpdecimate',
'-f', 'rawvideo',
['-f', 'rawvideo',
'-pix_fmt', 'rgb24'])
}
@ -48,6 +50,10 @@ GLOBAL_OBJECT_CONFIG = CONFIG.get('objects', {})
WEB_PORT = CONFIG.get('web_port', 5000)
DEBUG = (CONFIG.get('debug', '0') == '1')
# MODEL_PATH = CONFIG.get('tflite_model', '/lab/mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite')
MODEL_PATH = CONFIG.get('tflite_model', '/lab/detect.tflite')
LABEL_MAP = CONFIG.get('label_map', '/lab/labelmap.txt')
def main():
# connect to mqtt and setup last will
def on_connect(client, userdata, flags, rc):
@ -71,27 +77,43 @@ def main():
client.connect(MQTT_HOST, MQTT_PORT, 60)
client.loop_start()
# Queue for prepped frames, max size set to number of regions * 3
prepped_frame_queue = queue.Queue()
# start plasma store
plasma_cmd = ['plasma_store', '-m', '400000000', '-s', '/tmp/plasma']
plasma_process = sp.Popen(plasma_cmd, stdout=sp.DEVNULL, stderr=sp.DEVNULL)
cameras = {}
##
# Setup config defaults for cameras
##
for name, config in CONFIG['cameras'].items():
cameras[name] = Camera(name, FFMPEG_DEFAULT_CONFIG, GLOBAL_OBJECT_CONFIG, config,
prepped_frame_queue, client, MQTT_TOPIC_PREFIX)
config['snapshots'] = {
'show_timestamp': config.get('snapshots', {}).get('show_timestamp', True)
}
fps_tracker = EventsPerSecond()
# Queue for cameras to push tracked objects to
tracked_objects_queue = mp.Queue()
prepped_queue_processor = PreppedQueueProcessor(
cameras,
prepped_frame_queue,
fps_tracker
)
prepped_queue_processor.start()
fps_tracker.start()
# Start the shared tflite process
tflite_process = EdgeTPUProcess(MODEL_PATH)
for name, camera in cameras.items():
camera.start()
print("Capture process for {}: {}".format(name, camera.get_capture_pid()))
camera_processes = []
camera_stats_values = {}
for name, config in CONFIG['cameras'].items():
camera_stats_values[name] = {
'fps': mp.Value('d', 10.0),
'avg_wait': mp.Value('d', 0.0)
}
camera_process = mp.Process(target=track_camera, args=(name, config, FFMPEG_DEFAULT_CONFIG, GLOBAL_OBJECT_CONFIG,
tflite_process.detect_lock, tflite_process.detect_ready, tflite_process.frame_ready, tracked_objects_queue,
camera_stats_values[name]['fps'], camera_stats_values[name]['avg_wait']))
camera_process.daemon = True
camera_processes.append(camera_process)
for camera_process in camera_processes:
camera_process.start()
print(f"Camera_process started {camera_process.pid}")
object_processor = TrackedObjectProcessor(CONFIG['cameras'], client, MQTT_TOPIC_PREFIX, tracked_objects_queue)
object_processor.start()
# create a flask app that encodes frames a mjpeg on demand
app = Flask(__name__)
@ -105,21 +127,23 @@ def main():
def stats():
stats = {
'coral': {
'fps': fps_tracker.eps(),
'inference_speed': prepped_queue_processor.avg_inference_speed,
'queue_length': prepped_frame_queue.qsize()
'fps': tflite_process.fps.value,
'inference_speed': tflite_process.avg_inference_speed.value
}
}
for name, camera in cameras.items():
stats[name] = camera.stats()
for name, camera_stats in camera_stats_values.items():
stats[name] = {
'fps': camera_stats['fps'].value,
'avg_wait': camera_stats['avg_wait'].value
}
return jsonify(stats)
@app.route('/<camera_name>/<label>/best.jpg')
def best(camera_name, label):
if camera_name in cameras:
best_frame = cameras[camera_name].get_best(label)
if camera_name in CONFIG['cameras']:
best_frame = object_processor.get_best(camera_name, label)
if best_frame is None:
best_frame = np.zeros((720,1280,3), np.uint8)
best_frame = cv2.cvtColor(best_frame, cv2.COLOR_RGB2BGR)
@ -132,7 +156,7 @@ def main():
@app.route('/<camera_name>')
def mjpeg_feed(camera_name):
if camera_name in cameras:
if camera_name in CONFIG['cameras']:
# return a multipart response
return Response(imagestream(camera_name),
mimetype='multipart/x-mixed-replace; boundary=frame')
@ -143,13 +167,16 @@ def main():
while True:
# max out at 1 FPS
time.sleep(1)
frame = cameras[camera_name].get_current_frame_with_objects()
frame = object_processor.current_frame_with_objects(camera_name)
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n\r\n')
app.run(host='0.0.0.0', port=WEB_PORT, debug=False)
camera.join()
for camera_process in camera_processes:
camera_process.join()
plasma_process.terminate()
if __name__ == '__main__':
main()

18
frigate/edgetpu.py
View File

@ -1,8 +1,11 @@
import os
import datetime
import multiprocessing as mp
import numpy as np
import SharedArray as sa
import tflite_runtime.interpreter as tflite
from tflite_runtime.interpreter import load_delegate
from frigate.util import EventsPerSecond
def load_labels(path, encoding='utf-8'):
"""Loads labels from file (with or without index numbers).
@ -59,6 +62,7 @@ class ObjectDetector():
class EdgeTPUProcess():
def __init__(self, model):
# TODO: see if we can use the plasma store with a queue and maintain the same speeds
try:
sa.delete("frame")
except:
@ -74,22 +78,32 @@ class EdgeTPUProcess():
self.detect_lock = mp.Lock()
self.detect_ready = mp.Event()
self.frame_ready = mp.Event()
self.fps = mp.Value('d', 0.0)
self.avg_inference_speed = mp.Value('d', 10.0)
def run_detector(model, detect_ready, frame_ready):
def run_detector(model, detect_ready, frame_ready, fps, avg_speed):
print(f"Starting detection process: {os.getpid()}")
object_detector = ObjectDetector(model)
input_frame = sa.attach("frame")
detections = sa.attach("detections")
fps_tracker = EventsPerSecond()
fps_tracker.start()
while True:
# wait until a frame is ready
frame_ready.wait()
start = datetime.datetime.now().timestamp()
# signal that the process is busy
frame_ready.clear()
detections[:] = object_detector.detect_raw(input_frame)
# signal that the process is ready to detect
detect_ready.set()
fps_tracker.update()
fps.value = fps_tracker.eps()
duration = datetime.datetime.now().timestamp()-start
avg_speed.value = (avg_speed.value*9 + duration)/10
self.detect_process = mp.Process(target=run_detector, args=(model, self.detect_ready, self.frame_ready))
self.detect_process = mp.Process(target=run_detector, args=(model, self.detect_ready, self.frame_ready, self.fps, self.avg_inference_speed))
self.detect_process.daemon = True
self.detect_process.start()

9
frigate/motion.py
View File

@ -3,14 +3,15 @@ import imutils
import numpy as np
class MotionDetector():
# TODO: add motion masking
def __init__(self, frame_shape, resize_factor=4):
def __init__(self, frame_shape, mask, resize_factor=4):
self.resize_factor = resize_factor
self.motion_frame_size = (int(frame_shape[0]/resize_factor), int(frame_shape[1]/resize_factor))
self.avg_frame = np.zeros(self.motion_frame_size, np.float)
self.avg_delta = np.zeros(self.motion_frame_size, np.float)
self.motion_frame_count = 0
self.frame_counter = 0
resized_mask = cv2.resize(mask, dsize=(self.motion_frame_size[1], self.motion_frame_size[0]), interpolation=cv2.INTER_LINEAR)
self.mask = np.where(resized_mask==[0])
def detect(self, frame):
motion_boxes = []
@ -21,6 +22,9 @@ class MotionDetector():
# convert to grayscale
gray = cv2.cvtColor(resized_frame, cv2.COLOR_BGR2GRAY)
# mask frame
gray[self.mask] = [255]
# it takes ~30 frames to establish a baseline
# dont bother looking for motion
if self.frame_counter < 30:
@ -58,7 +62,6 @@ class MotionDetector():
# if the contour is big enough, count it as motion
contour_area = cv2.contourArea(c)
if contour_area > 100:
# cv2.drawContours(resized_frame, [c], -1, (255,255,255), 2)
x, y, w, h = cv2.boundingRect(c)
motion_boxes.append((x*self.resize_factor, y*self.resize_factor, (x+w)*self.resize_factor, (y+h)*self.resize_factor))

54
frigate/mqtt.py
View File

@ -1,54 +0,0 @@
import json
import cv2
import threading
import prctl
from collections import Counter, defaultdict
import itertools
class MqttObjectPublisher(threading.Thread):
def __init__(self, client, topic_prefix, camera):
threading.Thread.__init__(self)
self.client = client
self.topic_prefix = topic_prefix
self.camera = camera
def run(self):
prctl.set_name(self.__class__.__name__)
current_object_status = defaultdict(lambda: 'OFF')
while True:
# wait until objects have been tracked
with self.camera.objects_tracked:
self.camera.objects_tracked.wait()
# count objects with more than 2 entries in history by type
obj_counter = Counter()
for obj in self.camera.object_tracker.tracked_objects.values():
if len(obj['history']) > 1:
obj_counter[obj['name']] += 1
# report on detected objects
for obj_name, count in obj_counter.items():
new_status = 'ON' if count > 0 else 'OFF'
if new_status != current_object_status[obj_name]:
current_object_status[obj_name] = new_status
self.client.publish(self.topic_prefix+'/'+obj_name, new_status, retain=False)
# send the snapshot over mqtt if we have it as well
if obj_name in self.camera.best_frames.best_frames:
best_frame = cv2.cvtColor(self.camera.best_frames.best_frames[obj_name], cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(self.topic_prefix+'/'+obj_name+'/snapshot', jpg_bytes, retain=True)
# expire any objects that are ON and no longer detected
expired_objects = [obj_name for obj_name, status in current_object_status.items() if status == 'ON' and not obj_name in obj_counter]
for obj_name in expired_objects:
current_object_status[obj_name] = 'OFF'
self.client.publish(self.topic_prefix+'/'+obj_name, 'OFF', retain=False)
# send updated snapshot snapshot over mqtt if we have it as well
if obj_name in self.camera.best_frames.best_frames:
best_frame = cv2.cvtColor(self.camera.best_frames.best_frames[obj_name], cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(self.topic_prefix+'/'+obj_name+'/snapshot', jpg_bytes, retain=True)

2
frigate/object_detection.py
View File

@ -3,7 +3,7 @@ import time
import cv2
import threading
import copy
import prctl
# import prctl
import numpy as np
from edgetpu.detection.engine import DetectionEngine

146
frigate/object_processing.py Normal file
View File

@ -0,0 +1,146 @@
import json
import hashlib
import datetime
import copy
import cv2
import threading
import numpy as np
from collections import Counter, defaultdict
import itertools
import pyarrow.plasma as plasma
import SharedArray as sa
import matplotlib.pyplot as plt
from frigate.util import draw_box_with_label, ReadLabelFile
PATH_TO_LABELS = '/lab/labelmap.txt'
LABELS = ReadLabelFile(PATH_TO_LABELS)
cmap = plt.cm.get_cmap('tab10', len(LABELS.keys()))
COLOR_MAP = {}
for key, val in LABELS.items():
COLOR_MAP[val] = tuple(int(round(255 * c)) for c in cmap(key)[:3])
class TrackedObjectProcessor(threading.Thread):
def __init__(self, config, client, topic_prefix, tracked_objects_queue):
threading.Thread.__init__(self)
self.config = config
self.client = client
self.topic_prefix = topic_prefix
self.tracked_objects_queue = tracked_objects_queue
self.plasma_client = plasma.connect("/tmp/plasma")
self.camera_data = defaultdict(lambda: {
'best_objects': {},
'object_status': defaultdict(lambda: defaultdict(lambda: 'OFF')),
'tracked_objects': {}
})
def get_best(self, camera, label):
if label in self.camera_data[camera]['best_objects']:
return self.camera_data[camera]['best_objects'][label]['frame']
else:
return None
def get_frame(self, config, camera, obj):
object_id_hash = hashlib.sha1(str.encode(f"{camera}{obj['frame_time']}"))
object_id_bytes = object_id_hash.digest()
object_id = plasma.ObjectID(object_id_bytes)
best_frame = self.plasma_client.get(object_id)
box = obj['box']
draw_box_with_label(best_frame, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}")
# print a timestamp
if config['snapshots']['show_timestamp']:
time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
return best_frame
def current_frame_with_objects(self, camera):
frame_time = self.camera_data[camera]['current_frame']
object_id_hash = hashlib.sha1(str.encode(f"{camera}{frame_time}"))
object_id_bytes = object_id_hash.digest()
object_id = plasma.ObjectID(object_id_bytes)
current_frame = self.plasma_client.get(object_id)
tracked_objects = copy.deepcopy(self.camera_data[camera]['tracked_objects'])
# draw the bounding boxes on the screen
for obj in tracked_objects.values():
thickness = 2
color = COLOR_MAP[obj['label']]
if obj['frame_time'] != frame_time:
thickness = 1
color = (255,0,0)
box = obj['box']
draw_box_with_label(current_frame, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}", thickness=thickness, color=color)
# # print fps
# cv2.putText(frame, str(self.fps.eps())+'FPS', (10, 60), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
# convert to BGR
frame = cv2.cvtColor(current_frame, cv2.COLOR_RGB2BGR)
# encode the image into a jpg
ret, jpg = cv2.imencode('.jpg', frame)
return jpg.tobytes()
def run(self):
while True:
camera, frame_time, tracked_objects = self.tracked_objects_queue.get()
config = self.config[camera]
best_objects = self.camera_data[camera]['best_objects']
current_object_status = self.camera_data[camera]['object_status']
self.camera_data[camera]['tracked_objects'] = tracked_objects
self.camera_data[camera]['current_frame'] = frame_time
###
# Maintain the highest scoring recent object and frame for each label
###
for obj in tracked_objects.values():
if obj['label'] in best_objects:
now = datetime.datetime.now().timestamp()
# if the object is a higher score than the current best score
# or the current object is more than 1 minute old, use the new object
if obj['score'] > best_objects[obj['label']]['score'] or (now - best_objects[obj['label']]['frame_time']) > 60:
obj['frame'] = self.get_frame(config, camera, obj)
best_objects[obj['label']] = obj
else:
obj['frame'] = self.get_frame(config, camera, obj)
best_objects[obj['label']] = obj
###
# Report over MQTT
###
# count objects with more than 2 entries in history by type
obj_counter = Counter()
for obj in tracked_objects.values():
if len(obj['history']) > 1:
obj_counter[obj['label']] += 1
# report on detected objects
for obj_name, count in obj_counter.items():
new_status = 'ON' if count > 0 else 'OFF'
if new_status != current_object_status[obj_name]:
current_object_status[obj_name] = new_status
self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}", new_status, retain=False)
# send the best snapshot over mqtt
best_frame = cv2.cvtColor(best_objects[obj_name]['frame'], cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}/snapshot", jpg_bytes, retain=True)
# expire any objects that are ON and no longer detected
expired_objects = [obj_name for obj_name, status in current_object_status.items() if status == 'ON' and not obj_name in obj_counter]
for obj_name in expired_objects:
current_object_status[obj_name] = 'OFF'
self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}", 'OFF', retain=False)
# send updated snapshot over mqtt
best_frame = cv2.cvtColor(best_objects[obj_name]['frame'], cv2.COLOR_RGB2BGR)
ret, jpg = cv2.imencode('.jpg', best_frame)
if ret:
jpg_bytes = jpg.tobytes()
self.client.publish(f"{self.topic_prefix}/{camera}/{obj_name}/snapshot", jpg_bytes, retain=True)

515
frigate/objects.py
View File

@ -2,277 +2,266 @@ import time
import datetime
import threading
import cv2
import prctl
# import prctl
import itertools
import copy
import numpy as np
import multiprocessing as mp
from collections import defaultdict
from scipy.spatial import distance as dist
from frigate.util import draw_box_with_label, LABELS, compute_intersection_rectangle, compute_intersection_over_union, calculate_region
from frigate.util import draw_box_with_label, LABELS, calculate_region
class ObjectCleaner(threading.Thread):
def __init__(self, camera):
threading.Thread.__init__(self)
self.camera = camera
# class ObjectCleaner(threading.Thread):
# def __init__(self, camera):
# threading.Thread.__init__(self)
# self.camera = camera
def run(self):
prctl.set_name("ObjectCleaner")
while True:
# def run(self):
# prctl.set_name("ObjectCleaner")
# while True:
# wait a bit before checking for expired frames
time.sleep(0.2)
# # wait a bit before checking for expired frames
# time.sleep(0.2)
for frame_time in list(self.camera.detected_objects.keys()).copy():
if not frame_time in self.camera.frame_cache:
del self.camera.detected_objects[frame_time]
# for frame_time in list(self.camera.detected_objects.keys()).copy():
# if not frame_time in self.camera.frame_cache:
# del self.camera.detected_objects[frame_time]
objects_deregistered = False
with self.camera.object_tracker.tracked_objects_lock:
now = datetime.datetime.now().timestamp()
for id, obj in list(self.camera.object_tracker.tracked_objects.items()):
# if the object is more than 10 seconds old
# and not in the most recent frame, deregister
if (now - obj['frame_time']) > 10 and self.camera.object_tracker.most_recent_frame_time > obj['frame_time']:
self.camera.object_tracker.deregister(id)
objects_deregistered = True
# objects_deregistered = False
# with self.camera.object_tracker.tracked_objects_lock:
# now = datetime.datetime.now().timestamp()
# for id, obj in list(self.camera.object_tracker.tracked_objects.items()):
# # if the object is more than 10 seconds old
# # and not in the most recent frame, deregister
# if (now - obj['frame_time']) > 10 and self.camera.object_tracker.most_recent_frame_time > obj['frame_time']:
# self.camera.object_tracker.deregister(id)
# objects_deregistered = True
if objects_deregistered:
with self.camera.objects_tracked:
self.camera.objects_tracked.notify_all()
# if objects_deregistered:
# with self.camera.objects_tracked:
# self.camera.objects_tracked.notify_all()
class DetectedObjectsProcessor(threading.Thread):
def __init__(self, camera):
threading.Thread.__init__(self)
self.camera = camera
# class DetectedObjectsProcessor(threading.Thread):
# def __init__(self, camera):
# threading.Thread.__init__(self)
# self.camera = camera
def run(self):
prctl.set_name(self.__class__.__name__)
while True:
frame = self.camera.detected_objects_queue.get()
# def run(self):
# prctl.set_name(self.__class__.__name__)
# while True:
# frame = self.camera.detected_objects_queue.get()
objects = frame['detected_objects']
# objects = frame['detected_objects']
for raw_obj in objects:
name = str(LABELS[raw_obj.label_id])
# for raw_obj in objects:
# name = str(LABELS[raw_obj.label_id])
if not name in self.camera.objects_to_track:
continue
# if not name in self.camera.objects_to_track:
# continue
obj = {
'name': name,
'score': float(raw_obj.score),
'box': {
'xmin': int((raw_obj.bounding_box[0][0] * frame['size']) + frame['x_offset']),
'ymin': int((raw_obj.bounding_box[0][1] * frame['size']) + frame['y_offset']),
'xmax': int((raw_obj.bounding_box[1][0] * frame['size']) + frame['x_offset']),
'ymax': int((raw_obj.bounding_box[1][1] * frame['size']) + frame['y_offset'])
},
'region': {
'xmin': frame['x_offset'],
'ymin': frame['y_offset'],
'xmax': frame['x_offset']+frame['size'],
'ymax': frame['y_offset']+frame['size']
},
'frame_time': frame['frame_time'],
'region_id': frame['region_id']
}
# obj = {
# 'name': name,
# 'score': float(raw_obj.score),
# 'box': {
# 'xmin': int((raw_obj.bounding_box[0][0] * frame['size']) + frame['x_offset']),
# 'ymin': int((raw_obj.bounding_box[0][1] * frame['size']) + frame['y_offset']),
# 'xmax': int((raw_obj.bounding_box[1][0] * frame['size']) + frame['x_offset']),
# 'ymax': int((raw_obj.bounding_box[1][1] * frame['size']) + frame['y_offset'])
# },
# 'region': {
# 'xmin': frame['x_offset'],
# 'ymin': frame['y_offset'],
# 'xmax': frame['x_offset']+frame['size'],
# 'ymax': frame['y_offset']+frame['size']
# },
# 'frame_time': frame['frame_time'],
# 'region_id': frame['region_id']
# }
# if the object is within 5 pixels of the region border, and the region is not on the edge
# consider the object to be clipped
obj['clipped'] = False
if ((obj['region']['xmin'] > 5 and obj['box']['xmin']-obj['region']['xmin'] <= 5) or
(obj['region']['ymin'] > 5 and obj['box']['ymin']-obj['region']['ymin'] <= 5) or
(self.camera.frame_shape[1]-obj['region']['xmax'] > 5 and obj['region']['xmax']-obj['box']['xmax'] <= 5) or
(self.camera.frame_shape[0]-obj['region']['ymax'] > 5 and obj['region']['ymax']-obj['box']['ymax'] <= 5)):
obj['clipped'] = True
# # if the object is within 5 pixels of the region border, and the region is not on the edge
# # consider the object to be clipped
# obj['clipped'] = False
# if ((obj['region']['xmin'] > 5 and obj['box']['xmin']-obj['region']['xmin'] <= 5) or
# (obj['region']['ymin'] > 5 and obj['box']['ymin']-obj['region']['ymin'] <= 5) or
# (self.camera.frame_shape[1]-obj['region']['xmax'] > 5 and obj['region']['xmax']-obj['box']['xmax'] <= 5) or
# (self.camera.frame_shape[0]-obj['region']['ymax'] > 5 and obj['region']['ymax']-obj['box']['ymax'] <= 5)):
# obj['clipped'] = True
# Compute the area
# TODO: +1 right?
obj['area'] = (obj['box']['xmax']-obj['box']['xmin'])*(obj['box']['ymax']-obj['box']['ymin'])
# # Compute the area
# # TODO: +1 right?
# obj['area'] = (obj['box']['xmax']-obj['box']['xmin'])*(obj['box']['ymax']-obj['box']['ymin'])
self.camera.detected_objects[frame['frame_time']].append(obj)
# self.camera.detected_objects[frame['frame_time']].append(obj)
# TODO: use in_process and processed counts instead to avoid lock
with self.camera.regions_in_process_lock:
if frame['frame_time'] in self.camera.regions_in_process:
self.camera.regions_in_process[frame['frame_time']] -= 1
# print(f"{frame['frame_time']} remaining regions {self.camera.regions_in_process[frame['frame_time']]}")
# # TODO: use in_process and processed counts instead to avoid lock
# with self.camera.regions_in_process_lock:
# if frame['frame_time'] in self.camera.regions_in_process:
# self.camera.regions_in_process[frame['frame_time']] -= 1
# # print(f"{frame['frame_time']} remaining regions {self.camera.regions_in_process[frame['frame_time']]}")
if self.camera.regions_in_process[frame['frame_time']] == 0:
del self.camera.regions_in_process[frame['frame_time']]
# print(f"{frame['frame_time']} no remaining regions")
self.camera.finished_frame_queue.put(frame['frame_time'])
else:
self.camera.finished_frame_queue.put(frame['frame_time'])
# if self.camera.regions_in_process[frame['frame_time']] == 0:
# del self.camera.regions_in_process[frame['frame_time']]
# # print(f"{frame['frame_time']} no remaining regions")
# self.camera.finished_frame_queue.put(frame['frame_time'])
# else:
# self.camera.finished_frame_queue.put(frame['frame_time'])
# Thread that checks finished frames for clipped objects and sends back
# for processing if needed
# TODO: evaluate whether or not i really need separate threads/queues for each step
# given that only 1 thread will really be able to run at a time. you need a
# separate process to actually do things in parallel for when you are CPU bound.
# threads are good when you are waiting and could be processing while you wait
class RegionRefiner(threading.Thread):
def __init__(self, camera):
threading.Thread.__init__(self)
self.camera = camera
# # Thread that checks finished frames for clipped objects and sends back
# # for processing if needed
# # TODO: evaluate whether or not i really need separate threads/queues for each step
# # given that only 1 thread will really be able to run at a time. you need a
# # separate process to actually do things in parallel for when you are CPU bound.
# # threads are good when you are waiting and could be processing while you wait
# class RegionRefiner(threading.Thread):
# def __init__(self, camera):
# threading.Thread.__init__(self)
# self.camera = camera
def run(self):
prctl.set_name(self.__class__.__name__)
while True:
frame_time = self.camera.finished_frame_queue.get()
# def run(self):
# prctl.set_name(self.__class__.__name__)
# while True:
# frame_time = self.camera.finished_frame_queue.get()
detected_objects = self.camera.detected_objects[frame_time].copy()
# print(f"{frame_time} finished")
# detected_objects = self.camera.detected_objects[frame_time].copy()
# # print(f"{frame_time} finished")
# group by name
detected_object_groups = defaultdict(lambda: [])
for obj in detected_objects:
detected_object_groups[obj['name']].append(obj)
# # group by name
# detected_object_groups = defaultdict(lambda: [])
# for obj in detected_objects:
# detected_object_groups[obj['name']].append(obj)
look_again = False
selected_objects = []
for group in detected_object_groups.values():
# look_again = False
# selected_objects = []
# for group in detected_object_groups.values():
# apply non-maxima suppression to suppress weak, overlapping bounding boxes
boxes = [(o['box']['xmin'], o['box']['ymin'], o['box']['xmax']-o['box']['xmin'], o['box']['ymax']-o['box']['ymin'])
for o in group]
confidences = [o['score'] for o in group]
idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
# # apply non-maxima suppression to suppress weak, overlapping bounding boxes
# boxes = [(o['box']['xmin'], o['box']['ymin'], o['box']['xmax']-o['box']['xmin'], o['box']['ymax']-o['box']['ymin'])
# for o in group]
# confidences = [o['score'] for o in group]
# idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
for index in idxs:
obj = group[index[0]]
selected_objects.append(obj)
if obj['clipped']:
box = obj['box']
# calculate a new region that will hopefully get the entire object
(size, x_offset, y_offset) = calculate_region(self.camera.frame_shape,
box['xmin'], box['ymin'],
box['xmax'], box['ymax'])
# print(f"{frame_time} new region: {size} {x_offset} {y_offset}")
# for index in idxs:
# obj = group[index[0]]
# selected_objects.append(obj)
# if obj['clipped']:
# box = obj['box']
# # calculate a new region that will hopefully get the entire object
# (size, x_offset, y_offset) = calculate_region(self.camera.frame_shape,
# box['xmin'], box['ymin'],
# box['xmax'], box['ymax'])
# # print(f"{frame_time} new region: {size} {x_offset} {y_offset}")
with self.camera.regions_in_process_lock:
if not frame_time in self.camera.regions_in_process:
self.camera.regions_in_process[frame_time] = 1
else:
self.camera.regions_in_process[frame_time] += 1
# with self.camera.regions_in_process_lock:
# if not frame_time in self.camera.regions_in_process:
# self.camera.regions_in_process[frame_time] = 1
# else:
# self.camera.regions_in_process[frame_time] += 1
# add it to the queue
self.camera.resize_queue.put({
'camera_name': self.camera.name,
'frame_time': frame_time,
'region_id': -1,
'size': size,
'x_offset': x_offset,
'y_offset': y_offset
})
self.camera.dynamic_region_fps.update()
look_again = True
# # add it to the queue
# self.camera.resize_queue.put({
# 'camera_name': self.camera.name,
# 'frame_time': frame_time,
# 'region_id': -1,
# 'size': size,
# 'x_offset': x_offset,
# 'y_offset': y_offset
# })
# self.camera.dynamic_region_fps.update()
# look_again = True
# if we are looking again, then this frame is not ready for processing
if look_again:
# remove the clipped objects
self.camera.detected_objects[frame_time] = [o for o in selected_objects if not o['clipped']]
continue
# # if we are looking again, then this frame is not ready for processing
# if look_again:
# # remove the clipped objects
# self.camera.detected_objects[frame_time] = [o for o in selected_objects if not o['clipped']]
# continue
# filter objects based on camera settings
selected_objects = [o for o in selected_objects if not self.filtered(o)]
# # filter objects based on camera settings
# selected_objects = [o for o in selected_objects if not self.filtered(o)]
self.camera.detected_objects[frame_time] = selected_objects
# self.camera.detected_objects[frame_time] = selected_objects
# print(f"{frame_time} is actually finished")
# # print(f"{frame_time} is actually finished")
# keep adding frames to the refined queue as long as they are finished
with self.camera.regions_in_process_lock:
while self.camera.frame_queue.qsize() > 0 and self.camera.frame_queue.queue[0] not in self.camera.regions_in_process:
self.camera.last_processed_frame = self.camera.frame_queue.get()
self.camera.refined_frame_queue.put(self.camera.last_processed_frame)
# # keep adding frames to the refined queue as long as they are finished
# with self.camera.regions_in_process_lock:
# while self.camera.frame_queue.qsize() > 0 and self.camera.frame_queue.queue[0] not in self.camera.regions_in_process:
# self.camera.last_processed_frame = self.camera.frame_queue.get()
# self.camera.refined_frame_queue.put(self.camera.last_processed_frame)
def filtered(self, obj):
object_name = obj['name']
# def filtered(self, obj):
# object_name = obj['name']
if object_name in self.camera.object_filters:
obj_settings = self.camera.object_filters[object_name]
# if object_name in self.camera.object_filters:
# obj_settings = self.camera.object_filters[object_name]
# if the min area is larger than the
# detected object, don't add it to detected objects
if obj_settings.get('min_area',-1) > obj['area']:
return True
# # if the min area is larger than the
# # detected object, don't add it to detected objects
# if obj_settings.get('min_area',-1) > obj['area']:
# return True
# if the detected object is larger than the
# max area, don't add it to detected objects
if obj_settings.get('max_area', self.camera.frame_shape[0]*self.camera.frame_shape[1]) < obj['area']:
return True
# # if the detected object is larger than the
# # max area, don't add it to detected objects
# if obj_settings.get('max_area', self.camera.frame_shape[0]*self.camera.frame_shape[1]) < obj['area']:
# return True
# if the score is lower than the threshold, skip
if obj_settings.get('threshold', 0) > obj['score']:
return True
# # if the score is lower than the threshold, skip
# if obj_settings.get('threshold', 0) > obj['score']:
# return True
# compute the coordinates of the object and make sure
# the location isnt outside the bounds of the image (can happen from rounding)
y_location = min(int(obj['box']['ymax']), len(self.camera.mask)-1)
x_location = min(int((obj['box']['xmax']-obj['box']['xmin'])/2.0)+obj['box']['xmin'], len(self.camera.mask[0])-1)
# # compute the coordinates of the object and make sure
# # the location isnt outside the bounds of the image (can happen from rounding)
# y_location = min(int(obj['box']['ymax']), len(self.camera.mask)-1)
# x_location = min(int((obj['box']['xmax']-obj['box']['xmin'])/2.0)+obj['box']['xmin'], len(self.camera.mask[0])-1)
# if the object is in a masked location, don't add it to detected objects
if self.camera.mask[y_location][x_location] == [0]:
return True
# # if the object is in a masked location, don't add it to detected objects
# if self.camera.mask[y_location][x_location] == [0]:
# return True
return False
# return False
def has_overlap(self, new_obj, obj, overlap=.7):
# compute intersection rectangle with existing object and new objects region
existing_obj_current_region = compute_intersection_rectangle(obj['box'], new_obj['region'])
# def has_overlap(self, new_obj, obj, overlap=.7):
# # compute intersection rectangle with existing object and new objects region
# existing_obj_current_region = compute_intersection_rectangle(obj['box'], new_obj['region'])
# compute intersection rectangle with new object and existing objects region
new_obj_existing_region = compute_intersection_rectangle(new_obj['box'], obj['region'])
# # compute intersection rectangle with new object and existing objects region
# new_obj_existing_region = compute_intersection_rectangle(new_obj['box'], obj['region'])
# compute iou for the two intersection rectangles that were just computed
iou = compute_intersection_over_union(existing_obj_current_region, new_obj_existing_region)
# # compute iou for the two intersection rectangles that were just computed
# iou = compute_intersection_over_union(existing_obj_current_region, new_obj_existing_region)
# if intersection is greater than overlap
if iou > overlap:
return True
else:
return False
# # if intersection is greater than overlap
# if iou > overlap:
# return True
# else:
# return False
def find_group(self, new_obj, groups):
for index, group in enumerate(groups):
for obj in group:
if self.has_overlap(new_obj, obj):
return index
return None
# def find_group(self, new_obj, groups):
# for index, group in enumerate(groups):
# for obj in group:
# if self.has_overlap(new_obj, obj):
# return index
# return None
class ObjectTracker(threading.Thread):
def __init__(self, camera, max_disappeared):
threading.Thread.__init__(self)
self.camera = camera
class ObjectTracker():
def __init__(self, max_disappeared):
self.tracked_objects = {}
self.tracked_objects_lock = mp.Lock()
self.most_recent_frame_time = None
def run(self):
prctl.set_name(self.__class__.__name__)
while True:
frame_time = self.camera.refined_frame_queue.get()
with self.tracked_objects_lock:
self.match_and_update(self.camera.detected_objects[frame_time])
self.most_recent_frame_time = frame_time
self.camera.frame_output_queue.put((frame_time, copy.deepcopy(self.tracked_objects)))
if len(self.tracked_objects) > 0:
with self.camera.objects_tracked:
self.camera.objects_tracked.notify_all()
self.disappeared = {}
self.max_disappeared = max_disappeared
def register(self, index, obj):
id = "{}-{}".format(str(obj['frame_time']), index)
id = f"{obj['frame_time']}-{index}"
obj['id'] = id
obj['top_score'] = obj['score']
self.add_history(obj)
self.tracked_objects[id] = obj
self.disappeared[id] = 0
def deregister(self, id):
del self.tracked_objects[id]
del self.disappeared[id]
def update(self, id, new_obj):
self.disappeared[id] = 0
self.tracked_objects[id].update(new_obj)
self.add_history(self.tracked_objects[id])
if self.tracked_objects[id]['score'] > self.tracked_objects[id]['top_score']:
@ -291,25 +280,37 @@ class ObjectTracker(threading.Thread):
else:
obj['history'] = [entry]
def match_and_update(self, new_objects):
def match_and_update(self, frame_time, new_objects):
if len(new_objects) == 0:
for id in list(self.tracked_objects.keys()):
if self.disappeared[id] >= self.max_disappeared:
self.deregister(id)
else:
self.disappeared[id] += 1
return
# group by name
new_object_groups = defaultdict(lambda: [])
for obj in new_objects:
new_object_groups[obj['name']].append(obj)
new_object_groups[obj[0]].append({
'label': obj[0],
'score': obj[1],
'box': obj[2],
'area': obj[3],
'region': obj[4],
'frame_time': frame_time
})
# track objects for each label type
for label, group in new_object_groups.items():
current_objects = [o for o in self.tracked_objects.values() if o['name'] == label]
current_objects = [o for o in self.tracked_objects.values() if o['label'] == label]
current_ids = [o['id'] for o in current_objects]
current_centroids = np.array([o['centroid'] for o in current_objects])
# compute centroids of new objects
for obj in group:
centroid_x = int((obj['box']['xmin']+obj['box']['xmax']) / 2.0)
centroid_y = int((obj['box']['ymin']+obj['box']['ymax']) / 2.0)
centroid_x = int((obj['box'][0]+obj['box'][2]) / 2.0)
centroid_y = int((obj['box'][1]+obj['box'][3]) / 2.0)
obj['centroid'] = (centroid_x, centroid_y)
if len(current_objects) == 0:
@ -363,56 +364,66 @@ class ObjectTracker(threading.Thread):
usedCols.add(col)
# compute the column index we have NOT yet examined
unusedRows = set(range(0, D.shape[0])).difference(usedRows)
unusedCols = set(range(0, D.shape[1])).difference(usedCols)
# in the event that the number of object centroids is
# equal or greater than the number of input centroids
# we need to check and see if some of these objects have
# potentially disappeared
if D.shape[0] >= D.shape[1]:
for row in unusedRows:
id = current_ids[row]
if self.disappeared[id] >= self.max_disappeared:
self.deregister(id)
else:
self.disappeared[id] += 1
# if the number of input centroids is greater
# than the number of existing object centroids we need to
# register each new input centroid as a trackable object
# if D.shape[0] < D.shape[1]:
# TODO: rather than assuming these are new objects, we could
# look to see if any of the remaining boxes have a large amount
# of overlap...
for col in unusedCols:
self.register(col, group[col])
else:
for col in unusedCols:
self.register(col, group[col])
# Maintains the frame and object with the highest score
class BestFrames(threading.Thread):
def __init__(self, camera):
threading.Thread.__init__(self)
self.camera = camera
self.best_objects = {}
self.best_frames = {}
# class BestFrames(threading.Thread):
# def __init__(self, camera):
# threading.Thread.__init__(self)
# self.camera = camera
# self.best_objects = {}
# self.best_frames = {}
def run(self):
prctl.set_name(self.__class__.__name__)
while True:
# wait until objects have been tracked
with self.camera.objects_tracked:
self.camera.objects_tracked.wait()
# def run(self):
# prctl.set_name(self.__class__.__name__)
# while True:
# # wait until objects have been tracked
# with self.camera.objects_tracked:
# self.camera.objects_tracked.wait()
# make a copy of tracked objects
tracked_objects = list(self.camera.object_tracker.tracked_objects.values())
# # make a copy of tracked objects
# tracked_objects = list(self.camera.object_tracker.tracked_objects.values())
for obj in tracked_objects:
if obj['name'] in self.best_objects:
now = datetime.datetime.now().timestamp()
# if the object is a higher score than the current best score
# or the current object is more than 1 minute old, use the new object
if obj['score'] > self.best_objects[obj['name']]['score'] or (now - self.best_objects[obj['name']]['frame_time']) > 60:
self.best_objects[obj['name']] = copy.deepcopy(obj)
else:
self.best_objects[obj['name']] = copy.deepcopy(obj)
# for obj in tracked_objects:
# if obj['name'] in self.best_objects:
# now = datetime.datetime.now().timestamp()
# # if the object is a higher score than the current best score
# # or the current object is more than 1 minute old, use the new object
# if obj['score'] > self.best_objects[obj['name']]['score'] or (now - self.best_objects[obj['name']]['frame_time']) > 60:
# self.best_objects[obj['name']] = copy.deepcopy(obj)
# else:
# self.best_objects[obj['name']] = copy.deepcopy(obj)
for name, obj in self.best_objects.items():
if obj['frame_time'] in self.camera.frame_cache:
best_frame = self.camera.frame_cache[obj['frame_time']]
# for name, obj in self.best_objects.items():
# if obj['frame_time'] in self.camera.frame_cache:
# best_frame = self.camera.frame_cache[obj['frame_time']]
draw_box_with_label(best_frame, obj['box']['xmin'], obj['box']['ymin'],
obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']))
# draw_box_with_label(best_frame, obj['box']['xmin'], obj['box']['ymin'],
# obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']))
# print a timestamp
if self.camera.snapshot_config['show_timestamp']:
time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
# # print a timestamp
# if self.camera.snapshot_config['show_timestamp']:
# time_to_show = datetime.datetime.fromtimestamp(obj['frame_time']).strftime("%m/%d/%Y %H:%M:%S")
# cv2.putText(best_frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
self.best_frames[name] = best_frame
# self.best_frames[name] = best_frame

160
frigate/util.py Normal file → Executable file
View File

@ -15,73 +15,11 @@ def ReadLabelFile(file_path):
ret[int(pair[0])] = pair[1].strip()
return ret
def calculate_region(frame_shape, xmin, ymin, xmax, ymax):
# size is larger than longest edge
size = int(max(xmax-xmin, ymax-ymin)*2)
# if the size is too big to fit in the frame
if size > min(frame_shape[0], frame_shape[1]):
size = min(frame_shape[0], frame_shape[1])
# x_offset is midpoint of bounding box minus half the size
x_offset = int((xmax-xmin)/2.0+xmin-size/2.0)
# if outside the image
if x_offset < 0:
x_offset = 0
elif x_offset > (frame_shape[1]-size):
x_offset = (frame_shape[1]-size)
# y_offset is midpoint of bounding box minus half the size
y_offset = int((ymax-ymin)/2.0+ymin-size/2.0)
# if outside the image
if y_offset < 0:
y_offset = 0
elif y_offset > (frame_shape[0]-size):
y_offset = (frame_shape[0]-size)
return (size, x_offset, y_offset)
def compute_intersection_rectangle(box_a, box_b):
return {
'xmin': max(box_a['xmin'], box_b['xmin']),
'ymin': max(box_a['ymin'], box_b['ymin']),
'xmax': min(box_a['xmax'], box_b['xmax']),
'ymax': min(box_a['ymax'], box_b['ymax'])
}
def compute_intersection_over_union(box_a, box_b):
# determine the (x, y)-coordinates of the intersection rectangle
intersect = compute_intersection_rectangle(box_a, box_b)
# compute the area of intersection rectangle
inter_area = max(0, intersect['xmax'] - intersect['xmin'] + 1) * max(0, intersect['ymax'] - intersect['ymin'] + 1)
if inter_area == 0:
return 0.0
# compute the area of both the prediction and ground-truth
# rectangles
box_a_area = (box_a['xmax'] - box_a['xmin'] + 1) * (box_a['ymax'] - box_a['ymin'] + 1)
box_b_area = (box_b['xmax'] - box_b['xmin'] + 1) * (box_b['ymax'] - box_b['ymin'] + 1)
# compute the intersection over union by taking the intersection
# area and dividing it by the sum of prediction + ground-truth
# areas - the interesection area
iou = inter_area / float(box_a_area + box_b_area - inter_area)
# return the intersection over union value
return iou
# convert shared memory array into numpy array
def tonumpyarray(mp_arr):
return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8)
def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label, info, thickness=2, color=None, position='ul'):
if color is None:
color = COLOR_MAP[label]
color = (0,0,255)
display_text = "{}: {}".format(label, info)
cv2.rectangle(frame, (x_min, y_min),
(x_max, y_max),
color, thickness)
cv2.rectangle(frame, (x_min, y_min), (x_max, y_max), color, thickness)
font_scale = 0.5
font = cv2.FONT_HERSHEY_SIMPLEX
# get the width and height of the text box
@ -107,37 +45,81 @@ def draw_box_with_label(frame, x_min, y_min, x_max, y_max, label, info, thicknes
cv2.rectangle(frame, textbox_coords[0], textbox_coords[1], color, cv2.FILLED)
cv2.putText(frame, display_text, (text_offset_x, text_offset_y + line_height - 3), font, fontScale=font_scale, color=(0, 0, 0), thickness=2)
# Path to frozen detection graph. This is the actual model that is used for the object detection.
PATH_TO_CKPT = '/frozen_inference_graph.pb'
# List of the strings that is used to add correct label for each box.
PATH_TO_LABELS = '/label_map.pbtext'
def calculate_region(frame_shape, xmin, ymin, xmax, ymax, multiplier=2):
# size is larger than longest edge
size = int(max(xmax-xmin, ymax-ymin)*multiplier)
# if the size is too big to fit in the frame
if size > min(frame_shape[0], frame_shape[1]):
size = min(frame_shape[0], frame_shape[1])
LABELS = ReadLabelFile(PATH_TO_LABELS)
cmap = plt.cm.get_cmap('tab10', len(LABELS.keys()))
# x_offset is midpoint of bounding box minus half the size
x_offset = int((xmax-xmin)/2.0+xmin-size/2.0)
# if outside the image
if x_offset < 0:
x_offset = 0
elif x_offset > (frame_shape[1]-size):
x_offset = (frame_shape[1]-size)
COLOR_MAP = {}
for key, val in LABELS.items():
COLOR_MAP[val] = tuple(int(round(255 * c)) for c in cmap(key)[:3])
# y_offset is midpoint of bounding box minus half the size
y_offset = int((ymax-ymin)/2.0+ymin-size/2.0)
# if outside the image
if y_offset < 0:
y_offset = 0
elif y_offset > (frame_shape[0]-size):
y_offset = (frame_shape[0]-size)
class QueueMerger():
def __init__(self, from_queues, to_queue):
self.from_queues = from_queues
self.to_queue = to_queue
self.merge_threads = []
return (x_offset, y_offset, x_offset+size, y_offset+size)
def start(self):
for from_q in self.from_queues:
self.merge_threads.append(QueueTransfer(from_q,self.to_queue))
def intersection(box_a, box_b):
return (
max(box_a[0], box_b[0]),
max(box_a[1], box_b[1]),
min(box_a[2], box_b[2]),
min(box_a[3], box_b[3])
)
class QueueTransfer(threading.Thread):
def __init__(self, from_queue, to_queue):
threading.Thread.__init__(self)
self.from_queue = from_queue
self.to_queue = to_queue
def area(box):
return (box[2]-box[0] + 1)*(box[3]-box[1] + 1)
def run(self):
while True:
self.to_queue.put(self.from_queue.get())
def intersection_over_union(box_a, box_b):
# determine the (x, y)-coordinates of the intersection rectangle
intersect = intersection(box_a, box_b)
# compute the area of intersection rectangle
inter_area = max(0, intersect[2] - intersect[0] + 1) * max(0, intersect[3] - intersect[1] + 1)
if inter_area == 0:
return 0.0
# compute the area of both the prediction and ground-truth
# rectangles
box_a_area = (box_a[2] - box_a[0] + 1) * (box_a[3] - box_a[1] + 1)
box_b_area = (box_b[2] - box_b[0] + 1) * (box_b[3] - box_b[1] + 1)
# compute the intersection over union by taking the intersection
# area and dividing it by the sum of prediction + ground-truth
# areas - the interesection area
iou = inter_area / float(box_a_area + box_b_area - inter_area)
# return the intersection over union value
return iou
def clipped(obj, frame_shape):
# if the object is within 5 pixels of the region border, and the region is not on the edge
# consider the object to be clipped
box = obj[2]
region = obj[4]
if ((region[0] > 5 and box[0]-region[0] <= 5) or
(region[1] > 5 and box[1]-region[1] <= 5) or
(frame_shape[1]-region[2] > 5 and region[2]-box[2] <= 5) or
(frame_shape[0]-region[3] > 5 and region[3]-box[3] <= 5)):
return True
else:
return False
# convert shared memory array into numpy array
def tonumpyarray(mp_arr):
return np.frombuffer(mp_arr.get_obj(), dtype=np.uint8)
class EventsPerSecond:
def __init__(self, max_events=1000):

754
frigate/video.py Normal file → Executable file
View File

@ -8,40 +8,47 @@ import ctypes
import multiprocessing as mp
import subprocess as sp
import numpy as np
import prctl
import hashlib
import pyarrow.plasma as plasma
import SharedArray as sa
# import prctl
import copy
import itertools
import json
from collections import defaultdict
from frigate.util import tonumpyarray, LABELS, draw_box_with_label, calculate_region, EventsPerSecond
from frigate.object_detection import RegionPrepper, RegionRequester
from frigate.objects import ObjectCleaner, BestFrames, DetectedObjectsProcessor, RegionRefiner, ObjectTracker
from frigate.mqtt import MqttObjectPublisher
from frigate.util import tonumpyarray, LABELS, draw_box_with_label, area, calculate_region, clipped, intersection_over_union, intersection, EventsPerSecond
# from frigate.object_detection import RegionPrepper, RegionRequester
from frigate.objects import ObjectTracker
# from frigate.mqtt import MqttObjectPublisher
from frigate.edgetpu import RemoteObjectDetector
from frigate.motion import MotionDetector
# Stores 2 seconds worth of frames so they can be used for other threads
class FrameTracker(threading.Thread):
def __init__(self, frame_time, frame_ready, frame_lock, recent_frames):
threading.Thread.__init__(self)
self.frame_time = frame_time
self.frame_ready = frame_ready
self.frame_lock = frame_lock
self.recent_frames = recent_frames
# TODO: we do actually know when these frames are no longer needed
# class FrameTracker(threading.Thread):
# def __init__(self, frame_time, frame_ready, frame_lock, recent_frames):
# threading.Thread.__init__(self)
# self.frame_time = frame_time
# self.frame_ready = frame_ready
# self.frame_lock = frame_lock
# self.recent_frames = recent_frames
def run(self):
prctl.set_name(self.__class__.__name__)
while True:
# wait for a frame
with self.frame_ready:
self.frame_ready.wait()
# def run(self):
# prctl.set_name(self.__class__.__name__)
# while True:
# # wait for a frame
# with self.frame_ready:
# self.frame_ready.wait()
# delete any old frames
stored_frame_times = list(self.recent_frames.keys())
stored_frame_times.sort(reverse=True)
if len(stored_frame_times) > 100:
frames_to_delete = stored_frame_times[50:]
for k in frames_to_delete:
del self.recent_frames[k]
# # delete any old frames
# stored_frame_times = list(self.recent_frames.keys())
# stored_frame_times.sort(reverse=True)
# if len(stored_frame_times) > 100:
# frames_to_delete = stored_frame_times[50:]
# for k in frames_to_delete:
# del self.recent_frames[k]
# TODO: add back opencv fallback
def get_frame_shape(source):
ffprobe_cmd = " ".join([
'ffprobe',
@ -76,6 +83,7 @@ def get_ffmpeg_input(ffmpeg_input):
frigate_vars = {k: v for k, v in os.environ.items() if k.startswith('FRIGATE_')}
return ffmpeg_input.format(**frigate_vars)
<<<<<<< HEAD
class CameraWatchdog(threading.Thread):
def __init__(self, camera):
threading.Thread.__init__(self)
@ -294,112 +302,648 @@ class Camera:
self.capture_thread.join()
self.ffmpeg_process = None
self.capture_thread = None
=======
# class CameraWatchdog(threading.Thread):
# def __init__(self, camera):
# threading.Thread.__init__(self)
# self.camera = camera
# create the process to capture frames from the input stream and store in a shared array
print("Creating a new ffmpeg process...")
self.start_ffmpeg()
# def run(self):
# prctl.set_name(self.__class__.__name__)
# while True:
# # wait a bit before checking
# time.sleep(10)
print("Creating a new capture thread...")
self.capture_thread = CameraCapture(self)
print("Starting a new capture thread...")
self.capture_thread.start()
self.fps.start()
self.skipped_region_tracker.start()
# if self.camera.frame_time.value != 0.0 and (datetime.datetime.now().timestamp() - self.camera.frame_time.value) > self.camera.watchdog_timeout:
# print(self.camera.name + ": last frame is more than 5 minutes old, restarting camera capture...")
# self.camera.start_or_restart_capture()
# time.sleep(5)
def start_ffmpeg(self):
ffmpeg_cmd = (['ffmpeg'] +
self.ffmpeg_global_args +
self.ffmpeg_hwaccel_args +
self.ffmpeg_input_args +
['-i', self.ffmpeg_input] +
self.ffmpeg_output_args +
# # Thread to read the stdout of the ffmpeg process and update the current frame
# class CameraCapture(threading.Thread):
# def __init__(self, camera):
# threading.Thread.__init__(self)
# self.camera = camera
# def run(self):
# prctl.set_name(self.__class__.__name__)
# frame_num = 0
# while True:
# if self.camera.ffmpeg_process.poll() != None:
# print(self.camera.name + ": ffmpeg process is not running. exiting capture thread...")
# break
# raw_image = self.camera.ffmpeg_process.stdout.read(self.camera.frame_size)
# if len(raw_image) == 0:
# print(self.camera.name + ": ffmpeg didnt return a frame. something is wrong. exiting capture thread...")
# break
# frame_num += 1
# if (frame_num % self.camera.take_frame) != 0:
# continue
# with self.camera.frame_lock:
# # TODO: use frame_queue instead
# self.camera.frame_time.value = datetime.datetime.now().timestamp()
# self.camera.frame_cache[self.camera.frame_time.value] = (
# np
# .frombuffer(raw_image, np.uint8)
# .reshape(self.camera.frame_shape)
# )
# self.camera.frame_queue.put(self.camera.frame_time.value)
# # Notify with the condition that a new frame is ready
# with self.camera.frame_ready:
# self.camera.frame_ready.notify_all()
# self.camera.fps.update()
# class VideoWriter(threading.Thread):
# def __init__(self, camera):
# threading.Thread.__init__(self)
# self.camera = camera
# def run(self):
# prctl.set_name(self.__class__.__name__)
# while True:
# (frame_time, tracked_objects) = self.camera.frame_output_queue.get()
# # if len(tracked_objects) == 0:
# # continue
# # f = open(f"/debug/output/{self.camera.name}-{str(format(frame_time, '.8f'))}.jpg", 'wb')
# # f.write(self.camera.frame_with_objects(frame_time, tracked_objects))
# # f.close()
# class Camera:
# def __init__(self, name, ffmpeg_config, global_objects_config, config, tflite_process, mqtt_client, mqtt_prefix):
# self.name = name
# self.config = config
# self.detected_objects = defaultdict(lambda: [])
# self.frame_cache = {}
# self.last_processed_frame = None
# # queue for re-assembling frames in order
# self.frame_queue = queue.Queue()
# # track how many regions have been requested for a frame so we know when a frame is complete
# self.regions_in_process = {}
# # Lock to control access
# self.regions_in_process_lock = mp.Lock()
# self.finished_frame_queue = queue.Queue()
# self.refined_frame_queue = queue.Queue()
# self.frame_output_queue = queue.Queue()
# self.ffmpeg = config.get('ffmpeg', {})
# self.ffmpeg_input = get_ffmpeg_input(self.ffmpeg['input'])
# self.ffmpeg_global_args = self.ffmpeg.get('global_args', ffmpeg_config['global_args'])
# self.ffmpeg_hwaccel_args = self.ffmpeg.get('hwaccel_args', ffmpeg_config['hwaccel_args'])
# self.ffmpeg_input_args = self.ffmpeg.get('input_args', ffmpeg_config['input_args'])
# self.ffmpeg_output_args = self.ffmpeg.get('output_args', ffmpeg_config['output_args'])
# camera_objects_config = config.get('objects', {})
# self.take_frame = self.config.get('take_frame', 1)
# self.watchdog_timeout = self.config.get('watchdog_timeout', 300)
# self.snapshot_config = {
# 'show_timestamp': self.config.get('snapshots', {}).get('show_timestamp', True)
# }
# self.regions = self.config['regions']
# self.frame_shape = get_frame_shape(self.ffmpeg_input)
# self.frame_size = self.frame_shape[0] * self.frame_shape[1] * self.frame_shape[2]
# self.mqtt_client = mqtt_client
# self.mqtt_topic_prefix = '{}/{}'.format(mqtt_prefix, self.name)
# # create shared value for storing the frame_time
# self.frame_time = mp.Value('d', 0.0)
# # Lock to control access to the frame
# self.frame_lock = mp.Lock()
# # Condition for notifying that a new frame is ready
# self.frame_ready = mp.Condition()
# # Condition for notifying that objects were tracked
# self.objects_tracked = mp.Condition()
# # Queue for prepped frames, max size set to (number of regions * 5)
# self.resize_queue = queue.Queue()
# # Queue for raw detected objects
# self.detected_objects_queue = queue.Queue()
# self.detected_objects_processor = DetectedObjectsProcessor(self)
# self.detected_objects_processor.start()
# # initialize the frame cache
# self.cached_frame_with_objects = {
# 'frame_bytes': [],
# 'frame_time': 0
# }
# self.ffmpeg_process = None
# self.capture_thread = None
# self.fps = EventsPerSecond()
# self.skipped_region_tracker = EventsPerSecond()
# # combine tracked objects lists
# self.objects_to_track = set().union(global_objects_config.get('track', ['person', 'car', 'truck']), camera_objects_config.get('track', []))
# # merge object filters
# global_object_filters = global_objects_config.get('filters', {})
# camera_object_filters = camera_objects_config.get('filters', {})
# objects_with_config = set().union(global_object_filters.keys(), camera_object_filters.keys())
# self.object_filters = {}
# for obj in objects_with_config:
# self.object_filters[obj] = {**global_object_filters.get(obj, {}), **camera_object_filters.get(obj, {})}
# # start a thread to track objects
# self.object_tracker = ObjectTracker(self, 10)
# self.object_tracker.start()
# # start a thread to write tracked frames to disk
# self.video_writer = VideoWriter(self)
# self.video_writer.start()
# # start a thread to queue resize requests for regions
# self.region_requester = RegionRequester(self)
# self.region_requester.start()
# # start a thread to cache recent frames for processing
# self.frame_tracker = FrameTracker(self.frame_time,
# self.frame_ready, self.frame_lock, self.frame_cache)
# self.frame_tracker.start()
# # start a thread to resize regions
# self.region_prepper = RegionPrepper(self, self.frame_cache, self.resize_queue, prepped_frame_queue)
# self.region_prepper.start()
# # start a thread to store the highest scoring recent frames for monitored object types
# self.best_frames = BestFrames(self)
# self.best_frames.start()
# # start a thread to expire objects from the detected objects list
# self.object_cleaner = ObjectCleaner(self)
# self.object_cleaner.start()
# # start a thread to refine regions when objects are clipped
# self.dynamic_region_fps = EventsPerSecond()
# self.region_refiner = RegionRefiner(self)
# self.region_refiner.start()
# self.dynamic_region_fps.start()
# # start a thread to publish object scores
# mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self)
# mqtt_publisher.start()
# # create a watchdog thread for capture process
# self.watchdog = CameraWatchdog(self)
# # load in the mask for object detection
# if 'mask' in self.config:
# self.mask = cv2.imread("/config/{}".format(self.config['mask']), cv2.IMREAD_GRAYSCALE)
# else:
# self.mask = None
# if self.mask is None:
# self.mask = np.zeros((self.frame_shape[0], self.frame_shape[1], 1), np.uint8)
# self.mask[:] = 255
# def start_or_restart_capture(self):
# if not self.ffmpeg_process is None:
# print("Terminating the existing ffmpeg process...")
# self.ffmpeg_process.terminate()
# try:
# print("Waiting for ffmpeg to exit gracefully...")
# self.ffmpeg_process.wait(timeout=30)
# except sp.TimeoutExpired:
# print("FFmpeg didnt exit. Force killing...")
# self.ffmpeg_process.kill()
# self.ffmpeg_process.wait()
# print("Waiting for the capture thread to exit...")
# self.capture_thread.join()
# self.ffmpeg_process = None
# self.capture_thread = None
>>>>>>> 9b1c7e9... split into separate processes
# # create the process to capture frames from the input stream and store in a shared array
# print("Creating a new ffmpeg process...")
# self.start_ffmpeg()
# print("Creating a new capture thread...")
# self.capture_thread = CameraCapture(self)
# print("Starting a new capture thread...")
# self.capture_thread.start()
# self.fps.start()
# self.skipped_region_tracker.start()
# def start_ffmpeg(self):
# ffmpeg_cmd = (['ffmpeg'] +
# self.ffmpeg_global_args +
# self.ffmpeg_hwaccel_args +
# self.ffmpeg_input_args +
# ['-i', self.ffmpeg_input] +
# self.ffmpeg_output_args +
# ['pipe:'])
# print(" ".join(ffmpeg_cmd))
# self.ffmpeg_process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=self.frame_size)
# def start(self):
# self.start_or_restart_capture()
# self.watchdog.start()
# def join(self):
# self.capture_thread.join()
# def get_capture_pid(self):
# return self.ffmpeg_process.pid
# def get_best(self, label):
# return self.best_frames.best_frames.get(label)
# def stats(self):
# # TODO: anything else?
# return {
# 'camera_fps': self.fps.eps(60),
# 'resize_queue': self.resize_queue.qsize(),
# 'frame_queue': self.frame_queue.qsize(),
# 'finished_frame_queue': self.finished_frame_queue.qsize(),
# 'refined_frame_queue': self.refined_frame_queue.qsize(),
# 'regions_in_process': self.regions_in_process,
# 'dynamic_regions_per_sec': self.dynamic_region_fps.eps(),
# 'skipped_regions_per_sec': self.skipped_region_tracker.eps(60)
# }
# def frame_with_objects(self, frame_time, tracked_objects=None):
# if not frame_time in self.frame_cache:
# frame = np.zeros(self.frame_shape, np.uint8)
# else:
# frame = self.frame_cache[frame_time].copy()
# detected_objects = self.detected_objects[frame_time].copy()
# for region in self.regions:
# color = (255,255,255)
# cv2.rectangle(frame, (region['x_offset'], region['y_offset']),
# (region['x_offset']+region['size'], region['y_offset']+region['size']),
# color, 2)
# # draw the bounding boxes on the screen
# if tracked_objects is None:
# with self.object_tracker.tracked_objects_lock:
# tracked_objects = copy.deepcopy(self.object_tracker.tracked_objects)
# for obj in detected_objects:
# draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']), thickness=3)
# for id, obj in tracked_objects.items():
# color = (0, 255,0) if obj['frame_time'] == frame_time else (255, 0, 0)
# draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], id, color=color, thickness=1, position='bl')
# # print a timestamp
# time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
# cv2.putText(frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
# # print fps
# cv2.putText(frame, str(self.fps.eps())+'FPS', (10, 60), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
# # convert to BGR
# frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
# # encode the image into a jpg
# ret, jpg = cv2.imencode('.jpg', frame)
# return jpg.tobytes()
# def get_current_frame_with_objects(self):
# frame_time = self.last_processed_frame
# if frame_time == self.cached_frame_with_objects['frame_time']:
# return self.cached_frame_with_objects['frame_bytes']
# frame_bytes = self.frame_with_objects(frame_time)
# self.cached_frame_with_objects = {
# 'frame_bytes': frame_bytes,
# 'frame_time': frame_time
# }
# return frame_bytes
def filtered(obj, objects_to_track, object_filters, mask):
object_name = obj[0]
if not object_name in objects_to_track:
return True
if object_name in object_filters:
obj_settings = object_filters[object_name]
# if the min area is larger than the
# detected object, don't add it to detected objects
if obj_settings.get('min_area',-1) > obj[3]:
return True
# if the detected object is larger than the
# max area, don't add it to detected objects
if obj_settings.get('max_area', 24000000) < obj[3]:
return True
# if the score is lower than the threshold, skip
if obj_settings.get('threshold', 0) > obj[1]:
return True
# compute the coordinates of the object and make sure
# the location isnt outside the bounds of the image (can happen from rounding)
y_location = min(int(obj[2][3]), len(mask)-1)
x_location = min(int((obj[2][2]-obj[2][0])/2.0)+obj[2][0], len(mask[0])-1)
# if the object is in a masked location, don't add it to detected objects
if mask[y_location][x_location] == [0]:
return True
return False
def create_tensor_input(frame, region):
cropped_frame = frame[region[1]:region[3], region[0]:region[2]]
# Resize to 300x300 if needed
if cropped_frame.shape != (300, 300, 3):
cropped_frame = cv2.resize(cropped_frame, dsize=(300, 300), interpolation=cv2.INTER_LINEAR)
# Expand dimensions since the model expects images to have shape: [1, 300, 300, 3]
return np.expand_dims(cropped_frame, axis=0)
def track_camera(name, config, ffmpeg_global_config, global_objects_config, detect_lock, detect_ready, frame_ready, detected_objects_queue, fps, avg_wait):
print(f"Starting process for {name}: {os.getpid()}")
# Merge the ffmpeg config with the global config
ffmpeg = config.get('ffmpeg', {})
ffmpeg_input = get_ffmpeg_input(ffmpeg['input'])
ffmpeg_global_args = ffmpeg.get('global_args', ffmpeg_global_config['global_args'])
ffmpeg_hwaccel_args = ffmpeg.get('hwaccel_args', ffmpeg_global_config['hwaccel_args'])
ffmpeg_input_args = ffmpeg.get('input_args', ffmpeg_global_config['input_args'])
ffmpeg_output_args = ffmpeg.get('output_args', ffmpeg_global_config['output_args'])
# Merge the tracked object config with the global config
camera_objects_config = config.get('objects', {})
# combine tracked objects lists
objects_to_track = set().union(global_objects_config.get('track', ['person', 'car', 'truck']), camera_objects_config.get('track', []))
# merge object filters
global_object_filters = global_objects_config.get('filters', {})
camera_object_filters = camera_objects_config.get('filters', {})
objects_with_config = set().union(global_object_filters.keys(), camera_object_filters.keys())
object_filters = {}
for obj in objects_with_config:
object_filters[obj] = {**global_object_filters.get(obj, {}), **camera_object_filters.get(obj, {})}
take_frame = config.get('take_frame', 1)
# watchdog_timeout = config.get('watchdog_timeout', 300)
frame_shape = get_frame_shape(ffmpeg_input)
frame_size = frame_shape[0] * frame_shape[1] * frame_shape[2]
try:
sa.delete(name)
except:
pass
frame = sa.create(name, shape=frame_shape, dtype=np.uint8)
# load in the mask for object detection
if 'mask' in config:
mask = cv2.imread("/config/{}".format(config['mask']), cv2.IMREAD_GRAYSCALE)
else:
mask = None
if mask is None:
mask = np.zeros((frame_shape[0], frame_shape[1], 1), np.uint8)
mask[:] = 255
motion_detector = MotionDetector(frame_shape, mask, resize_factor=6)
object_detector = RemoteObjectDetector('/lab/labelmap.txt', detect_lock, detect_ready, frame_ready)
object_tracker = ObjectTracker(10)
ffmpeg_cmd = (['ffmpeg'] +
ffmpeg_global_args +
ffmpeg_hwaccel_args +
ffmpeg_input_args +
['-i', ffmpeg_input] +
ffmpeg_output_args +
['pipe:'])
print(" ".join(ffmpeg_cmd))
print(" ".join(ffmpeg_cmd))
self.ffmpeg_process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=self.frame_size)
ffmpeg_process = sp.Popen(ffmpeg_cmd, stdout = sp.PIPE, bufsize=frame_size)
def start(self):
self.start_or_restart_capture()
self.watchdog.start()
plasma_client = plasma.connect("/tmp/plasma")
frame_num = 0
fps_tracker = EventsPerSecond()
fps_tracker.start()
while True:
# TODO: implement something to determine if it had to wait for a frame at all
# to determine if it might be behind and the buffer is filling up
start = datetime.datetime.now().timestamp()
frame_bytes = ffmpeg_process.stdout.read(frame_size)
duration = datetime.datetime.now().timestamp()-start
avg_wait.value = (avg_wait.value*9 + duration)/10
def join(self):
self.capture_thread.join()
if not frame_bytes:
# TODO: restart the ffmpeg process and track number of restarts
break
def get_capture_pid(self):
return self.ffmpeg_process.pid
# limit frame rate
frame_num += 1
if (frame_num % take_frame) != 0:
continue
def get_best(self, label):
return self.best_frames.best_frames.get(label)
fps_tracker.update()
fps.value = fps_tracker.eps()
def stats(self):
return {
'camera_fps': self.fps.eps(60),
'resize_queue': self.resize_queue.qsize(),
'frame_queue': self.frame_queue.qsize(),
'finished_frame_queue': self.finished_frame_queue.qsize(),
'refined_frame_queue': self.refined_frame_queue.qsize(),
'regions_in_process': self.regions_in_process,
'dynamic_regions_per_sec': self.dynamic_region_fps.eps(),
'skipped_regions_per_sec': self.skipped_region_tracker.eps(60)
}
frame_time = datetime.datetime.now().timestamp()
def frame_with_objects(self, frame_time, tracked_objects=None):
if not frame_time in self.frame_cache:
frame = np.zeros(self.frame_shape, np.uint8)
else:
frame = self.frame_cache[frame_time].copy()
# Store frame in numpy array
frame[:] = (np
.frombuffer(frame_bytes, np.uint8)
.reshape(frame_shape))
detected_objects = self.detected_objects[frame_time].copy()
# look for motion
motion_boxes = motion_detector.detect(frame)
for region in self.regions:
color = (255,255,255)
cv2.rectangle(frame, (region['x_offset'], region['y_offset']),
(region['x_offset']+region['size'], region['y_offset']+region['size']),
color, 2)
tracked_objects = object_tracker.tracked_objects.values()
# draw the bounding boxes on the screen
# merge areas of motion that intersect with a known tracked object into a single area to look at
areas_of_interest = []
used_motion_boxes = []
for obj in tracked_objects:
x_min, y_min, x_max, y_max = obj['box']
for m_index, motion_box in enumerate(motion_boxes):
if area(intersection(obj['box'], motion_box))/area(motion_box) > .5:
used_motion_boxes.append(m_index)
x_min = min(obj['box'][0], motion_box[0])
y_min = min(obj['box'][1], motion_box[1])
x_max = max(obj['box'][2], motion_box[2])
y_max = max(obj['box'][3], motion_box[3])
areas_of_interest.append((x_min, y_min, x_max, y_max))
unused_motion_boxes = set(range(0, len(motion_boxes))).difference(used_motion_boxes)
if tracked_objects is None:
with self.object_tracker.tracked_objects_lock:
tracked_objects = copy.deepcopy(self.object_tracker.tracked_objects)
# compute motion regions
motion_regions = [calculate_region(frame_shape, motion_boxes[i][0], motion_boxes[i][1], motion_boxes[i][2], motion_boxes[i][3], 1.2)
for i in unused_motion_boxes]
for obj in detected_objects:
draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], "{}% {}".format(int(obj['score']*100), obj['area']), thickness=3)
# compute tracked object regions
object_regions = [calculate_region(frame_shape, a[0], a[1], a[2], a[3], 1.2)
for a in areas_of_interest]
for id, obj in tracked_objects.items():
color = (0, 255,0) if obj['frame_time'] == frame_time else (255, 0, 0)
draw_box_with_label(frame, obj['box']['xmin'], obj['box']['ymin'], obj['box']['xmax'], obj['box']['ymax'], obj['name'], id, color=color, thickness=1, position='bl')
# merge regions with high IOU
merged_regions = motion_regions+object_regions
while True:
max_iou = 0.0
max_indices = None
region_indices = range(len(merged_regions))
for a, b in itertools.combinations(region_indices, 2):
iou = intersection_over_union(merged_regions[a], merged_regions[b])
if iou > max_iou:
max_iou = iou
max_indices = (a, b)
if max_iou > 0.1:
a = merged_regions[max_indices[0]]
b = merged_regions[max_indices[1]]
merged_regions.append(calculate_region(frame_shape,
min(a[0], b[0]),
min(a[1], b[1]),
max(a[2], b[2]),
max(a[3], b[3]),
1
))
del merged_regions[max(max_indices[0], max_indices[1])]
del merged_regions[min(max_indices[0], max_indices[1])]
else:
break
# print a timestamp
time_to_show = datetime.datetime.fromtimestamp(frame_time).strftime("%m/%d/%Y %H:%M:%S")
cv2.putText(frame, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
# resize regions and detect
detections = []
for region in merged_regions:
# print fps
cv2.putText(frame, str(self.fps.eps())+'FPS', (10, 60), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2)
tensor_input = create_tensor_input(frame, region)
# convert to BGR
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
region_detections = object_detector.detect(tensor_input)
# encode the image into a jpg
ret, jpg = cv2.imencode('.jpg', frame)
for d in region_detections:
box = d[2]
size = region[2]-region[0]
x_min = int((box[1] * size) + region[0])
y_min = int((box[0] * size) + region[1])
x_max = int((box[3] * size) + region[0])
y_max = int((box[2] * size) + region[1])
det = (d[0],
d[1],
(x_min, y_min, x_max, y_max),
(x_max-x_min)*(y_max-y_min),
region)
if filtered(det, objects_to_track, object_filters, mask):
continue
detections.append(det)
return jpg.tobytes()
#########
# merge objects, check for clipped objects and look again up to N times
#########
refining = True
refine_count = 0
while refining and refine_count < 4:
refining = False
def get_current_frame_with_objects(self):
frame_time = self.last_processed_frame
if frame_time == self.cached_frame_with_objects['frame_time']:
return self.cached_frame_with_objects['frame_bytes']
# group by name
detected_object_groups = defaultdict(lambda: [])
for detection in detections:
detected_object_groups[detection[0]].append(detection)
frame_bytes = self.frame_with_objects(frame_time)
self.cached_frame_with_objects = {
'frame_bytes': frame_bytes,
'frame_time': frame_time
}
return frame_bytes
selected_objects = []
for group in detected_object_groups.values():
# apply non-maxima suppression to suppress weak, overlapping bounding boxes
boxes = [(o[2][0], o[2][1], o[2][2]-o[2][0], o[2][3]-o[2][1])
for o in group]
confidences = [o[1] for o in group]
idxs = cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
for index in idxs:
obj = group[index[0]]
if clipped(obj, frame_shape): #obj['clipped']:
box = obj[2]
# calculate a new region that will hopefully get the entire object
region = calculate_region(frame_shape,
box[0], box[1],
box[2], box[3])
tensor_input = create_tensor_input(frame, region)
# run detection on new region
refined_detections = object_detector.detect(tensor_input)
for d in refined_detections:
box = d[2]
size = region[2]-region[0]
x_min = int((box[1] * size) + region[0])
y_min = int((box[0] * size) + region[1])
x_max = int((box[3] * size) + region[0])
y_max = int((box[2] * size) + region[1])
det = (d[0],
d[1],
(x_min, y_min, x_max, y_max),
(x_max-x_min)*(y_max-y_min),
region)
if filtered(det, objects_to_track, object_filters, mask):
continue
selected_objects.append(det)
refining = True
else:
selected_objects.append(obj)
# set the detections list to only include top, complete objects
# and new detections
detections = selected_objects
if refining:
refine_count += 1
# now that we have refined our detections, we need to track objects
object_tracker.match_and_update(frame_time, detections)
# put the frame in the plasma store
object_id = hashlib.sha1(str.encode(f"{name}{frame_time}")).digest()
plasma_client.put(frame, plasma.ObjectID(object_id))
# add to the queue
detected_objects_queue.put((name, frame_time, object_tracker.tracked_objects))
# if (frames >= 700 and frames <= 1635) or (frames >= 2500):
# if (frames >= 300 and frames <= 600):
# if (frames >= 0):
# row1 = cv2.hconcat([gray, cv2.convertScaleAbs(avg_frame)])
# row2 = cv2.hconcat([frameDelta, thresh])
# cv2.imwrite(f"/lab/debug/output/{frames}.jpg", cv2.vconcat([row1, row2]))
# # cv2.imwrite(f"/lab/debug/output/resized-frame-{frames}.jpg", resized_frame)
# for region in motion_regions:
# cv2.rectangle(frame, (region[0], region[1]), (region[2], region[3]), (255,128,0), 2)
# for region in object_regions:
# cv2.rectangle(frame, (region[0], region[1]), (region[2], region[3]), (0,128,255), 2)
# for region in merged_regions:
# cv2.rectangle(frame, (region[0], region[1]), (region[2], region[3]), (0,255,0), 2)
# for box in motion_boxes:
# cv2.rectangle(frame, (box[0], box[1]), (box[2], box[3]), (255,0,0), 2)
# for detection in detections:
# box = detection[2]
# draw_box_with_label(frame, box[0], box[1], box[2], box[3], detection[0], f"{detection[1]*100}%")
# for obj in object_tracker.tracked_objects.values():
# box = obj['box']
# draw_box_with_label(frame, box[0], box[1], box[2], box[3], obj['label'], obj['id'], thickness=1, color=(0,0,255), position='bl')
# cv2.putText(frame, str(total_detections), (10, 10), cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.5, color=(0, 0, 0), thickness=2)
# cv2.putText(frame, str(frame_detections), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.5, color=(0, 0, 0), thickness=2)
# cv2.imwrite(f"/lab/debug/output/frame-{frames}.jpg", frame)
# break
# start a thread to publish object scores
# mqtt_publisher = MqttObjectPublisher(self.mqtt_client, self.mqtt_topic_prefix, self)
# mqtt_publisher.start()
# create a watchdog thread for capture process
# self.watchdog = CameraWatchdog(self)