OpenShot Library | libopenshot  0.2.7
Tracker.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for Tracker effect class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  * @author Brenno Caldato <brenno.caldato@outlook.com>
6  *
7  * @ref License
8  */
9 
10 /* LICENSE
11  *
12  * Copyright (c) 2008-2019 OpenShot Studios, LLC
13  * <http://www.openshotstudios.com/>. This file is part of
14  * OpenShot Library (libopenshot), an open-source project dedicated to
15  * delivering high quality video editing and animation solutions to the
16  * world. For more information visit <http://www.openshot.org/>.
17  *
18  * OpenShot Library (libopenshot) is free software: you can redistribute it
19  * and/or modify it under the terms of the GNU Lesser General Public License
20  * as published by the Free Software Foundation, either version 3 of the
21  * License, or (at your option) any later version.
22  *
23  * OpenShot Library (libopenshot) is distributed in the hope that it will be
24  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
25  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
26  * GNU Lesser General Public License for more details.
27  *
28  * You should have received a copy of the GNU Lesser General Public License
29  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
30  */
31 
32 #include <string>
33 #include <memory>
34 #include <fstream>
35 #include <iostream>
36 
37 #include "effects/Tracker.h"
38 #include "Exceptions.h"
39 #include "Timeline.h"
40 
41 #include <google/protobuf/util/time_util.h>
42 
43 #include <QImage>
44 #include <QPainter>
45 #include <QRectF>
46 
47 using namespace std;
48 using namespace openshot;
49 using google::protobuf::util::TimeUtil;
50 
51 /// Blank constructor, useful when using Json to load the effect properties
52 Tracker::Tracker(std::string clipTrackerDataPath)
53 {
54  // Init effect properties
55  init_effect_details();
56  // Instantiate a TrackedObjectBBox object and point to it
57  TrackedObjectBBox trackedDataObject;
58  trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
59  // Tries to load the tracked object's data from protobuf file
60  trackedData->LoadBoxData(clipTrackerDataPath);
61  ClipBase* parentClip = this->ParentClip();
62  trackedData->ParentClip(parentClip);
63  trackedData->Id(std::to_string(0));
64  // Insert TrackedObject with index 0 to the trackedObjects map
65  trackedObjects.insert({0, trackedData});
66 }
67 
68 // Default constructor
69 Tracker::Tracker()
70 {
71  // Init effect properties
72  init_effect_details();
73  // Instantiate a TrackedObjectBBox object and point to it
74  TrackedObjectBBox trackedDataObject;
75  trackedData = std::make_shared<TrackedObjectBBox>(trackedDataObject);
76  ClipBase* parentClip = this->ParentClip();
77  trackedData->ParentClip(parentClip);
78  trackedData->Id(std::to_string(0));
79  // Insert TrackedObject with index 0 to the trackedObjects map
80  trackedObjects.insert({0, trackedData});
81 }
82 
83 
84 // Init effect settings
85 void Tracker::init_effect_details()
86 {
87  /// Initialize the values of the EffectInfo struct.
88  InitEffectInfo();
89 
90  /// Set the effect info
91  info.class_name = "Tracker";
92  info.name = "Tracker";
93  info.description = "Track the selected bounding box through the video.";
94  info.has_audio = false;
95  info.has_video = true;
96  info.has_tracked_object = true;
97 
98  this->TimeScale = 1.0;
99 }
100 
101 // This method is required for all derived classes of EffectBase, and returns a
102 // modified openshot::Frame object
103 std::shared_ptr<Frame> Tracker::GetFrame(std::shared_ptr<Frame> frame, int64_t frame_number)
104 {
105  // Get the frame's image
106  cv::Mat frame_image = frame->GetImageCV();
107 
108  // Initialize the Qt rectangle that will hold the positions of the bounding-box
109  QRectF boxRect;
110  // Initialize the image of the TrackedObject child clip
111  std::shared_ptr<QImage> childClipImage = nullptr;
112 
113  // Check if frame isn't NULL
114  if(!frame_image.empty() &&
115  trackedData->Contains(frame_number) &&
116  trackedData->visible.GetValue(frame_number) == 1)
117  {
118  // Get the width and height of the image
119  float fw = frame_image.size().width;
120  float fh = frame_image.size().height;
121 
122  // Get the bounding-box of given frame
123  BBox fd = trackedData->GetBox(frame_number);
124 
125  // Check if track data exists for the requested frame
126  if (trackedData->draw_box.GetValue(frame_number) == 1)
127  {
128  std::vector<int> stroke_rgba = trackedData->stroke.GetColorRGBA(frame_number);
129  int stroke_width = trackedData->stroke_width.GetValue(frame_number);
130  float stroke_alpha = trackedData->stroke_alpha.GetValue(frame_number);
131  std::vector<int> bg_rgba = trackedData->background.GetColorRGBA(frame_number);
132  float bg_alpha = trackedData->background_alpha.GetValue(frame_number);
133 
134  // Create a rotated rectangle object that holds the bounding box
135  cv::RotatedRect box ( cv::Point2f( (int)(fd.cx*fw), (int)(fd.cy*fh) ),
136  cv::Size2f( (int)(fd.width*fw), (int)(fd.height*fh) ),
137  (int) (fd.angle) );
138 
139  DrawRectangleRGBA(frame_image, box, bg_rgba, bg_alpha, 1, true);
140  DrawRectangleRGBA(frame_image, box, stroke_rgba, stroke_alpha, stroke_width, false);
141  }
142 
143  // Get the image of the Tracked Object' child clip
144  if (trackedData->ChildClipId() != ""){
145  // Cast the parent timeline of this effect
146  Timeline* parentTimeline = (Timeline *) ParentTimeline();
147  if (parentTimeline){
148  // Get the Tracked Object's child clip
149  Clip* childClip = parentTimeline->GetClip(trackedData->ChildClipId());
150  if (childClip){
151  // Get the image of the child clip for this frame
152  std::shared_ptr<Frame> f(new Frame(1, frame->GetWidth(), frame->GetHeight(), "#00000000"));
153  std::shared_ptr<Frame> childClipFrame = childClip->GetFrame(f, frame_number);
154  childClipImage = childClipFrame->GetImage();
155 
156  // Set the Qt rectangle with the bounding-box properties
157  boxRect.setRect((int)((fd.cx-fd.width/2)*fw),
158  (int)((fd.cy - fd.height/2)*fh),
159  (int)(fd.width*fw),
160  (int)(fd.height*fh) );
161  }
162  }
163  }
164 
165  }
166 
167  // Set image with drawn box to frame
168  // If the input image is NULL or doesn't have tracking data, it's returned as it came
169  frame->SetImageCV(frame_image);
170 
171  // Set the bounding-box image with the Tracked Object's child clip image
172  if (childClipImage){
173  // Get the frame image
174  QImage frameImage = *(frame->GetImage());
175 
176  // Set a Qt painter to the frame image
177  QPainter painter(&frameImage);
178 
179  // Draw the child clip image inside the bounding-box
180  painter.drawImage(boxRect, *childClipImage, QRectF(0, 0, frameImage.size().width(), frameImage.size().height()));
181 
182  // Set the frame image as the composed image
183  frame->AddImage(std::make_shared<QImage>(frameImage));
184  }
185 
186  return frame;
187 }
188 
189 void Tracker::DrawRectangleRGBA(cv::Mat &frame_image, cv::RotatedRect box, std::vector<int> color, float alpha, int thickness, bool is_background){
190  // Get the bouding box vertices
191  cv::Point2f vertices2f[4];
192  box.points(vertices2f);
193 
194  // TODO: take a rectangle of frame_image by refencence and draw on top of that to improve speed
195  // select min enclosing rectangle to draw on a small portion of the image
196  // cv::Rect rect = box.boundingRect();
197  // cv::Mat image = frame_image(rect)
198 
199  if(is_background){
200  cv::Mat overlayFrame;
201  frame_image.copyTo(overlayFrame);
202 
203  // draw bounding box background
204  cv::Point vertices[4];
205  for(int i = 0; i < 4; ++i){
206  vertices[i] = vertices2f[i];}
207 
208  cv::Rect rect = box.boundingRect();
209  cv::fillConvexPoly(overlayFrame, vertices, 4, cv::Scalar(color[2],color[1],color[0]), cv::LINE_AA);
210  // add opacity
211  cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
212  }
213  else{
214  cv::Mat overlayFrame;
215  frame_image.copyTo(overlayFrame);
216 
217  // Draw bounding box
218  for (int i = 0; i < 4; i++)
219  {
220  cv::line(overlayFrame, vertices2f[i], vertices2f[(i+1)%4], cv::Scalar(color[2],color[1],color[0]),
221  thickness, cv::LINE_AA);
222  }
223 
224  // add opacity
225  cv::addWeighted(overlayFrame, 1-alpha, frame_image, alpha, 0, frame_image);
226  }
227 }
228 
229 // Get the indexes and IDs of all visible objects in the given frame
230 std::string Tracker::GetVisibleObjects(int64_t frame_number) const{
231 
232  // Initialize the JSON objects
233  Json::Value root;
234  root["visible_objects_index"] = Json::Value(Json::arrayValue);
235  root["visible_objects_id"] = Json::Value(Json::arrayValue);
236 
237  // Iterate through the tracked objects
238  for (const auto& trackedObject : trackedObjects){
239  // Get the tracked object JSON properties for this frame
240  Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(frame_number);
241  if (trackedObjectJSON["visible"]["value"].asBool()){
242  // Save the object's index and ID if it's visible in this frame
243  root["visible_objects_index"].append(trackedObject.first);
244  root["visible_objects_id"].append(trackedObject.second->Id());
245  }
246  }
247 
248  return root.toStyledString();
249 }
250 
251 // Generate JSON string of this object
252 std::string Tracker::Json() const {
253 
254  // Return formatted string
255  return JsonValue().toStyledString();
256 }
257 
258 // Generate Json::Value for this object
259 Json::Value Tracker::JsonValue() const {
260 
261  // Create root json object
262  Json::Value root = EffectBase::JsonValue(); // get parent properties
263 
264  // Save the effect's properties on root
265  root["type"] = info.class_name;
266  root["protobuf_data_path"] = protobuf_data_path;
267  root["BaseFPS"]["num"] = BaseFPS.num;
268  root["BaseFPS"]["den"] = BaseFPS.den;
269  root["TimeScale"] = this->TimeScale;
270 
271  // Add trackedObjects IDs to JSON
272  Json::Value objects;
273  for (auto const& trackedObject : trackedObjects){
274  Json::Value trackedObjectJSON = trackedObject.second->JsonValue();
275  // add object json
276  objects[trackedObject.second->Id()] = trackedObjectJSON;
277  }
278  root["objects"] = objects;
279 
280  // return JsonValue
281  return root;
282 }
283 
284 // Load JSON string into this object
285 void Tracker::SetJson(const std::string value) {
286 
287  // Parse JSON string into JSON objects
288  try
289  {
290  const Json::Value root = openshot::stringToJson(value);
291  // Set all values that match
292  SetJsonValue(root);
293  }
294  catch (const std::exception& e)
295  {
296  // Error parsing JSON (or missing keys)
297  throw InvalidJSON("JSON is invalid (missing keys or invalid data types)");
298  }
299  return;
300 }
301 
302 // Load Json::Value into this object
303 void Tracker::SetJsonValue(const Json::Value root) {
304 
305  // Set parent data
306  EffectBase::SetJsonValue(root);
307 
308  if(!root["type"].isNull())
309  info.class_name = root["type"].asString();
310 
311  if (!root["BaseFPS"].isNull() && root["BaseFPS"].isObject())
312  {
313  if (!root["BaseFPS"]["num"].isNull())
314  {
315  BaseFPS.num = (int) root["BaseFPS"]["num"].asInt();
316  }
317  if (!root["BaseFPS"]["den"].isNull())
318  {
319  BaseFPS.den = (int) root["BaseFPS"]["den"].asInt();
320  }
321  }
322 
323  if (!root["TimeScale"].isNull())
324  TimeScale = (double) root["TimeScale"].asDouble();
325 
326  // Set data from Json (if key is found)
327  if (!root["protobuf_data_path"].isNull() && protobuf_data_path.size() <= 1)
328  {
329  protobuf_data_path = root["protobuf_data_path"].asString();
330  if(!trackedData->LoadBoxData(protobuf_data_path))
331  {
332  std::clog << "Invalid protobuf data path " << protobuf_data_path << '\n';
333  protobuf_data_path = "";
334  }
335  }
336 
337  if (!root["objects"].isNull()){
338  for (auto const& trackedObject : trackedObjects){
339  std::string obj_id = std::to_string(trackedObject.first);
340  if(!root["objects"][obj_id].isNull()){
341  trackedObject.second->SetJsonValue(root["objects"][obj_id]);
342  }
343  }
344  }
345 
346  // Set the tracked object's ids
347  if (!root["objects_id"].isNull()){
348  for (auto const& trackedObject : trackedObjects){
349  Json::Value trackedObjectJSON;
350  trackedObjectJSON["box_id"] = root["objects_id"][trackedObject.first].asString();
351  trackedObject.second->SetJsonValue(trackedObjectJSON);
352  }
353  }
354 
355  return;
356 }
357 
358 // Get all properties for a specific frame
359 std::string Tracker::PropertiesJSON(int64_t requested_frame) const {
360 
361  // Generate JSON properties list
362  Json::Value root;
363 
364  // Add trackedObject properties to JSON
365  Json::Value objects;
366  for (auto const& trackedObject : trackedObjects){
367  Json::Value trackedObjectJSON = trackedObject.second->PropertiesJSON(requested_frame);
368  // add object json
369  objects[trackedObject.second->Id()] = trackedObjectJSON;
370  }
371  root["objects"] = objects;
372 
373  // Append effect's properties
374  root["id"] = add_property_json("ID", 0.0, "string", Id(), NULL, -1, -1, true, requested_frame);
375  root["position"] = add_property_json("Position", Position(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
376  root["layer"] = add_property_json("Track", Layer(), "int", "", NULL, 0, 20, false, requested_frame);
377  root["start"] = add_property_json("Start", Start(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
378  root["end"] = add_property_json("End", End(), "float", "", NULL, 0, 1000 * 60 * 30, false, requested_frame);
379  root["duration"] = add_property_json("Duration", Duration(), "float", "", NULL, 0, 1000 * 60 * 30, true, requested_frame);
380 
381  // Return formatted string
382  return root.toStyledString();
383 }
Header file for all Exception classes.
Header file for Timeline class.
Header file for Tracker effect class.
This abstract class is the base class, used by all clips in libopenshot.
Definition: ClipBase.h:51
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
std::shared_ptr< openshot::Frame > GetFrame(int64_t frame_number) override
Get an openshot::Frame object for a specific frame number of this clip. The image size and number of ...
Definition: Clip.cpp:360
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:108
Exception for invalid JSON.
Definition: Exceptions.h:206
This class represents a timeline.
Definition: Timeline.h:168
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
Definition: Timeline.cpp:404
This class contains the properties of a tracked object and functions to manipulate it.
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
const Json::Value stringToJson(const std::string value)
Definition: Json.cpp:34
This struct holds the information of a bounding-box.
float cy
y-coordinate of the bounding box center
float height
bounding box height
float cx
x-coordinate of the bounding box center
float width
bounding box width
float angle
bounding box rotation angle [degrees]