OpenShot Library | libopenshot  0.2.7
Timeline.h
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Header file for Timeline class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @ref License
7  */
8 
9 /* LICENSE
10  *
11  * Copyright (c) 2008-2019 OpenShot Studios, LLC
12  * <http://www.openshotstudios.com/>. This file is part of
13  * OpenShot Library (libopenshot), an open-source project dedicated to
14  * delivering high quality video editing and animation solutions to the
15  * world. For more information visit <http://www.openshot.org/>.
16  *
17  * OpenShot Library (libopenshot) is free software: you can redistribute it
18  * and/or modify it under the terms of the GNU Lesser General Public License
19  * as published by the Free Software Foundation, either version 3 of the
20  * License, or (at your option) any later version.
21  *
22  * OpenShot Library (libopenshot) is distributed in the hope that it will be
23  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
24  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25  * GNU Lesser General Public License for more details.
26  *
27  * You should have received a copy of the GNU Lesser General Public License
28  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
29  */
30 
31 #ifndef OPENSHOT_TIMELINE_H
32 #define OPENSHOT_TIMELINE_H
33 
34 #include <list>
35 #include <memory>
36 #include <mutex>
37 #include <set>
38 #include <QtGui/QImage>
39 #include <QtGui/QPainter>
40 #include <QtCore/QRegularExpression>
41 
42 #include "TimelineBase.h"
43 #include "ReaderBase.h"
44 
45 #include "Color.h"
46 #include "Clip.h"
47 #include "EffectBase.h"
48 #include "Fraction.h"
49 #include "Frame.h"
50 #include "KeyFrame.h"
51 #ifdef USE_OPENCV
52 #include "TrackedObjectBBox.h"
53 #endif
54 #include "TrackedObjectBase.h"
55 
56 
57 
58 namespace openshot {
59 
60  // Forward decls
61  class FrameMapper;
62  class CacheBase;
63 
64  /// Comparison method for sorting clip pointers (by Layer and then Position). Clips are sorted
65  /// from lowest layer to top layer (since that is the sequence they need to be combined), and then
66  /// by position (left to right).
67  struct CompareClips{
69  if( lhs->Layer() < rhs->Layer() ) return true;
70  if( lhs->Layer() == rhs->Layer() && lhs->Position() <= rhs->Position() ) return true;
71  return false;
72  }};
73 
74  /// Comparison method for sorting effect pointers (by Position, Layer, and Order). Effects are sorted
75  /// from lowest layer to top layer (since that is sequence clips are combined), and then by
76  /// position, and then by effect order.
79  if( lhs->Layer() < rhs->Layer() ) return true;
80  if( lhs->Layer() == rhs->Layer() && lhs->Position() < rhs->Position() ) return true;
81  if( lhs->Layer() == rhs->Layer() && lhs->Position() == rhs->Position() && lhs->Order() > rhs->Order() ) return true;
82  return false;
83  }};
84 
85  /// Comparison method for finding the far end of the timeline, by locating
86  /// the Clip with the highest end-frame number using std::max_element
88  bool operator()(const openshot::Clip* lhs, const openshot::Clip* rhs) {
89  return (lhs->Position() + lhs->Duration())
90  <= (rhs->Position() + rhs->Duration());
91  }};
92 
93  /// Like CompareClipEndFrames, but for effects
95  bool operator()(const openshot::EffectBase* lhs, const openshot::EffectBase* rhs) {
96  return (lhs->Position() + lhs->Duration())
97  <= (rhs->Position() + rhs->Duration());
98  }};
99 
100  /**
101  * @brief This class represents a timeline
102  *
103  * The timeline is one of the <b>most important</b> features of a video editor, and controls all
104  * aspects of how video, image, and audio clips are combined together, and how the final
105  * video output will be rendered. It has a collection of layers and clips, that arrange,
106  * sequence, and generate the final video output.
107  *
108  * The <b>following graphic</b> displays a timeline, and how clips can be arranged, scaled, and layered together. It
109  * also demonstrates how the viewport can be scaled smaller than the canvas, which can be used to zoom and pan around the
110  * canvas (i.e. pan & scan).
111  * \image html /doc/images/Timeline_Layers.png
112  *
113  * The <b>following graphic</b> displays how the playhead determines which frames to combine and layer.
114  * \image html /doc/images/Playhead.png
115  *
116  * Lets take a look at what the code looks like:
117  * @code
118  * // Create a Timeline
119  * Timeline t(1280, // width
120  * 720, // height
121  * Fraction(25,1), // framerate
122  * 44100, // sample rate
123  * 2 // channels
124  * ChannelLayout::LAYOUT_STEREO,
125  * );
126  *
127  * // Create some clips
128  * Clip c1(new ImageReader("MyAwesomeLogo.jpeg"));
129  * Clip c2(new FFmpegReader("BackgroundVideo.webm"));
130  *
131  * // CLIP 1 (logo) - Set some clip properties (with Keyframes)
132  * c1.Position(0.0); // Set the position or location (in seconds) on the timeline
133  * c1.gravity = GRAVITY_LEFT; // Set the alignment / gravity of the clip (position on the screen)
134  * c1.scale = SCALE_CROP; // Set the scale mode (how the image is resized to fill the screen)
135  * c1.Layer(1); // Set the layer of the timeline (higher layers cover up images of lower layers)
136  * c1.Start(0.0); // Set the starting position of the video (trim the left side of the video)
137  * c1.End(16.0); // Set the ending position of the video (trim the right side of the video)
138  * c1.alpha.AddPoint(1, 0.0); // Set the alpha to transparent on frame #1
139  * c1.alpha.AddPoint(500, 0.0); // Keep the alpha transparent until frame #500
140  * c1.alpha.AddPoint(565, 1.0); // Animate the alpha from transparent to visible (between frame #501 and #565)
141  *
142  * // CLIP 2 (background video) - Set some clip properties (with Keyframes)
143  * c2.Position(0.0); // Set the position or location (in seconds) on the timeline
144  * c2.Start(10.0); // Set the starting position of the video (trim the left side of the video)
145  * c2.Layer(0); // Set the layer of the timeline (higher layers cover up images of lower layers)
146  * c2.alpha.AddPoint(1, 1.0); // Set the alpha to visible on frame #1
147  * c2.alpha.AddPoint(150, 0.0); // Animate the alpha to transparent (between frame 2 and frame #150)
148  * c2.alpha.AddPoint(360, 0.0, LINEAR); // Keep the alpha transparent until frame #360
149  * c2.alpha.AddPoint(384, 1.0); // Animate the alpha to visible (between frame #360 and frame #384)
150  *
151  * // Add clips to timeline
152  * t.AddClip(&c1);
153  * t.AddClip(&c2);
154  *
155  * // Open the timeline reader
156  * t.Open();
157  *
158  * // Get frame number 1 from the timeline (This will generate a new frame, made up from the previous clips and settings)
159  * std::shared_ptr<Frame> f = t.GetFrame(1);
160  *
161  * // Now that we have an openshot::Frame object, lets have some fun!
162  * f->Display(); // Display the frame on the screen
163  *
164  * // Close the timeline reader
165  * t.Close();
166  * @endcode
167  */
169  private:
170  bool is_open; ///<Is Timeline Open?
171  bool auto_map_clips; ///< Auto map framerates and sample rates to all clips
172  std::list<openshot::Clip*> clips; ///<List of clips on this timeline
173  std::list<openshot::Clip*> closing_clips; ///<List of clips that need to be closed
174  std::map<openshot::Clip*, openshot::Clip*> open_clips; ///<List of 'opened' clips on this timeline
175  std::list<openshot::EffectBase*> effects; ///<List of clips on this timeline
176  openshot::CacheBase *final_cache; ///<Final cache of timeline frames
177  std::set<openshot::FrameMapper*> allocated_frame_mappers; ///< all the frame mappers we allocated and must free
178  bool managed_cache; ///< Does this timeline instance manage the cache object
179  std::string path; ///< Optional path of loaded UTF-8 OpenShot JSON project file
180  std::mutex get_frame_mutex; ///< Mutex to protect GetFrame method from different threads calling it
181  int max_concurrent_frames; ///< Max concurrent frames to process at one time
182 
183  std::map<std::string, std::shared_ptr<openshot::TrackedObjectBase>> tracked_objects; ///< map of TrackedObjectBBoxes and their IDs
184 
185  /// Process a new layer of video or audio
186  void add_layer(std::shared_ptr<openshot::Frame> new_frame, openshot::Clip* source_clip, int64_t clip_frame_number, bool is_top_clip, float max_volume);
187 
188  /// Apply a FrameMapper to a clip which matches the settings of this timeline
189  void apply_mapper_to_clip(openshot::Clip* clip);
190 
191  // Apply JSON Diffs to various objects contained in this timeline
192  void apply_json_to_clips(Json::Value change); ///<Apply JSON diff to clips
193  void apply_json_to_effects(Json::Value change); ///< Apply JSON diff to effects
194  void apply_json_to_effects(Json::Value change, openshot::EffectBase* existing_effect); ///<Apply JSON diff to a specific effect
195  void apply_json_to_timeline(Json::Value change); ///<Apply JSON diff to timeline properties
196 
197  /// Calculate time of a frame number, based on a framerate
198  double calculate_time(int64_t number, openshot::Fraction rate);
199 
200  /// Find intersecting (or non-intersecting) openshot::Clip objects
201  ///
202  /// @returns A list of openshot::Clip objects
203  /// @param requested_frame The frame number that is requested.
204  /// @param number_of_frames The number of frames to check
205  /// @param include Include or Exclude intersecting clips
206  std::vector<openshot::Clip*> find_intersecting_clips(int64_t requested_frame, int number_of_frames, bool include);
207 
208  /// Get a clip's frame or generate a blank frame
209  std::shared_ptr<openshot::Frame> GetOrCreateFrame(std::shared_ptr<Frame> background_frame, openshot::Clip* clip, int64_t number, openshot::TimelineInfoStruct* options);
210 
211  /// Compare 2 floating point numbers for equality
212  bool isEqual(double a, double b);
213 
214  /// Sort clips by position on the timeline
215  void sort_clips();
216 
217  /// Sort effects by position on the timeline
218  void sort_effects();
219 
220  /// Update the list of 'opened' clips
221  void update_open_clips(openshot::Clip *clip, bool does_clip_intersect);
222 
223  public:
224 
225  /// @brief Constructor for the timeline (which configures the default frame properties)
226  /// @param width The image width of generated openshot::Frame objects
227  /// @param height The image height of generated openshot::Frame objects
228  /// @param fps The frame rate of the generated video
229  /// @param sample_rate The audio sample rate
230  /// @param channels The number of audio channels
231  /// @param channel_layout The channel layout (i.e. mono, stereo, 3 point surround, etc...)
232  Timeline(int width, int height, openshot::Fraction fps, int sample_rate, int channels, openshot::ChannelLayout channel_layout);
233 
234  /// @brief Constructor which takes a ReaderInfo struct to configure parameters
235  /// @param info The reader parameters to configure the new timeline with
237 
238  /// @brief Project-file constructor for the timeline
239  ///
240  /// Loads a JSON structure from a file path, and
241  /// initializes the timeline described within.
242  ///
243  /// @param projectPath The path of the UTF-8 *.osp project file (JSON contents). Contents will be loaded automatically.
244  /// @param convert_absolute_paths Should all paths be converted to absolute paths (relative to the location of projectPath)
245  Timeline(const std::string& projectPath, bool convert_absolute_paths);
246 
247  virtual ~Timeline();
248 
249  /// Add to the tracked_objects map a pointer to a tracked object (TrackedObjectBBox)
250  void AddTrackedObject(std::shared_ptr<openshot::TrackedObjectBase> trackedObject);
251  /// Return tracked object pointer by it's id
252  std::shared_ptr<openshot::TrackedObjectBase> GetTrackedObject(std::string id) const;
253  /// Return the ID's of the tracked objects as a list of strings
254  std::list<std::string> GetTrackedObjectsIds() const;
255  /// Return the trackedObject's properties as a JSON string
256  #ifdef USE_OPENCV
257  std::string GetTrackedObjectValues(std::string id, int64_t frame_number) const;
258  #endif
259 
260  /// @brief Add an openshot::Clip to the timeline
261  /// @param clip Add an openshot::Clip to the timeline. A clip can contain any type of Reader.
262  void AddClip(openshot::Clip* clip);
263 
264  /// @brief Add an effect to the timeline
265  /// @param effect Add an effect to the timeline. An effect can modify the audio or video of an openshot::Frame.
266  void AddEffect(openshot::EffectBase* effect);
267 
268  /// Apply global/timeline effects to the source frame (if any)
269  std::shared_ptr<openshot::Frame> apply_effects(std::shared_ptr<openshot::Frame> frame, int64_t timeline_frame_number, int layer);
270 
271  /// Apply the timeline's framerate and samplerate to all clips
272  void ApplyMapperToClips();
273 
274  /// Determine if clips are automatically mapped to the timeline's framerate and samplerate
275  bool AutoMapClips() { return auto_map_clips; };
276 
277  /// @brief Automatically map all clips to the timeline's framerate and samplerate
278  void AutoMapClips(bool auto_map) { auto_map_clips = auto_map; };
279 
280  /// Clear all cache for this timeline instance, and all clips, mappers, and readers under it
281  void ClearAllCache();
282 
283  /// Return a list of clips on the timeline
284  std::list<openshot::Clip*> Clips() { return clips; };
285 
286  /// Look up a single clip by ID
287  openshot::Clip* GetClip(const std::string& id);
288 
289  /// Look up a clip effect by ID
290  openshot::EffectBase* GetClipEffect(const std::string& id);
291 
292  /// Look up a timeline effect by ID
293  openshot::EffectBase* GetEffect(const std::string& id);
294 
295  /// Look up the end time of the latest timeline element
296  double GetMaxTime();
297  /// Look up the end frame number of the latest element on the timeline
298  int64_t GetMaxFrame();
299 
300  /// Close the timeline reader (and any resources it was consuming)
301  void Close() override;
302 
303  /// Return the list of effects on the timeline
304  std::list<openshot::EffectBase*> Effects() { return effects; };
305 
306  /// Return the list of effects on all clips
307  std::list<openshot::EffectBase*> ClipEffects() const;
308 
309  /// Get the cache object used by this reader
310  openshot::CacheBase* GetCache() override { return final_cache; };
311 
312  /// Set the cache object used by this reader. You must now manage the lifecycle
313  /// of this cache object though (Timeline will not delete it for you).
314  void SetCache(openshot::CacheBase* new_cache);
315 
316  /// Get an openshot::Frame object for a specific frame number of this timeline.
317  ///
318  /// @returns The requested frame (containing the image)
319  /// @param requested_frame The frame number that is requested.
320  std::shared_ptr<openshot::Frame> GetFrame(int64_t requested_frame) override;
321 
322  // Curves for the viewport
323  openshot::Keyframe viewport_scale; ///<Curve representing the scale of the viewport (0 to 100)
324  openshot::Keyframe viewport_x; ///<Curve representing the x coordinate for the viewport
325  openshot::Keyframe viewport_y; ///<Curve representing the y coordinate for the viewport
326 
327  // Background color
328  openshot::Color color; ///<Background color of timeline canvas
329 
330  /// Determine if reader is open or closed
331  bool IsOpen() override { return is_open; };
332 
333  /// Return the type name of the class
334  std::string Name() override { return "Timeline"; };
335 
336  // Get and Set JSON methods
337  std::string Json() const override; ///< Generate JSON string of this object
338  void SetJson(const std::string value) override; ///< Load JSON string into this object
339  Json::Value JsonValue() const override; ///< Generate Json::Value for this object
340  void SetJsonValue(const Json::Value root) override; ///< Load Json::Value into this object
341 
342  /// Set Max Image Size (used for performance optimization). Convenience function for setting
343  /// Settings::Instance()->MAX_WIDTH and Settings::Instance()->MAX_HEIGHT.
344  void SetMaxSize(int width, int height);
345 
346  /// @brief Apply a special formatted JSON object, which represents a change to the timeline (add, update, delete)
347  /// This is primarily designed to keep the timeline (and its child objects... such as clips and effects) in sync
348  /// with another application... such as OpenShot Video Editor (http://www.openshot.org).
349  /// @param value A JSON string containing a key, value, and type of change.
350  void ApplyJsonDiff(std::string value);
351 
352  /// Open the reader (and start consuming resources)
353  void Open() override;
354 
355  /// @brief Remove an openshot::Clip from the timeline
356  /// @param clip Remove an openshot::Clip from the timeline.
358 
359  /// @brief Remove an effect from the timeline
360  /// @param effect Remove an effect from the timeline.
361  void RemoveEffect(openshot::EffectBase* effect);
362  };
363 
364 }
365 
366 #endif // OPENSHOT_TIMELINE_H
Header file for Clip class.
Header file for Color class.
Header file for EffectBase class.
Header file for Fraction class.
Header file for Frame class.
Header file for the Keyframe class.
Header file for ReaderBase class.
Header file for Timeline class.
Header file for the TrackedObjectBBox class.
Header file for the TrackedObjectBase class.
All cache managers in libopenshot are based on this CacheBase class.
Definition: CacheBase.h:49
float Duration() const
Get the length of this clip (in seconds)
Definition: ClipBase.h:112
int Layer() const
Get layer of clip on timeline (lower number is covered by higher numbers)
Definition: ClipBase.h:109
float Position() const
Get position on timeline (in seconds)
Definition: ClipBase.h:108
This class represents a clip (used to arrange readers on the timeline)
Definition: Clip.h:109
This class represents a color (used on the timeline and clips)
Definition: Color.h:45
This abstract class is the base class, used by all effects in libopenshot.
Definition: EffectBase.h:71
int Order() const
Get the order that this effect should be executed.
Definition: EffectBase.h:130
This class represents a fraction.
Definition: Fraction.h:48
A Keyframe is a collection of Point instances, which is used to vary a number or property over time.
Definition: KeyFrame.h:72
This abstract class is the base class, used by all readers in libopenshot.
Definition: ReaderBase.h:98
openshot::ReaderInfo info
Information about the current media file.
Definition: ReaderBase.h:111
openshot::ClipBase * clip
Pointer to the parent clip instance (if any)
Definition: ReaderBase.h:103
This class represents a timeline (used for building generic timeline implementations)
Definition: TimelineBase.h:54
This class represents a timeline.
Definition: Timeline.h:168
void AddTrackedObject(std::shared_ptr< openshot::TrackedObjectBase > trackedObject)
Add to the tracked_objects map a pointer to a tracked object (TrackedObjectBBox)
Definition: Timeline.cpp:247
Json::Value JsonValue() const override
Generate Json::Value for this object.
Definition: Timeline.cpp:987
std::string Name() override
Return the type name of the class.
Definition: Timeline.h:334
openshot::Keyframe viewport_scale
Curve representing the scale of the viewport (0 to 100)
Definition: Timeline.h:323
void ApplyJsonDiff(std::string value)
Apply a special formatted JSON object, which represents a change to the timeline (add,...
Definition: Timeline.cpp:1117
openshot::EffectBase * GetClipEffect(const std::string &id)
Look up a clip effect by ID.
Definition: Timeline.cpp:427
void AddClip(openshot::Clip *clip)
Add an openshot::Clip to the timeline.
Definition: Timeline.cpp:356
virtual ~Timeline()
Definition: Timeline.cpp:222
std::list< openshot::EffectBase * > ClipEffects() const
Return the list of effects on all clips.
Definition: Timeline.cpp:440
void AutoMapClips(bool auto_map)
Automatically map all clips to the timeline's framerate and samplerate.
Definition: Timeline.h:278
openshot::CacheBase * GetCache() override
Get the cache object used by this reader.
Definition: Timeline.h:310
std::list< std::string > GetTrackedObjectsIds() const
Return the ID's of the tracked objects as a list of strings.
Definition: Timeline.cpp:282
std::string Json() const override
Generate JSON string of this object.
Definition: Timeline.cpp:980
int64_t GetMaxFrame()
Look up the end frame number of the latest element on the timeline.
Definition: Timeline.cpp:477
std::shared_ptr< openshot::Frame > GetFrame(int64_t requested_frame) override
Definition: Timeline.cpp:771
std::list< openshot::EffectBase * > Effects()
Return the list of effects on the timeline.
Definition: Timeline.h:304
bool AutoMapClips()
Determine if clips are automatically mapped to the timeline's framerate and samplerate.
Definition: Timeline.h:275
void ClearAllCache()
Clear all cache for this timeline instance, and all clips, mappers, and readers under it.
Definition: Timeline.cpp:1496
void ApplyMapperToClips()
Apply the timeline's framerate and samplerate to all clips.
Definition: Timeline.cpp:513
openshot::Color color
Background color of timeline canvas.
Definition: Timeline.h:328
std::string GetTrackedObjectValues(std::string id, int64_t frame_number) const
Return the trackedObject's properties as a JSON string.
Definition: Timeline.cpp:298
Timeline(int width, int height, openshot::Fraction fps, int sample_rate, int channels, openshot::ChannelLayout channel_layout)
Constructor for the timeline (which configures the default frame properties)
Definition: Timeline.cpp:46
std::shared_ptr< openshot::TrackedObjectBase > GetTrackedObject(std::string id) const
Return tracked object pointer by it's id.
Definition: Timeline.cpp:265
openshot::EffectBase * GetEffect(const std::string &id)
Look up a timeline effect by ID.
Definition: Timeline.cpp:416
void SetJsonValue(const Json::Value root) override
Load Json::Value into this object.
Definition: Timeline.cpp:1041
openshot::Clip * GetClip(const std::string &id)
Look up a single clip by ID.
Definition: Timeline.cpp:404
void AddEffect(openshot::EffectBase *effect)
Add an effect to the timeline.
Definition: Timeline.cpp:379
std::shared_ptr< openshot::Frame > apply_effects(std::shared_ptr< openshot::Frame > frame, int64_t timeline_frame_number, int layer)
Apply global/timeline effects to the source frame (if any)
Definition: Timeline.cpp:537
void SetCache(openshot::CacheBase *new_cache)
Definition: Timeline.cpp:967
std::list< openshot::Clip * > Clips()
Return a list of clips on the timeline.
Definition: Timeline.h:284
bool IsOpen() override
Determine if reader is open or closed.
Definition: Timeline.h:331
openshot::Keyframe viewport_x
Curve representing the x coordinate for the viewport.
Definition: Timeline.h:324
void RemoveClip(openshot::Clip *clip)
Remove an openshot::Clip from the timeline.
Definition: Timeline.cpp:398
void SetMaxSize(int width, int height)
Definition: Timeline.cpp:1522
double GetMaxTime()
Look up the end time of the latest timeline element.
Definition: Timeline.cpp:459
void RemoveEffect(openshot::EffectBase *effect)
Remove an effect from the timeline.
Definition: Timeline.cpp:392
void Open() override
Open the reader (and start consuming resources)
Definition: Timeline.cpp:759
void SetJson(const std::string value) override
Load JSON string into this object.
Definition: Timeline.cpp:1021
openshot::Keyframe viewport_y
Curve representing the y coordinate for the viewport.
Definition: Timeline.h:325
void Close() override
Close the timeline reader (and any resources it was consuming)
Definition: Timeline.cpp:739
This namespace is the default namespace for all code in the openshot library.
Definition: Compressor.h:47
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround,...
bool operator()(const openshot::Clip *lhs, const openshot::Clip *rhs)
Definition: Timeline.h:88
bool operator()(openshot::Clip *lhs, openshot::Clip *rhs)
Definition: Timeline.h:68
Like CompareClipEndFrames, but for effects.
Definition: Timeline.h:94
bool operator()(const openshot::EffectBase *lhs, const openshot::EffectBase *rhs)
Definition: Timeline.h:95
bool operator()(openshot::EffectBase *lhs, openshot::EffectBase *rhs)
Definition: Timeline.h:78
This struct contains info about a media file, such as height, width, frames per second,...
Definition: ReaderBase.h:61
This struct contains info about the current Timeline clip instance.
Definition: TimelineBase.h:47