Blender  V3.3
tracking_region_tracker.c
Go to the documentation of this file.
1 /* SPDX-License-Identifier: GPL-2.0-or-later
2  * Copyright 2011 Blender Foundation. All rights reserved. */
3 
11 #include "MEM_guardedalloc.h"
12 
13 #include "DNA_defaults.h"
14 #include "DNA_movieclip_types.h"
15 
16 #include "BLI_threads.h"
17 #include "BLI_utildefines.h"
18 
19 #include "BKE_movieclip.h"
20 #include "BKE_tracking.h"
21 
22 #include "IMB_imbuf.h"
23 #include "IMB_imbuf_types.h"
24 
25 #include "libmv-capi.h"
26 #include "tracking_private.h"
27 
28 /* **** utility functions for tracking **** */
29 
31 static void float_rgba_to_gray(const float *rgba,
32  float *gray,
33  int num_pixels,
34  float weight_red,
35  float weight_green,
36  float weight_blue)
37 {
38  for (int i = 0; i < num_pixels; i++) {
39  const float *pixel = rgba + 4 * i;
40 
41  gray[i] = weight_red * pixel[0] + weight_green * pixel[1] + weight_blue * pixel[2];
42  }
43 }
44 
45 static void uint8_rgba_to_float_gray(const unsigned char *rgba,
46  float *gray,
47  int num_pixels,
48  float weight_red,
49  float weight_green,
50  float weight_blue)
51 {
52  for (int i = 0; i < num_pixels; i++) {
53  const unsigned char *pixel = rgba + i * 4;
54 
55  gray[i] = (weight_red * pixel[0] + weight_green * pixel[1] + weight_blue * pixel[2]) / 255.0f;
56  }
57 }
58 
60 static float *track_get_search_floatbuf(ImBuf *ibuf,
61  MovieTrackingTrack *track,
62  MovieTrackingMarker *marker,
63  int *r_width,
64  int *r_height)
65 {
66  ImBuf *searchibuf;
67  float *gray_pixels;
68  int width, height;
69 
70  searchibuf = BKE_tracking_get_search_imbuf(ibuf, track, marker, false, true);
71 
72  if (!searchibuf) {
73  *r_width = 0;
74  *r_height = 0;
75  return NULL;
76  }
77 
78  width = searchibuf->x;
79  height = searchibuf->y;
80 
81  gray_pixels = MEM_callocN(width * height * sizeof(float), "tracking floatBuf");
82 
83  if (searchibuf->rect_float) {
85  searchibuf->rect_float, gray_pixels, width * height, 0.2126f, 0.7152f, 0.0722f);
86  }
87  else {
89  (unsigned char *)searchibuf->rect, gray_pixels, width * height, 0.2126f, 0.7152f, 0.0722f);
90  }
91 
92  IMB_freeImBuf(searchibuf);
93 
94  *r_width = width;
95  *r_height = height;
96 
97  return gray_pixels;
98 }
99 
100 /* Get image buffer for a given frame
101  *
102  * Frame is in clip space.
103  */
105  MovieClipUser *user,
106  int clip_flag,
107  int framenr)
108 {
109  ImBuf *ibuf;
110  MovieClipUser new_user = *user;
111 
112  new_user.framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, framenr);
113 
114  ibuf = BKE_movieclip_get_ibuf_flag(clip, &new_user, clip_flag, MOVIECLIP_CACHE_SKIP);
115 
116  return ibuf;
117 }
118 
119 /* Get image buffer for previous marker's keyframe. */
121  MovieClipUser *user,
122  int clip_flag,
123  MovieTrackingTrack *track,
124  int curfra,
125  bool backwards,
126  MovieTrackingMarker **r_marker_keyed)
127 {
128  MovieTrackingMarker *marker_keyed;
129  int keyed_framenr;
130 
131  marker_keyed = tracking_get_keyframed_marker(track, curfra, backwards);
132  if (marker_keyed == NULL) {
133  return NULL;
134  }
135 
136  keyed_framenr = marker_keyed->framenr;
137 
138  *r_marker_keyed = marker_keyed;
139 
140  return tracking_context_get_frame_ibuf(clip, user, clip_flag, keyed_framenr);
141 }
142 
143 /* Get image buffer which is used as reference for track. */
145  MovieClipUser *user,
146  int clip_flag,
147  MovieTrackingTrack *track,
148  int curfra,
149  bool backwards,
150  MovieTrackingMarker **reference_marker)
151 {
152  ImBuf *ibuf = NULL;
153 
154  if (track->pattern_match == TRACK_MATCH_KEYFRAME) {
156  clip, user, clip_flag, track, curfra, backwards, reference_marker);
157  }
158  else {
159  ibuf = tracking_context_get_frame_ibuf(clip, user, clip_flag, curfra);
160 
161  /* use current marker as keyframed position */
162  *reference_marker = BKE_tracking_marker_get(track, curfra);
163  }
164 
165  return ibuf;
166 }
167 
169  float *mask,
170  const bool is_backwards,
172 {
173  options->direction = is_backwards ? LIBMV_TRACK_REGION_BACKWARD : LIBMV_TRACK_REGION_FORWARD;
174 
175  /* TODO(sergey): Use explicit conversion, so that options are decoupled between the Libmv library
176  * and enumerator values in DNA. */
177  options->motion_model = track->motion_model;
178 
179  options->use_brute = ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_BRUTE) != 0);
180 
181  options->use_normalization = ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_NORMALIZATION) !=
182  0);
183 
184  options->num_iterations = 50;
185  options->minimum_correlation = track->minimum_correlation;
186  options->sigma = 0.9;
187 
188  if ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_MASK) != 0) {
189  options->image1_mask = mask;
190  }
191  else {
192  options->image1_mask = NULL;
193  }
194 }
195 
196 /* Perform tracking from a reference_marker to destination_ibuf.
197  * Uses marker as an initial position guess.
198  *
199  * Returns truth if tracker returned success, puts result
200  * to dst_pixel_x and dst_pixel_y.
201  */
202 static bool configure_and_run_tracker(ImBuf *destination_ibuf,
203  MovieTrackingTrack *track,
204  MovieTrackingMarker *reference_marker,
205  MovieTrackingMarker *marker,
206  float *reference_search_area,
207  int reference_search_area_width,
208  int reference_search_area_height,
209  float *mask,
210  const bool is_backward,
211  double dst_pixel_x[5],
212  double dst_pixel_y[5])
213 {
214  /* To convert to the x/y split array format for libmv. */
215  double src_pixel_x[5], src_pixel_y[5];
216 
217  /* Settings for the tracker */
220 
221  float *patch_new;
222 
223  int new_search_area_width, new_search_area_height;
224  int frame_width, frame_height;
225 
226  bool tracked;
227 
228  frame_width = destination_ibuf->x;
229  frame_height = destination_ibuf->y;
230 
231  /* for now track to the same search area dimension as marker has got for current frame
232  * will make all tracked markers in currently tracked segment have the same search area
233  * size, but it's quite close to what is actually needed
234  */
235  patch_new = track_get_search_floatbuf(
236  destination_ibuf, track, marker, &new_search_area_width, &new_search_area_height);
237 
238  /* configure the tracker */
239  tracking_configure_tracker(track, mask, is_backward, &options);
240 
241  /* Convert the marker corners and center into pixel coordinates in the
242  * search/destination images. */
244  frame_width, frame_height, reference_marker, src_pixel_x, src_pixel_y);
246  frame_width, frame_height, marker, dst_pixel_x, dst_pixel_y);
247 
248  if (patch_new == NULL || reference_search_area == NULL) {
249  return false;
250  }
251 
252  /* run the tracker! */
253  tracked = libmv_trackRegion(&options,
254  reference_search_area,
255  reference_search_area_width,
256  reference_search_area_height,
257  patch_new,
258  new_search_area_width,
259  new_search_area_height,
260  src_pixel_x,
261  src_pixel_y,
262  &result,
263  dst_pixel_x,
264  dst_pixel_y);
265 
266  MEM_freeN(patch_new);
267 
268  return tracked;
269 }
270 
272  MovieTrackingMarker *marker,
273  bool backwards,
274  int *reference_framenr)
275 {
276  const MovieTrackingMarker *first_marker = track->markers;
277  const MovieTrackingMarker *last_marker = track->markers + track->markersnr - 1;
278  MovieTrackingMarker *reference = backwards ? marker + 1 : marker - 1;
279 
280  while (reference >= first_marker && reference <= last_marker &&
281  (reference->flag & MARKER_DISABLED) != 0) {
282  if (backwards) {
283  reference++;
284  }
285  else {
286  reference--;
287  }
288  }
289 
290  if (reference < first_marker || reference > last_marker) {
291  return false;
292  }
293 
294  *reference_framenr = reference->framenr;
295  return (reference->flag & MARKER_DISABLED) == 0;
296 }
297 
299  MovieTrackingTrack *track,
300  MovieTrackingMarker *marker,
301  bool backwards)
302 {
303  MovieTrackingMarker *reference_marker = NULL;
304  ImBuf *reference_ibuf, *destination_ibuf;
305  float *search_area, *mask = NULL;
306  int frame_width, frame_height;
307  int search_area_height, search_area_width;
308  int clip_flag = clip->flag & MCLIP_TIMECODE_FLAGS;
309  int reference_framenr;
311  double dst_pixel_x[5], dst_pixel_y[5];
312  bool tracked;
313 
314  /* Construct a temporary clip used, used to acquire image buffers. */
316 
317  BKE_movieclip_get_size(clip, &user, &frame_width, &frame_height);
318 
319  /* Get an image buffer for reference frame, also gets reference marker. */
320  if (!refine_marker_reference_frame_get(track, marker, backwards, &reference_framenr)) {
321  return;
322  }
323 
324  reference_ibuf = tracking_context_get_reference_ibuf(
325  clip, &user, clip_flag, track, reference_framenr, backwards, &reference_marker);
326  if (reference_ibuf == NULL) {
327  return;
328  }
329 
330  /* Could not refine with self. */
331  if (reference_marker == marker) {
332  return;
333  }
334 
335  /* Destination image buffer has got frame number corresponding to refining marker. */
336  destination_ibuf = BKE_movieclip_get_ibuf_flag(clip, &user, clip_flag, MOVIECLIP_CACHE_SKIP);
337  if (destination_ibuf == NULL) {
338  IMB_freeImBuf(reference_ibuf);
339  return;
340  }
341 
342  /* Get search area from reference image. */
343  search_area = track_get_search_floatbuf(
344  reference_ibuf, track, reference_marker, &search_area_width, &search_area_height);
345 
346  /* If needed, compute track's mask. */
347  if ((track->algorithm_flag & TRACK_ALGORITHM_FLAG_USE_MASK) != 0) {
348  mask = BKE_tracking_track_get_mask(frame_width, frame_height, track, marker);
349  }
350 
351  /* Run the tracker from reference frame to current one. */
352  tracked = configure_and_run_tracker(destination_ibuf,
353  track,
354  reference_marker,
355  marker,
356  search_area,
357  search_area_width,
358  search_area_height,
359  mask,
360  backwards,
361  dst_pixel_x,
362  dst_pixel_y);
363 
364  /* Refine current marker's position if track was successful. */
365  if (tracked) {
367  frame_width, frame_height, marker, dst_pixel_x, dst_pixel_y);
368  marker->flag |= MARKER_TRACKED;
369  }
370 
371  /* Free memory used for refining */
372  MEM_freeN(search_area);
373  if (mask) {
374  MEM_freeN(mask);
375  }
376  IMB_freeImBuf(reference_ibuf);
377  IMB_freeImBuf(destination_ibuf);
378 }
#define MOVIECLIP_CACHE_SKIP
struct ImBuf * BKE_movieclip_get_ibuf_flag(struct MovieClip *clip, struct MovieClipUser *user, int flag, int cache_flag)
Definition: movieclip.c:1334
void BKE_movieclip_get_size(struct MovieClip *clip, struct MovieClipUser *user, int *width, int *height)
Definition: movieclip.c:1520
float BKE_movieclip_remap_clip_to_scene_frame(const struct MovieClip *clip, float framenr)
float * BKE_tracking_track_get_mask(int frame_width, int frame_height, struct MovieTrackingTrack *track, struct MovieTrackingMarker *marker)
Definition: tracking.c:1221
struct ImBuf * BKE_tracking_get_search_imbuf(struct ImBuf *ibuf, struct MovieTrackingTrack *track, struct MovieTrackingMarker *marker, bool anchored, bool disable_channels)
Definition: tracking.c:2760
struct MovieTrackingMarker * BKE_tracking_marker_get(struct MovieTrackingTrack *track, int framenr)
Definition: tracking.c:1424
#define DNA_struct_default_get(struct_name)
Definition: DNA_defaults.h:29
@ MCLIP_TIMECODE_FLAGS
@ TRACK_ALGORITHM_FLAG_USE_NORMALIZATION
@ TRACK_ALGORITHM_FLAG_USE_BRUTE
@ TRACK_ALGORITHM_FLAG_USE_MASK
@ MARKER_TRACKED
@ MARKER_DISABLED
@ TRACK_MATCH_KEYFRAME
_GL_VOID GLfloat value _GL_VOID_RET _GL_VOID const GLuint GLboolean *residences _GL_BOOL_RET _GL_VOID GLsizei height
_GL_VOID GLfloat value _GL_VOID_RET _GL_VOID const GLuint GLboolean *residences _GL_BOOL_RET _GL_VOID GLsizei GLfloat GLfloat GLfloat GLfloat const GLubyte *bitmap _GL_VOID_RET _GL_VOID GLenum const void *lists _GL_VOID_RET _GL_VOID const GLdouble *equation _GL_VOID_RET _GL_VOID GLdouble GLdouble blue _GL_VOID_RET _GL_VOID GLfloat GLfloat blue _GL_VOID_RET _GL_VOID GLint GLint blue _GL_VOID_RET _GL_VOID GLshort GLshort blue _GL_VOID_RET _GL_VOID GLubyte GLubyte blue _GL_VOID_RET _GL_VOID GLuint GLuint blue _GL_VOID_RET _GL_VOID GLushort GLushort blue _GL_VOID_RET _GL_VOID GLbyte GLbyte GLbyte alpha _GL_VOID_RET _GL_VOID GLdouble GLdouble GLdouble alpha _GL_VOID_RET _GL_VOID GLfloat GLfloat GLfloat alpha _GL_VOID_RET _GL_VOID GLint GLint GLint alpha _GL_VOID_RET _GL_VOID GLshort GLshort GLshort alpha _GL_VOID_RET _GL_VOID GLubyte GLubyte GLubyte alpha _GL_VOID_RET _GL_VOID GLuint GLuint GLuint alpha _GL_VOID_RET _GL_VOID GLushort GLushort GLushort alpha _GL_VOID_RET _GL_VOID GLenum mode _GL_VOID_RET _GL_VOID GLint GLsizei width
Contains defines and structs used throughout the imbuf module.
Read Guarded memory(de)allocation.
CCL_NAMESPACE_BEGIN struct Options options
void IMB_freeImBuf(ImBuf *UNUSED(ibuf))
@ LIBMV_TRACK_REGION_FORWARD
@ LIBMV_TRACK_REGION_BACKWARD
ccl_gpu_kernel_postfix ccl_global float int num_pixels
void(* MEM_freeN)(void *vmemh)
Definition: mallocn.c:27
void *(* MEM_callocN)(size_t len, const char *str)
Definition: mallocn.c:31
ccl_device_inline float4 mask(const int4 &mask, const float4 &a)
Definition: math_float4.h:513
static const pxr::TfToken rgba("rgba", pxr::TfToken::Immortal)
unsigned int * rect
float * rect_float
MovieTrackingMarker * markers
int libmv_trackRegion(const libmv_TrackRegionOptions *, const float *, int, int, const float *, int, int, const double *x1, const double *y1, libmv_TrackRegionResult *result, double *x2, double *y2)
Definition: stub.cc:23
void BKE_tracking_refine_marker(MovieClip *clip, MovieTrackingTrack *track, MovieTrackingMarker *marker, bool backwards)
static float * track_get_search_floatbuf(ImBuf *ibuf, MovieTrackingTrack *track, MovieTrackingMarker *marker, int *r_width, int *r_height)
static ImBuf * tracking_context_get_frame_ibuf(MovieClip *clip, MovieClipUser *user, int clip_flag, int framenr)
static bool refine_marker_reference_frame_get(MovieTrackingTrack *track, MovieTrackingMarker *marker, bool backwards, int *reference_framenr)
static ImBuf * tracking_context_get_keyframed_ibuf(MovieClip *clip, MovieClipUser *user, int clip_flag, MovieTrackingTrack *track, int curfra, bool backwards, MovieTrackingMarker **r_marker_keyed)
static void uint8_rgba_to_float_gray(const unsigned char *rgba, float *gray, int num_pixels, float weight_red, float weight_green, float weight_blue)
static void float_rgba_to_gray(const float *rgba, float *gray, int num_pixels, float weight_red, float weight_green, float weight_blue)
static ImBuf * tracking_context_get_reference_ibuf(MovieClip *clip, MovieClipUser *user, int clip_flag, MovieTrackingTrack *track, int curfra, bool backwards, MovieTrackingMarker **reference_marker)
static bool configure_and_run_tracker(ImBuf *destination_ibuf, MovieTrackingTrack *track, MovieTrackingMarker *reference_marker, MovieTrackingMarker *marker, float *reference_search_area, int reference_search_area_width, int reference_search_area_height, float *mask, const bool is_backward, double dst_pixel_x[5], double dst_pixel_y[5])
void tracking_configure_tracker(const MovieTrackingTrack *track, float *mask, const bool is_backwards, libmv_TrackRegionOptions *options)
void tracking_get_marker_coords_for_tracking(int frame_width, int frame_height, const MovieTrackingMarker *marker, double search_pixel_x[5], double search_pixel_y[5])
MovieTrackingMarker * tracking_get_keyframed_marker(MovieTrackingTrack *track, int current_frame, bool backwards)
void tracking_set_marker_coords_from_tracking(int frame_width, int frame_height, MovieTrackingMarker *marker, const double search_pixel_x[5], const double search_pixel_y[5])