Blender  V3.3
FN_multi_function_builder.hh
Go to the documentation of this file.
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 
3 #pragma once
4 
11 #include <functional>
12 
14 
15 #include "FN_multi_function.hh"
16 
17 namespace blender::fn {
18 
19 namespace devi = devirtualize_parameters;
20 
25 namespace CustomMF_presets {
26 
28 enum class FallbackMode {
30  Simple,
33 };
34 
40 struct Simple {
41  static constexpr bool use_devirtualization = false;
43 };
44 
51 struct Materialized {
52  static constexpr bool use_devirtualization = false;
54 };
55 
63  static constexpr bool use_devirtualization = true;
65 
66  template<typename Fn, typename... ParamTypes>
68  {
69  using devi::DeviMode;
70  devirtualizer.try_execute_devirtualized(
72  DeviMode::Span | DeviMode::Single | DeviMode::Range,
73  sizeof...(ParamTypes)>());
74  }
75 };
76 
82 template<size_t... Indices> struct SomeSpanOrSingle {
83  static constexpr bool use_devirtualization = true;
85 
86  template<typename Fn, typename... ParamTypes>
88  {
89  using devi::DeviMode;
90  devirtualizer.try_execute_devirtualized(
92  DeviMode::Span | DeviMode::Single | DeviMode::Range,
93  DeviMode::Single,
94  sizeof...(ParamTypes),
95  0,
96  (Indices + 1)...>());
97  }
98 };
99 
100 } // namespace CustomMF_presets
101 
102 namespace detail {
103 
109 template<typename MaskT, typename... Args, typename... ParamTags, size_t... I, typename ElementFn>
111  std::index_sequence<I...> /* indices */,
112  ElementFn element_fn,
113  MaskT mask,
114  /* Use restrict to tell the compiler that pointer inputs do not alias each
115  * other. This is important for some compiler optimizations. */
116  Args &&__restrict... args)
117 {
118  for (const int64_t i : mask) {
119  element_fn([&]() -> decltype(auto) {
120  using ParamTag = typename TypeSequence<ParamTags...>::template at_index<I>;
121  if constexpr (ParamTag::category == MFParamCategory::SingleInput) {
122  /* For inputs, pass the value (or a reference to it) to the function. */
123  return args[i];
124  }
125  else if constexpr (ParamTag::category == MFParamCategory::SingleOutput) {
126  /* For outputs, pass a pointer to the function. This is done instead of passing a
127  * reference, because the pointer points to uninitialized memory. */
128  return &args[i];
129  }
130  }()...);
131  }
132 }
133 
134 } // namespace detail
135 
136 namespace materialize_detail {
137 
138 enum class ArgMode {
139  Unknown,
140  Single,
141  Span,
142  Materialized,
143 };
144 
145 template<typename ParamTag> struct ArgInfo {
148 };
149 
153 template<typename... ParamTags, typename ElementFn, typename... Chunks>
155  const ElementFn element_fn,
156  const IndexRange in_mask,
157  const IndexMask out_mask,
158  Chunks &&__restrict... chunks)
159 {
160  BLI_assert(in_mask.size() == out_mask.size());
161  for (const int64_t i : IndexRange(in_mask.size())) {
162  const int64_t in_i = in_mask[i];
163  const int64_t out_i = out_mask[i];
164  element_fn([&]() -> decltype(auto) {
165  using ParamTag = ParamTags;
166  if constexpr (ParamTag::category == MFParamCategory::SingleInput) {
167  return chunks[in_i];
168  }
169  else if constexpr (ParamTag::category == MFParamCategory::SingleOutput) {
170  /* For outputs, a pointer is passed, because the memory is uninitialized. */
171  return &chunks[out_i];
172  }
173  }()...);
174  }
175 }
176 
182 template<typename... ParamTags, size_t... I, typename ElementFn, typename... Args>
184  std::index_sequence<I...> /* indices */,
185  const ElementFn element_fn,
186  const IndexMask mask,
187  Args &&...args)
188 {
189 
190  /* In theory, all elements could be processed in one chunk. However, that has the disadvantage
191  * that large temporary arrays are needed. Using small chunks allows using small arrays, which
192  * are reused multiple times, which improves cache efficiency. The chunk size also shouldn't be
193  * too small, because then overhead of the outer loop over chunks becomes significant again. */
194  static constexpr int64_t MaxChunkSize = 32;
195  const int64_t mask_size = mask.size();
196  const int64_t buffer_size = std::min(mask_size, MaxChunkSize);
197 
198  /* Local buffers that are used to temporarily store values retrieved from virtual arrays. */
199  std::tuple<TypedBuffer<typename ParamTags::base_type, MaxChunkSize>...> buffers_owner;
200 
201  /* A span for each parameter which is either empty or points to memory in #buffers_owner. */
202  std::tuple<MutableSpan<typename ParamTags::base_type>...> buffers;
203 
204  /* Information about every parameter. */
205  std::tuple<ArgInfo<ParamTags>...> args_info;
206 
207  (
208  /* Setup information for all parameters. */
209  [&] {
210  typedef ParamTags ParamTag;
211  typedef typename ParamTag::base_type T;
212  [[maybe_unused]] ArgInfo<ParamTags> &arg_info = std::get<I>(args_info);
213  if constexpr (ParamTag::category == MFParamCategory::SingleInput) {
214  VArray<T> &varray = *args;
215  if (varray.is_single()) {
216  /* If an input #VArray is a single value, we have to fill the buffer with that value
217  * only once. The same unchanged buffer can then be reused in every chunk. */
218  MutableSpan<T> in_chunk{std::get<I>(buffers_owner).ptr(), buffer_size};
219  const T in_single = varray.get_internal_single();
220  uninitialized_fill_n(in_chunk.data(), in_chunk.size(), in_single);
221  std::get<I>(buffers) = in_chunk;
222  arg_info.mode = ArgMode::Single;
223  }
224  else if (varray.is_span()) {
225  /* Remember the span so that it doesn't have to be retrieved in every iteration. */
226  arg_info.internal_span = varray.get_internal_span();
227  }
228  }
229  }(),
230  ...);
231 
232  /* Outer loop over all chunks. */
233  for (int64_t chunk_start = 0; chunk_start < mask_size; chunk_start += MaxChunkSize) {
234  const IndexMask sliced_mask = mask.slice(chunk_start, MaxChunkSize);
235  const int64_t chunk_size = sliced_mask.size();
236  const bool sliced_mask_is_range = sliced_mask.is_range();
237 
240  element_fn,
241  /* Inputs are "compressed" into contiguous arrays without gaps. */
243  /* Outputs are written directly into the correct place in the output arrays. */
244  sliced_mask,
245  /* Prepare every parameter for this chunk. */
246  [&] {
247  using ParamTag = ParamTags;
248  using T = typename ParamTag::base_type;
249  [[maybe_unused]] ArgInfo<ParamTags> &arg_info = std::get<I>(args_info);
250  if constexpr (ParamTag::category == MFParamCategory::SingleInput) {
251  if (arg_info.mode == ArgMode::Single) {
252  /* The single value has been filled into a buffer already reused for every chunk. */
253  return Span<T>(std::get<I>(buffers));
254  }
255  else {
256  const VArray<T> &varray = *args;
257  if (sliced_mask_is_range) {
258  if (!arg_info.internal_span.is_empty()) {
259  /* In this case we can just use an existing span instead of "compressing" it into
260  * a new temporary buffer. */
261  const IndexRange sliced_mask_range = sliced_mask.as_range();
262  arg_info.mode = ArgMode::Span;
263  return arg_info.internal_span.slice(sliced_mask_range);
264  }
265  }
266  /* As a fallback, do a virtual function call to retrieve all elements in the current
267  * chunk. The elements are stored in a temporary buffer reused for every chunk. */
268  MutableSpan<T> in_chunk{std::get<I>(buffers_owner).ptr(), chunk_size};
269  varray.materialize_compressed_to_uninitialized(sliced_mask, in_chunk);
270  /* Remember that this parameter has been materialized, so that the values are
271  * destructed properly when the chunk is done. */
272  arg_info.mode = ArgMode::Materialized;
273  return Span<T>(in_chunk);
274  }
275  }
276  else if constexpr (ParamTag::category == MFParamCategory::SingleOutput) {
277  /* For outputs, just pass a pointer. This is important so that `__restrict` works. */
278  return args->data();
279  }
280  }()...);
281 
282  (
283  /* Destruct values that have been materialized before. */
284  [&] {
285  typedef ParamTags ParamTag;
286  typedef typename ParamTag::base_type T;
287  [[maybe_unused]] ArgInfo<ParamTags> &arg_info = std::get<I>(args_info);
288  if constexpr (ParamTag::category == MFParamCategory::SingleInput) {
289  if (arg_info.mode == ArgMode::Materialized) {
290  T *in_chunk = std::get<I>(buffers_owner).ptr();
291  destruct_n(in_chunk, chunk_size);
292  }
293  }
294  }(),
295  ...);
296  }
297 
298  (
299  /* Destruct buffers for single value inputs. */
300  [&] {
301  typedef ParamTags ParamTag;
302  typedef typename ParamTag::base_type T;
303  [[maybe_unused]] ArgInfo<ParamTags> &arg_info = std::get<I>(args_info);
304  if constexpr (ParamTag::category == MFParamCategory::SingleInput) {
305  if (arg_info.mode == ArgMode::Single) {
306  MutableSpan<T> in_chunk = std::get<I>(buffers);
307  destruct_n(in_chunk.data(), in_chunk.size());
308  }
309  }
310  }(),
311  ...);
312 }
313 } // namespace materialize_detail
314 
315 template<typename... ParamTags> class CustomMF : public MultiFunction {
316  private:
317  std::function<void(IndexMask mask, MFParams params)> fn_;
318  MFSignature signature_;
319 
320  using TagsSequence = TypeSequence<ParamTags...>;
321 
322  public:
323  template<typename ElementFn, typename ExecPreset = CustomMF_presets::Materialized>
324  CustomMF(const char *name,
325  ElementFn element_fn,
326  ExecPreset exec_preset = CustomMF_presets::Materialized())
327  {
329  add_signature_parameters(signature, std::make_index_sequence<TagsSequence::size()>());
330  signature_ = signature.build();
331  this->set_signature(&signature_);
332 
333  fn_ = [element_fn, exec_preset](IndexMask mask, MFParams params) {
334  execute(
335  element_fn, exec_preset, mask, params, std::make_index_sequence<TagsSequence::size()>());
336  };
337  }
338 
339  template<typename ElementFn, typename ExecPreset, size_t... I>
340  static void execute(ElementFn element_fn,
341  ExecPreset exec_preset,
342  IndexMask mask,
344  std::index_sequence<I...> /* indices */)
345  {
346  std::tuple<typename ParamTags::array_type...> retrieved_params;
347  (
348  /* Get all parameters from #params and store them in #retrieved_params. */
349  [&]() {
350  typedef typename TagsSequence::template at_index<I> ParamTag;
351  typedef typename ParamTag::base_type T;
352 
353  if constexpr (ParamTag::category == MFParamCategory::SingleInput) {
354  std::get<I>(retrieved_params) = params.readonly_single_input<T>(I);
355  }
356  if constexpr (ParamTag::category == MFParamCategory::SingleOutput) {
357  std::get<I>(retrieved_params) = params.uninitialized_single_output<T>(I);
358  }
359  }(),
360  ...);
361 
362  auto array_executor = [&](auto &&...args) {
364  std::make_index_sequence<TagsSequence::size()>(),
365  element_fn,
366  std::forward<decltype(args)>(args)...);
367  };
368 
369  /* First try devirtualized execution, since this is the most efficient. */
370  bool executed_devirtualized = false;
371  if constexpr (ExecPreset::use_devirtualization) {
372  devi::Devirtualizer<decltype(array_executor), IndexMask, typename ParamTags::array_type...>
373  devirtualizer{
374  array_executor, &mask, [&] { return &std::get<I>(retrieved_params); }()...};
375  exec_preset.try_devirtualize(devirtualizer);
376  executed_devirtualized = devirtualizer.executed();
377  }
378 
379  /* If devirtualized execution was disabled or not possible, use a fallback method which is
380  * slower but always works. */
381  if (!executed_devirtualized) {
382  if constexpr (ExecPreset::fallback_mode == CustomMF_presets::FallbackMode::Materialized) {
384  TypeSequence<ParamTags...>(), std::index_sequence<I...>(), element_fn, mask, [&] {
385  return &std::get<I>(retrieved_params);
386  }()...);
387  }
388  else {
390  std::make_index_sequence<TagsSequence::size()>(),
391  element_fn,
392  mask,
393  std::get<I>(retrieved_params)...);
394  }
395  }
396  }
397 
398  template<size_t... I>
400  std::index_sequence<I...> /* indices */)
401  {
402  (
403  /* Loop over all parameter types and add an entry for each in the signature. */
404  [&] {
405  typedef typename TagsSequence::template at_index<I> ParamTag;
406  signature.add(ParamTag(), "");
407  }(),
408  ...);
409  }
410 
412  {
413  fn_(mask, params);
414  }
415 };
416 
425 template<typename In1, typename Out1>
426 class CustomMF_SI_SO : public CustomMF<MFParamTag<MFParamCategory::SingleInput, In1>,
427  MFParamTag<MFParamCategory::SingleOutput, Out1>> {
428  public:
429  template<typename ElementFn, typename ExecPreset = CustomMF_presets::Materialized>
430  CustomMF_SI_SO(const char *name,
431  ElementFn element_fn,
432  ExecPreset exec_preset = CustomMF_presets::Materialized())
435  name,
436  [element_fn](const In1 &in1, Out1 *out1) { new (out1) Out1(element_fn(in1)); },
437  exec_preset)
438  {
439  }
440 };
441 
448 template<typename In1, typename In2, typename Out1>
449 class CustomMF_SI_SI_SO : public CustomMF<MFParamTag<MFParamCategory::SingleInput, In1>,
450  MFParamTag<MFParamCategory::SingleInput, In2>,
451  MFParamTag<MFParamCategory::SingleOutput, Out1>> {
452  public:
453  template<typename ElementFn, typename ExecPreset = CustomMF_presets::Materialized>
454  CustomMF_SI_SI_SO(const char *name,
455  ElementFn element_fn,
456  ExecPreset exec_preset = CustomMF_presets::Materialized())
460  name,
461  [element_fn](const In1 &in1, const In2 &in2, Out1 *out1) {
462  new (out1) Out1(element_fn(in1, in2));
463  },
464  exec_preset)
465  {
466  }
467 };
468 
476 template<typename In1, typename In2, typename In3, typename Out1>
477 class CustomMF_SI_SI_SI_SO : public CustomMF<MFParamTag<MFParamCategory::SingleInput, In1>,
478  MFParamTag<MFParamCategory::SingleInput, In2>,
479  MFParamTag<MFParamCategory::SingleInput, In3>,
480  MFParamTag<MFParamCategory::SingleOutput, Out1>> {
481  public:
482  template<typename ElementFn, typename ExecPreset = CustomMF_presets::Materialized>
484  ElementFn element_fn,
485  ExecPreset exec_preset = CustomMF_presets::Materialized())
490  name,
491  [element_fn](const In1 &in1, const In2 &in2, const In3 &in3, Out1 *out1) {
492  new (out1) Out1(element_fn(in1, in2, in3));
493  },
494  exec_preset)
495  {
496  }
497 };
498 
507 template<typename In1, typename In2, typename In3, typename In4, typename Out1>
508 class CustomMF_SI_SI_SI_SI_SO : public CustomMF<MFParamTag<MFParamCategory::SingleInput, In1>,
509  MFParamTag<MFParamCategory::SingleInput, In2>,
510  MFParamTag<MFParamCategory::SingleInput, In3>,
511  MFParamTag<MFParamCategory::SingleInput, In4>,
512  MFParamTag<MFParamCategory::SingleOutput, Out1>> {
513  public:
514  template<typename ElementFn, typename ExecPreset = CustomMF_presets::Materialized>
516  ElementFn element_fn,
517  ExecPreset exec_preset = CustomMF_presets::Materialized())
523  name,
524  [element_fn](
525  const In1 &in1, const In2 &in2, const In3 &in3, const In4 &in4, Out1 *out1) {
526  new (out1) Out1(element_fn(in1, in2, in3, in4));
527  },
528  exec_preset)
529  {
530  }
531 };
532 
537 template<typename Mut1> class CustomMF_SM : public MultiFunction {
538  private:
539  using FunctionT = std::function<void(IndexMask, MutableSpan<Mut1>)>;
540  FunctionT function_;
541  MFSignature signature_;
542 
543  public:
544  CustomMF_SM(const char *name, FunctionT function) : function_(std::move(function))
545  {
547  signature.single_mutable<Mut1>("Mut1");
548  signature_ = signature.build();
549  this->set_signature(&signature_);
550  }
551 
552  template<typename ElementFuncT>
553  CustomMF_SM(const char *name, ElementFuncT element_fn)
555  {
556  }
557 
558  template<typename ElementFuncT> static FunctionT create_function(ElementFuncT element_fn)
559  {
560  return [=](IndexMask mask, MutableSpan<Mut1> mut1) {
561  mask.to_best_mask_type([&](const auto &mask) {
562  for (const int64_t i : mask) {
563  element_fn(mut1[i]);
564  }
565  });
566  };
567  }
568 
570  {
571  MutableSpan<Mut1> mut1 = params.single_mutable<Mut1>(0);
572  function_(mask, mut1);
573  }
574 };
575 
582  private:
583  const CPPType &type_;
584  const void *value_;
585  MFSignature signature_;
586  bool owns_value_;
587 
588  template<typename T> friend class CustomMF_Constant;
589 
590  public:
591  CustomMF_GenericConstant(const CPPType &type, const void *value, bool make_value_copy);
593  void call(IndexMask mask, MFParams params, MFContext context) const override;
594  uint64_t hash() const override;
595  bool equals(const MultiFunction &other) const override;
596 };
597 
603  private:
604  GSpan array_;
605  MFSignature signature_;
606 
607  public:
609  void call(IndexMask mask, MFParams params, MFContext context) const override;
610 };
611 
615 template<typename T> class CustomMF_Constant : public MultiFunction {
616  private:
617  T value_;
618  MFSignature signature_;
619 
620  public:
621  template<typename U> CustomMF_Constant(U &&value) : value_(std::forward<U>(value))
622  {
623  MFSignatureBuilder signature{"Constant"};
624  signature.single_output<T>("Value");
625  signature_ = signature.build();
626  this->set_signature(&signature_);
627  }
628 
630  {
631  MutableSpan<T> output = params.uninitialized_single_output<T>(0);
632  mask.to_best_mask_type([&](const auto &mask) {
633  for (const int64_t i : mask) {
634  new (&output[i]) T(value_);
635  }
636  });
637  }
638 
639  uint64_t hash() const override
640  {
641  return get_default_hash(value_);
642  }
643 
644  bool equals(const MultiFunction &other) const override
645  {
646  const CustomMF_Constant *other1 = dynamic_cast<const CustomMF_Constant *>(&other);
647  if (other1 != nullptr) {
648  return value_ == other1->value_;
649  }
650  const CustomMF_GenericConstant *other2 = dynamic_cast<const CustomMF_GenericConstant *>(
651  &other);
652  if (other2 != nullptr) {
653  const CPPType &type = CPPType::get<T>();
654  if (type == other2->type_) {
655  return type.is_equal_or_false(static_cast<const void *>(&value_), other2->value_);
656  }
657  }
658  return false;
659  }
660 };
661 
663  private:
664  int output_amount_;
665  MFSignature signature_;
666 
667  public:
668  CustomMF_DefaultOutput(Span<MFDataType> input_types, Span<MFDataType> output_types);
669  void call(IndexMask mask, MFParams params, MFContext context) const override;
670 };
671 
673  private:
674  MFSignature signature_;
675 
676  public:
677  CustomMF_GenericCopy(MFDataType data_type);
678  void call(IndexMask mask, MFParams params, MFContext context) const override;
679 };
680 
681 } // namespace blender::fn
#define BLI_assert(a)
Definition: BLI_assert.h:46
#define UNUSED(x)
_GL_VOID GLfloat value _GL_VOID_RET _GL_VOID const GLuint GLboolean *residences _GL_BOOL_RET _GL_VOID GLsizei GLfloat GLfloat GLfloat GLfloat const GLubyte *bitmap _GL_VOID_RET _GL_VOID GLenum type
unsigned int U
Definition: btGjkEpa3.h:78
int64_t size() const
bool is_range() const
IndexRange as_range() const
constexpr int64_t size() const
constexpr int64_t size() const
Definition: BLI_span.hh:511
constexpr T * data() const
Definition: BLI_span.hh:548
constexpr Span slice(int64_t start, int64_t size) const
Definition: BLI_span.hh:142
constexpr bool is_empty() const
Definition: BLI_span.hh:248
void materialize_compressed_to_uninitialized(IndexMask mask, MutableSpan< T > r_span) const
Span< T > get_internal_span() const
void try_execute_devirtualized(DeviModeSequence< AllowedModes... >)
void call(IndexMask mask, MFParams params, MFContext UNUSED(context)) const override
bool equals(const MultiFunction &other) const override
void call(IndexMask mask, MFParams params, MFContext context) const override
CustomMF_DefaultOutput(Span< MFDataType > input_types, Span< MFDataType > output_types)
void call(IndexMask mask, MFParams params, MFContext context) const override
CustomMF_GenericConstant(const CPPType &type, const void *value, bool make_value_copy)
void call(IndexMask mask, MFParams params, MFContext context) const override
bool equals(const MultiFunction &other) const override
void call(IndexMask mask, MFParams params, MFContext context) const override
CustomMF_SI_SI_SI_SI_SO(const char *name, ElementFn element_fn, ExecPreset exec_preset=CustomMF_presets::Materialized())
CustomMF_SI_SI_SI_SO(const char *name, ElementFn element_fn, ExecPreset exec_preset=CustomMF_presets::Materialized())
CustomMF_SI_SI_SO(const char *name, ElementFn element_fn, ExecPreset exec_preset=CustomMF_presets::Materialized())
CustomMF_SI_SO(const char *name, ElementFn element_fn, ExecPreset exec_preset=CustomMF_presets::Materialized())
void call(IndexMask mask, MFParams params, MFContext UNUSED(context)) const override
CustomMF_SM(const char *name, ElementFuncT element_fn)
static FunctionT create_function(ElementFuncT element_fn)
CustomMF_SM(const char *name, FunctionT function)
void call(IndexMask mask, MFParams params, MFContext UNUSED(context)) const override
CustomMF(const char *name, ElementFn element_fn, ExecPreset exec_preset=CustomMF_presets::Materialized())
static void execute(ElementFn element_fn, ExecPreset exec_preset, IndexMask mask, MFParams params, std::index_sequence< I... >)
static void add_signature_parameters(MFSignatureBuilder &signature, std::index_sequence< I... >)
StringRefNull name() const
void set_signature(const MFSignature *signature)
const MFSignature & signature() const
SyclQueue void void size_t num_bytes void
uiWidgetBaseParameters params[MAX_WIDGET_BASE_BATCH]
ccl_global KernelShaderEvalInput ccl_global float * output
ccl_device_inline float4 mask(const int4 &mask, const float4 &a)
Definition: math_float4.h:513
#define T
void execute_array(TypeSequence< ParamTags... >, std::index_sequence< I... >, ElementFn element_fn, MaskT mask, Args &&__restrict... args)
void execute_materialized_impl(TypeSequence< ParamTags... >, const ElementFn element_fn, const IndexRange in_mask, const IndexMask out_mask, Chunks &&__restrict... chunks)
void execute_materialized(TypeSequence< ParamTags... >, std::index_sequence< I... >, const ElementFn element_fn, const IndexMask mask, Args &&...args)
static const int chunk_size
decltype(detail::make_value_sequence_impl< T, Value >(std::make_index_sequence< Size >())) make_value_sequence
decltype(detail::make_two_value_sequence_impl< T, Value1, Value2 >(ValueSequence< size_t, Value1Indices... >(), std::make_index_sequence< Size >())) make_two_value_sequence
uint64_t get_default_hash(const T &v)
Definition: BLI_hash.hh:218
void uninitialized_fill_n(T *dst, int64_t n, const T &value)
void destruct_n(T *ptr, int64_t n)
#define I
#define min(a, b)
Definition: sort.c:35
__int64 int64_t
Definition: stdint.h:89
unsigned __int64 uint64_t
Definition: stdint.h:90
static constexpr size_t size() noexcept
void try_devirtualize(devi::Devirtualizer< Fn, ParamTypes... > &devirtualizer)
void try_devirtualize(devi::Devirtualizer< Fn, ParamTypes... > &devirtualizer)
Span< typename ParamTag::base_type > internal_span