21 if (!ensure_initialised_) {
23 ensure_initialised_ =
true;
26 #if MTL_DEBUG_MEMORY_STATISTICS == 1
28 per_frame_allocation_count_ = 0;
29 allocations_in_pool_ = 0;
44 void MTLBufferPool::free()
47 for (
auto buffer : allocations_) {
53 for (std::multiset<blender::gpu::MTLBufferHandle, blender::gpu::CompareMTLBuffer> *buffer_pool :
57 buffer_pools_.
clear();
88 options = ([device_ hasUnifiedMemory]) ? MTLResourceStorageModeShared :
89 MTLResourceStorageModeManaged;
92 options = MTLResourceStorageModePrivate;
97 std::multiset<MTLBufferHandle, CompareMTLBuffer> **pool_search = buffer_pools_.
lookup_ptr(
100 if (pool_search !=
nullptr) {
101 std::multiset<MTLBufferHandle, CompareMTLBuffer> *
pool = *pool_search;
103 auto result =
pool->lower_bound(size_compare);
112 if (found_size >= aligned_alloc_size &&
113 found_size <= (aligned_alloc_size * mtl_buffer_size_threshold_factor_)) {
115 "[MemoryAllocator] Suitable Buffer of size %lld found, for requested size: %lld\n",
119 new_buffer = found_buffer;
127 "[MemoryAllocator] Buffer of size %lld found, but was incompatible with requested "
132 new_buffer =
nullptr;
138 if (new_buffer ==
nullptr) {
142 allocations_.append(new_buffer);
143 total_allocation_bytes_ += aligned_alloc_size;
149 #if MTL_DEBUG_MEMORY_STATISTICS == 1
151 allocations_in_pool_ -= new_buffer->
get_size();
162 #if MTL_DEBUG_MEMORY_STATISTICS == 1
163 this->per_frame_allocation_count++;
189 bool buffer_in_use =
buffer->get_in_use();
199 buffer->flag_in_use(
false);
211 safelist_lock_.lock();
213 #if MTL_DEBUG_MEMORY_STATISTICS == 1
214 int num_buffers_added = 0;
218 for (
int safe_pool_free_index = 0; safe_pool_free_index < completed_safelist_queue_.size();
219 safe_pool_free_index++) {
220 MTLSafeFreeList *current_pool = completed_safelist_queue_[safe_pool_free_index];
223 while (current_pool !=
nullptr) {
224 current_pool->lock_.lock();
228 int size =
min_ii(current_pool->current_list_index_, MTLSafeFreeList::MAX_NUM_BUFFERS_);
240 #if MTL_DEBUG_MEMORY_STATISTICS == 1
247 if (current_pool->has_next_pool_ > 0) {
248 next_list = current_pool->next_.load();
252 current_pool->lock_.unlock();
254 current_pool =
nullptr;
257 if (next_list !=
nullptr) {
258 current_pool = next_list;
263 #if MTL_DEBUG_MEMORY_STATISTICS == 1
264 printf(
"--- Allocation Stats ---\n");
265 printf(
" Num buffers processed in pool (this frame): %u\n", num_buffers_added);
267 uint framealloc = (
uint)this->per_frame_allocation_count;
268 printf(
" Allocations in frame: %u\n", framealloc);
269 printf(
" Total Buffers allocated: %u\n", (
uint)allocations_.size());
270 printf(
" Total Memory allocated: %u MB\n", (
uint)total_allocation_bytes_ / (1024 * 1024));
272 uint allocs = (
uint)(allocations_in_pool_) / 1024 / 2024;
273 printf(
" Free memory in pools: %u MB\n", allocs);
275 uint buffs = (
uint)buffers_in_pool_;
276 printf(
" Buffers in pools: %u\n", buffs);
278 printf(
" Pools %u:\n", (
uint)buffer_pools_.
size());
279 auto key_iterator = buffer_pools_.
keys().begin();
280 auto value_iterator = buffer_pools_.
values().begin();
281 while (key_iterator != buffer_pools_.
keys().end()) {
284 for (
auto it = (*value_iterator)->begin(); it != (*value_iterator)->end(); it++) {
285 mem_in_pool += it->buffer_size;
289 printf(
" Buffers in pool (%u)(%llu): %u (%u MB)\n",
292 (
uint)((*value_iterator)->size()),
293 (
uint)mem_in_pool / 1024 / 1024);
298 this->per_frame_allocation_count = 0;
302 completed_safelist_queue_.clear();
303 safelist_lock_.unlock();
311 safe_list->lock_.lock();
313 BLI_assert(safe_list->reference_count_ == 0 &&
314 "Pool must be fully dereferenced by all in-use cmd buffers before returning.\n");
315 BLI_assert(safe_list->in_free_queue_ ==
false &&
"Pool must not already be in queue");
319 safelist_lock_.lock();
320 completed_safelist_queue_.append(safe_list);
321 safelist_lock_.unlock();
322 safe_list->lock_.unlock();
328 return current_free_list_;
333 safelist_lock_.lock();
335 safelist_lock_.unlock();
338 void MTLBufferPool::ensure_buffer_pool(MTLResourceOptions
options)
340 std::multiset<MTLBufferHandle, CompareMTLBuffer> **pool_search = buffer_pools_.
lookup_ptr(
342 if (pool_search ==
nullptr) {
343 std::multiset<MTLBufferHandle, CompareMTLBuffer> *
pool =
344 new std::multiset<MTLBufferHandle, CompareMTLBuffer>();
358 this->ensure_buffer_pool(
options);
368 std::multiset<MTLBufferHandle, CompareMTLBuffer> *
pool = buffer_pools_.
lookup(
options);
371 #if MTL_DEBUG_MEMORY_STATISTICS == 1
373 allocations_in_pool_ +=
buffer->get_size();
380 reference_count_ = 1;
381 in_free_queue_ =
false;
382 current_list_index_ = 0;
392 uint insert_index = current_list_index_++;
396 if (insert_index >= MTLSafeFreeList::MAX_NUM_BUFFERS_) {
399 int has_next = has_next_pool_++;
408 current_list_index_ = MTLSafeFreeList::MAX_NUM_BUFFERS_;
412 safe_free_pool_[insert_index] =
buffer;
430 int ref_count = --reference_count_;
432 if (ref_count == 0) {
454 alignment_ = alignment;
455 device_ = mtl_device;
456 is_external_ =
false;
461 metal_buffer_ = [device_ newBufferWithLength:aligned_alloc_size
options:
options];
463 [metal_buffer_ retain];
465 size_ = aligned_alloc_size;
467 if (!(options_ & MTLResourceStorageModePrivate)) {
468 data_ = [metal_buffer_ contents];
480 metal_buffer_ = external_buffer;
481 [metal_buffer_ retain];
487 options_ = [metal_buffer_ resourceOptions];
488 size_ = [metal_buffer_ allocatedSize];
490 data_ = [metal_buffer_ contents];
496 if (metal_buffer_ != nil) {
497 [metal_buffer_ release];
508 if (metal_buffer_ != nil) {
509 [metal_buffer_ release];
517 return metal_buffer_;
522 BLI_assert(!(options_ & MTLResourceStorageModePrivate));
540 return options_ & MTLResourceStorageModeManaged;
545 metal_buffer_.label =
str;
552 "Buffer should be marked as 'in-use' if being actively used by an instance. Buffer "
553 "has likely already been freed.");
558 this->debug_ensure_used();
559 if (this->requires_flush()) {
560 [metal_buffer_ didModifyRange:NSMakeRange(0, size_)];
566 this->debug_ensure_used();
567 if (this->requires_flush()) {
569 [metal_buffer_ didModifyRange:NSMakeRange(
offset,
length)];
585 BLI_assert(size_used > 0 && size_used <= size_);
586 usage_size_ = size_used;
602 return this->options & MTLResourceStorageModeManaged;
630 if (!this->initialised_) {
634 for (
int sb = 0; sb < mtl_max_scratch_buffers_; sb++) {
638 BLI_assert(&(scratch_buffers_[sb]->own_context_) == &context_);
640 current_scratch_buffer_ = 0;
647 initialised_ =
false;
650 for (
int sb = 0; sb < mtl_max_scratch_buffers_; sb++) {
651 delete scratch_buffers_[sb];
652 scratch_buffers_[sb] =
nullptr;
654 current_scratch_buffer_ = 0;
666 alignment =
max_uu(alignment, 256);
668 BLI_assert(current_scratch_buffer_ >= 0 &&
"Scratch Buffer index not set");
669 MTLCircularBuffer *current_scratch_buff = this->scratch_buffers_[current_scratch_buffer_];
670 BLI_assert(current_scratch_buff !=
nullptr &&
"Scratch Buffer does not exist");
673 BLI_assert(allocated_range.
size >= alloc_size && allocated_range.
size <= alloc_size + alignment);
675 return allocated_range;
681 MTLCircularBuffer *active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
682 BLI_assert(&active_scratch_buf->own_context_ == &context_);
689 current_scratch_buffer_ = (current_scratch_buffer_ + 1) % mtl_max_scratch_buffers_;
690 active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
691 active_scratch_buf->
reset();
692 BLI_assert(&active_scratch_buf->own_context_ == &context_);
693 MTL_LOG_INFO(
"Scratch buffer %d reset - (ctx %p)(Frame index: %d)\n",
694 current_scratch_buffer_,
703 MTLCircularBuffer *active_scratch_buf = scratch_buffers_[current_scratch_buffer_];
704 BLI_assert(&active_scratch_buf->own_context_ == &context_);
705 active_scratch_buf->
flush();
713 MTLResourceOptions
options = ([own_context_.
device hasUnifiedMemory]) ?
714 MTLResourceStorageModeShared :
715 MTLResourceStorageModeManaged;
718 can_resize_ = allow_grow;
722 last_flush_base_offset_ = 0;
726 cbuffer_->
set_label(
@"Circular Scratch Buffer");
746 alignment =
max_ulul(alignment, 256);
751 bool can_allocate = (aligned_current_offset + aligned_alloc_size) < cbuffer_->
get_size();
753 BLI_assert(aligned_current_offset >= current_offset_);
756 BLI_assert(aligned_current_offset % alignment == 0);
757 BLI_assert(aligned_alloc_size % alignment == 0);
768 aligned_current_offset + aligned_alloc_size);
770 #if MTL_SCRATCH_BUFFER_ALLOW_TEMPORARY_EXPANSION == 1
779 new_size = aligned_alloc_size;
780 MTL_LOG_INFO(
"Temporarily growing Scratch buffer to %d MB\n",
781 (
int)new_size / 1024 / 1024);
785 MTL_LOG_INFO(
"Shrinking Scratch buffer back to %d MB\n", (
int)new_size / 1024 / 1024);
792 if (aligned_alloc_size > new_size) {
798 alloc_range.
data =
nullptr;
800 alloc_range.
size = 0;
801 alloc_range.
options = cbuffer_->options;
807 "Performance Warning: Reached the end of circular buffer of size: %llu, but cannot "
808 "resize. Starting new buffer\n",
815 alloc_range.
data =
nullptr;
817 alloc_range.
size = 0;
832 last_flush_base_offset_ = 0;
836 cbuffer_->
set_label(
@"Circular Scratch Buffer");
838 MTL_LOG_INFO(
"Resized Metal circular buffer to %llu bytes\n", new_size);
841 aligned_current_offset = 0;
849 aligned_current_offset);
851 alloc_range.
size = aligned_alloc_size;
856 current_offset_ = aligned_current_offset + aligned_alloc_size;
857 BLI_assert(current_offset_ <= cbuffer_->get_size());
865 uint64_t len = current_offset_ - last_flush_base_offset_;
868 last_flush_base_offset_ = current_offset_;
877 if (current_offset_ > 0) {
882 "Trying to reset Circular scratch buffer's while its data is still being used by "
883 "an in-flight frame");
886 last_flush_base_offset_ = 0;
MINLINE int min_ii(int a, int b)
MINLINE uint max_uu(uint a, uint b)
MINLINE uint64_t ceil_to_multiple_ul(uint64_t a, uint64_t b)
static DBVT_INLINE btScalar size(const btDbvtVolume &a)
ValueIterator values() const
const Value & lookup(const Key &key) const
void add_new(const Key &key, const Value &value)
const Value * lookup_ptr(const Key &key) const
void push_completed_safe_list(MTLSafeFreeList *list)
void init(id< MTLDevice > device)
gpu::MTLBuffer * allocate_with_data(uint64_t size, bool cpu_visible, const void *data=nullptr)
MTLSafeFreeList * get_current_safe_list()
void update_memory_pools()
void begin_new_safe_list()
gpu::MTLBuffer * allocate(uint64_t size, bool cpu_visible)
gpu::MTLBuffer * allocate_aligned(uint64_t size, uint alignment, bool cpu_visible)
gpu::MTLBuffer * allocate_aligned_with_data(uint64_t size, uint alignment, bool cpu_visible, const void *data=nullptr)
bool free_buffer(gpu::MTLBuffer *buffer)
void flag_in_use(bool used)
id< MTLBuffer > get_metal_buffer() const
void flush_range(uint64_t offset, uint64_t length)
void set_usage_size(uint64_t size_used)
MTLResourceOptions get_resource_options()
void * get_host_ptr() const
MTLBuffer(id< MTLDevice > device, uint64_t size, MTLResourceOptions options, uint alignment=1)
uint64_t get_size_used() const
uint64_t get_size() const
void set_label(NSString *str)
MTLCircularBuffer(MTLContext &ctx, uint64_t initial_size, bool allow_grow)
MTLTemporaryBuffer allocate_range_aligned(uint64_t alloc_size, uint alignment)
MTLTemporaryBuffer allocate_range(uint64_t alloc_size)
uint get_current_frame_index()
static MTLBufferPool & get_global_memory_manager()
void increment_reference()
void decrement_reference()
void insert_buffer(gpu::MTLBuffer *buffer)
static constexpr uint mtl_scratch_buffer_max_size_
static constexpr uint mtl_scratch_buffer_initial_size_
MTLTemporaryBuffer scratch_buffer_allocate_range_aligned(uint64_t alloc_size, uint alignment)
void flush_active_scratch_buffer()
void ensure_increment_scratch_buffer()
~MTLScratchBufferManager()
MTLTemporaryBuffer scratch_buffer_allocate_range(uint64_t alloc_size)
CCL_NAMESPACE_BEGIN struct Options options
ccl_global float * buffer
ccl_gpu_kernel_postfix ccl_global int * counter
ccl_gpu_kernel_postfix ccl_global float int int int int float bool int offset
MINLINE unsigned long long min_ulul(unsigned long long a, unsigned long long b)
MINLINE unsigned long long max_ulul(unsigned long long a, unsigned long long b)
#define MTL_NUM_SAFE_FRAMES
#define MTL_LOG_INFO(info,...)
#define MTL_LOG_WARNING(info,...)
T length(const vec_base< T, Size > &a)
unsigned __int64 uint64_t
MTLResourceOptions options
id< MTLBuffer > metal_buffer