Blender  V3.3
mallocn_lockfree_impl.c
Go to the documentation of this file.
1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 
9 #include <stdarg.h>
10 #include <stdio.h> /* printf */
11 #include <stdlib.h>
12 #include <string.h> /* memcpy */
13 #include <sys/types.h>
14 
15 #include "MEM_guardedalloc.h"
16 
17 /* to ensure strict conversions */
18 #include "../../source/blender/blenlib/BLI_strict_flags.h"
19 
20 #include "atomic_ops.h"
21 #include "mallocn_intern.h"
22 
23 typedef struct MemHead {
24  /* Length of allocated memory block. */
25  size_t len;
27 
28 typedef struct MemHeadAligned {
29  short alignment;
30  size_t len;
32 
33 static unsigned int totblock = 0;
34 static size_t mem_in_use = 0, peak_mem = 0;
35 static bool malloc_debug_memset = false;
36 
37 static void (*error_callback)(const char *) = NULL;
38 
39 enum {
41 };
42 
43 #define MEMHEAD_FROM_PTR(ptr) (((MemHead *)ptr) - 1)
44 #define PTR_FROM_MEMHEAD(memhead) (memhead + 1)
45 #define MEMHEAD_ALIGNED_FROM_PTR(ptr) (((MemHeadAligned *)ptr) - 1)
46 #define MEMHEAD_IS_ALIGNED(memhead) ((memhead)->len & (size_t)MEMHEAD_ALIGN_FLAG)
47 #define MEMHEAD_LEN(memhead) ((memhead)->len & ~((size_t)(MEMHEAD_ALIGN_FLAG)))
48 
49 /* Uncomment this to have proper peak counter. */
50 #define USE_ATOMIC_MAX
51 
52 MEM_INLINE void update_maximum(size_t *maximum_value, size_t value)
53 {
54 #ifdef USE_ATOMIC_MAX
55  atomic_fetch_and_update_max_z(maximum_value, value);
56 #else
57  *maximum_value = value > *maximum_value ? value : *maximum_value;
58 #endif
59 }
60 
61 #ifdef __GNUC__
62 __attribute__((format(printf, 1, 2)))
63 #endif
64 static void
65 print_error(const char *str, ...)
66 {
67  char buf[512];
68  va_list ap;
69 
70  va_start(ap, str);
71  vsnprintf(buf, sizeof(buf), str, ap);
72  va_end(ap);
73  buf[sizeof(buf) - 1] = '\0';
74 
75  if (error_callback) {
76  error_callback(buf);
77  }
78 }
79 
80 size_t MEM_lockfree_allocN_len(const void *vmemh)
81 {
82  if (LIKELY(vmemh)) {
83  return MEMHEAD_LEN(MEMHEAD_FROM_PTR(vmemh));
84  }
85 
86  return 0;
87 }
88 
89 void MEM_lockfree_freeN(void *vmemh)
90 {
93  }
94 
95  if (UNLIKELY(vmemh == NULL)) {
96  print_error("Attempt to free NULL pointer\n");
97 #ifdef WITH_ASSERT_ABORT
98  abort();
99 #endif
100  return;
101  }
102 
103  MemHead *memh = MEMHEAD_FROM_PTR(vmemh);
104  size_t len = MEMHEAD_LEN(memh);
105 
108 
109  if (UNLIKELY(malloc_debug_memset && len)) {
110  memset(memh + 1, 255, len);
111  }
112  if (UNLIKELY(MEMHEAD_IS_ALIGNED(memh))) {
113  MemHeadAligned *memh_aligned = MEMHEAD_ALIGNED_FROM_PTR(vmemh);
114  aligned_free(MEMHEAD_REAL_PTR(memh_aligned));
115  }
116  else {
117  free(memh);
118  }
119 }
120 
121 void *MEM_lockfree_dupallocN(const void *vmemh)
122 {
123  void *newp = NULL;
124  if (vmemh) {
125  MemHead *memh = MEMHEAD_FROM_PTR(vmemh);
126  const size_t prev_size = MEM_lockfree_allocN_len(vmemh);
127  if (UNLIKELY(MEMHEAD_IS_ALIGNED(memh))) {
128  MemHeadAligned *memh_aligned = MEMHEAD_ALIGNED_FROM_PTR(vmemh);
130  prev_size, (size_t)memh_aligned->alignment, "dupli_malloc");
131  }
132  else {
133  newp = MEM_lockfree_mallocN(prev_size, "dupli_malloc");
134  }
135  memcpy(newp, vmemh, prev_size);
136  }
137  return newp;
138 }
139 
140 void *MEM_lockfree_reallocN_id(void *vmemh, size_t len, const char *str)
141 {
142  void *newp = NULL;
143 
144  if (vmemh) {
145  MemHead *memh = MEMHEAD_FROM_PTR(vmemh);
146  size_t old_len = MEM_lockfree_allocN_len(vmemh);
147 
148  if (LIKELY(!MEMHEAD_IS_ALIGNED(memh))) {
149  newp = MEM_lockfree_mallocN(len, "realloc");
150  }
151  else {
152  MemHeadAligned *memh_aligned = MEMHEAD_ALIGNED_FROM_PTR(vmemh);
153  newp = MEM_lockfree_mallocN_aligned(len, (size_t)memh_aligned->alignment, "realloc");
154  }
155 
156  if (newp) {
157  if (len < old_len) {
158  /* shrink */
159  memcpy(newp, vmemh, len);
160  }
161  else {
162  /* grow (or remain same size) */
163  memcpy(newp, vmemh, old_len);
164  }
165  }
166 
167  MEM_lockfree_freeN(vmemh);
168  }
169  else {
170  newp = MEM_lockfree_mallocN(len, str);
171  }
172 
173  return newp;
174 }
175 
176 void *MEM_lockfree_recallocN_id(void *vmemh, size_t len, const char *str)
177 {
178  void *newp = NULL;
179 
180  if (vmemh) {
181  MemHead *memh = MEMHEAD_FROM_PTR(vmemh);
182  size_t old_len = MEM_lockfree_allocN_len(vmemh);
183 
184  if (LIKELY(!MEMHEAD_IS_ALIGNED(memh))) {
185  newp = MEM_lockfree_mallocN(len, "recalloc");
186  }
187  else {
188  MemHeadAligned *memh_aligned = MEMHEAD_ALIGNED_FROM_PTR(vmemh);
189  newp = MEM_lockfree_mallocN_aligned(len, (size_t)memh_aligned->alignment, "recalloc");
190  }
191 
192  if (newp) {
193  if (len < old_len) {
194  /* shrink */
195  memcpy(newp, vmemh, len);
196  }
197  else {
198  memcpy(newp, vmemh, old_len);
199 
200  if (len > old_len) {
201  /* grow */
202  /* zero new bytes */
203  memset(((char *)newp) + old_len, 0, len - old_len);
204  }
205  }
206  }
207 
208  MEM_lockfree_freeN(vmemh);
209  }
210  else {
211  newp = MEM_lockfree_callocN(len, str);
212  }
213 
214  return newp;
215 }
216 
217 void *MEM_lockfree_callocN(size_t len, const char *str)
218 {
219  MemHead *memh;
220 
221  len = SIZET_ALIGN_4(len);
222 
223  memh = (MemHead *)calloc(1, len + sizeof(MemHead));
224 
225  if (LIKELY(memh)) {
226  memh->len = len;
230 
231  return PTR_FROM_MEMHEAD(memh);
232  }
233  print_error("Calloc returns null: len=" SIZET_FORMAT " in %s, total %u\n",
234  SIZET_ARG(len),
235  str,
236  (unsigned int)mem_in_use);
237  return NULL;
238 }
239 
240 void *MEM_lockfree_calloc_arrayN(size_t len, size_t size, const char *str)
241 {
242  size_t total_size;
243  if (UNLIKELY(!MEM_size_safe_multiply(len, size, &total_size))) {
244  print_error(
245  "Calloc array aborted due to integer overflow: "
246  "len=" SIZET_FORMAT "x" SIZET_FORMAT " in %s, total %u\n",
247  SIZET_ARG(len),
248  SIZET_ARG(size),
249  str,
250  (unsigned int)mem_in_use);
251  abort();
252  return NULL;
253  }
254 
255  return MEM_lockfree_callocN(total_size, str);
256 }
257 
258 void *MEM_lockfree_mallocN(size_t len, const char *str)
259 {
260  MemHead *memh;
261 
262  len = SIZET_ALIGN_4(len);
263 
264  memh = (MemHead *)malloc(len + sizeof(MemHead));
265 
266  if (LIKELY(memh)) {
267  if (UNLIKELY(malloc_debug_memset && len)) {
268  memset(memh + 1, 255, len);
269  }
270 
271  memh->len = len;
275 
276  return PTR_FROM_MEMHEAD(memh);
277  }
278  print_error("Malloc returns null: len=" SIZET_FORMAT " in %s, total %u\n",
279  SIZET_ARG(len),
280  str,
281  (unsigned int)mem_in_use);
282  return NULL;
283 }
284 
285 void *MEM_lockfree_malloc_arrayN(size_t len, size_t size, const char *str)
286 {
287  size_t total_size;
288  if (UNLIKELY(!MEM_size_safe_multiply(len, size, &total_size))) {
289  print_error(
290  "Malloc array aborted due to integer overflow: "
291  "len=" SIZET_FORMAT "x" SIZET_FORMAT " in %s, total %u\n",
292  SIZET_ARG(len),
293  SIZET_ARG(size),
294  str,
295  (unsigned int)mem_in_use);
296  abort();
297  return NULL;
298  }
299 
300  return MEM_lockfree_mallocN(total_size, str);
301 }
302 
303 void *MEM_lockfree_mallocN_aligned(size_t len, size_t alignment, const char *str)
304 {
305  /* Huge alignment values doesn't make sense and they wouldn't fit into 'short' used in the
306  * MemHead. */
307  assert(alignment < 1024);
308 
309  /* We only support alignments that are a power of two. */
310  assert(IS_POW2(alignment));
311 
312  /* Some OS specific aligned allocators require a certain minimal alignment. */
313  if (alignment < ALIGNED_MALLOC_MINIMUM_ALIGNMENT) {
315  }
316 
317  /* It's possible that MemHead's size is not properly aligned,
318  * do extra padding to deal with this.
319  *
320  * We only support small alignments which fits into short in
321  * order to save some bits in MemHead structure.
322  */
323  size_t extra_padding = MEMHEAD_ALIGN_PADDING(alignment);
324 
325  len = SIZET_ALIGN_4(len);
326 
328  len + extra_padding + sizeof(MemHeadAligned), alignment);
329 
330  if (LIKELY(memh)) {
331  /* We keep padding in the beginning of MemHead,
332  * this way it's always possible to get MemHead
333  * from the data pointer.
334  */
335  memh = (MemHeadAligned *)((char *)memh + extra_padding);
336 
337  if (UNLIKELY(malloc_debug_memset && len)) {
338  memset(memh + 1, 255, len);
339  }
340 
341  memh->len = len | (size_t)MEMHEAD_ALIGN_FLAG;
342  memh->alignment = (short)alignment;
346 
347  return PTR_FROM_MEMHEAD(memh);
348  }
349  print_error("Malloc returns null: len=" SIZET_FORMAT " in %s, total %u\n",
350  SIZET_ARG(len),
351  str,
352  (unsigned int)mem_in_use);
353  return NULL;
354 }
355 
357 {
358 }
359 
361 {
362 }
363 
364 /* unused */
365 void MEM_lockfree_callbackmemlist(void (*func)(void *))
366 {
367  (void)func; /* Ignored. */
368 }
369 
371 {
372  printf("\ntotal memory len: %.3f MB\n", (double)mem_in_use / (double)(1024 * 1024));
373  printf("peak memory len: %.3f MB\n", (double)peak_mem / (double)(1024 * 1024));
374  printf(
375  "\nFor more detailed per-block statistics run Blender with memory debugging command line "
376  "argument.\n");
377 
378 #ifdef HAVE_MALLOC_STATS
379  printf("System Statistics:\n");
380  malloc_stats();
381 #endif
382 }
383 
384 void MEM_lockfree_set_error_callback(void (*func)(const char *))
385 {
386  error_callback = func;
387 }
388 
390 {
391  return true;
392 }
393 
395 {
396  malloc_debug_memset = true;
397 }
398 
400 {
401  return mem_in_use;
402 }
403 
405 {
406  return totblock;
407 }
408 
409 /* dummy */
411 {
413 }
414 
416 {
417  return peak_mem;
418 }
419 
420 #ifndef NDEBUG
421 const char *MEM_lockfree_name_ptr(void *vmemh)
422 {
423  if (vmemh) {
424  return "unknown block name ptr";
425  }
426 
427  return "MEM_lockfree_name_ptr(NULL)";
428 }
429 #endif /* NDEBUG */
void BLI_kdtree_nd_() free(KDTree *tree)
Definition: kdtree_impl.h:102
#define UNLIKELY(x)
#define LIKELY(x)
Read Guarded memory(de)allocation.
Provides wrapper around system-specific atomic primitives, and some extensions (faked-atomic operatio...
ATOMIC_INLINE size_t atomic_add_and_fetch_z(size_t *p, size_t x)
ATOMIC_INLINE size_t atomic_sub_and_fetch_z(size_t *p, size_t x)
ATOMIC_INLINE unsigned int atomic_add_and_fetch_u(unsigned int *p, unsigned int x)
ATOMIC_INLINE size_t atomic_fetch_and_update_max_z(size_t *p, size_t x)
ATOMIC_INLINE unsigned int atomic_sub_and_fetch_u(unsigned int *p, unsigned int x)
struct AtomicSpinLock __attribute__((aligned(32))) AtomicSpinLock
static DBVT_INLINE btScalar size(const btDbvtVolume &a)
Definition: btDbvt.cpp:52
SyclQueue void void size_t num_bytes void
int len
Definition: draw_manager.c:108
#define str(s)
bool leak_detector_has_run
char free_after_leak_detection_message[]
format
Definition: logImageCore.h:38
void aligned_free(void *ptr)
Definition: mallocn.c:76
void * aligned_malloc(size_t size, size_t alignment)
Definition: mallocn.c:54
MEM_INLINE bool MEM_size_safe_multiply(size_t a, size_t b, size_t *result)
#define MEMHEAD_REAL_PTR(memh)
#define IS_POW2(a)
#define MEMHEAD_ALIGN_PADDING(alignment)
#define SIZET_ARG(a)
#define MEM_INLINE
#define SIZET_ALIGN_4(len)
#define SIZET_FORMAT
#define ALIGNED_MALLOC_MINIMUM_ALIGNMENT
static size_t mem_in_use
void * MEM_lockfree_dupallocN(const void *vmemh)
void * MEM_lockfree_mallocN_aligned(size_t len, size_t alignment, const char *str)
void * MEM_lockfree_calloc_arrayN(size_t len, size_t size, const char *str)
void MEM_lockfree_reset_peak_memory(void)
static size_t peak_mem
void MEM_lockfree_set_error_callback(void(*func)(const char *))
void MEM_lockfree_printmemlist_stats(void)
void MEM_lockfree_set_memory_debug(void)
#define PTR_FROM_MEMHEAD(memhead)
void * MEM_lockfree_reallocN_id(void *vmemh, size_t len, const char *str)
static bool malloc_debug_memset
static unsigned int totblock
static void(* error_callback)(const char *)
bool MEM_lockfree_consistency_check(void)
void * MEM_lockfree_callocN(size_t len, const char *str)
const char * MEM_lockfree_name_ptr(void *vmemh)
static void print_error(const char *str,...)
struct MemHead MemHead
void MEM_lockfree_printmemlist_pydict(void)
@ MEMHEAD_ALIGN_FLAG
#define MEMHEAD_LEN(memhead)
#define MEMHEAD_IS_ALIGNED(memhead)
void MEM_lockfree_printmemlist(void)
void MEM_lockfree_freeN(void *vmemh)
void * MEM_lockfree_mallocN(size_t len, const char *str)
void * MEM_lockfree_recallocN_id(void *vmemh, size_t len, const char *str)
struct MemHeadAligned MemHeadAligned
void MEM_lockfree_callbackmemlist(void(*func)(void *))
#define MEMHEAD_FROM_PTR(ptr)
void * MEM_lockfree_malloc_arrayN(size_t len, size_t size, const char *str)
#define MEMHEAD_ALIGNED_FROM_PTR(ptr)
size_t MEM_lockfree_get_memory_in_use(void)
size_t MEM_lockfree_get_peak_memory(void)
size_t MEM_lockfree_allocN_len(const void *vmemh)
unsigned int MEM_lockfree_get_memory_blocks_in_use(void)
MEM_INLINE void update_maximum(size_t *maximum_value, size_t value)