Leptonica  1.83.1
Image processing and image analysis suite
pixalloc.c
Go to the documentation of this file.
1 /*====================================================================*
2  - Copyright (C) 2001 Leptonica. All rights reserved.
3  -
4  - Redistribution and use in source and binary forms, with or without
5  - modification, are permitted provided that the following conditions
6  - are met:
7  - 1. Redistributions of source code must retain the above copyright
8  - notice, this list of conditions and the following disclaimer.
9  - 2. Redistributions in binary form must reproduce the above
10  - copyright notice, this list of conditions and the following
11  - disclaimer in the documentation and/or other materials
12  - provided with the distribution.
13  -
14  - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
15  - ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
16  - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
17  - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL ANY
18  - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19  - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20  - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
21  - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
22  - OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
23  - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24  - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *====================================================================*/
26 
44 #ifdef HAVE_CONFIG_H
45 #include <config_auto.h>
46 #endif /* HAVE_CONFIG_H */
47 
48 #include "allheaders.h"
49 
50 /*-------------------------------------------------------------------------*
51  * Pix Memory Storage *
52  * *
53  * This is a simple utility for handling pix memory storage. It is *
54  * enabled by setting the PixMemoryManager allocators to the functions *
55  * that are defined here *
56  * pmsCustomAlloc() *
57  * pmsCustomDealloc() *
58  * Use pmsCreate() at the beginning to do the pre-allocation, and *
59  * pmsDestroy() at the end to clean it up. *
60  *-------------------------------------------------------------------------*/
61 /*
62  * In the following, the "memory" refers to the image data
63  * field that is used within the pix. The memory store is a
64  * continuous block of memory, that is logically divided into
65  * smaller "chunks" starting with a set at a minimum size, and
66  * followed by sets of increasing size that are a power of 2 larger
67  * than the minimum size. You must specify the number of chunks
68  * of each size.
69  *
70  * A requested data chunk, if it exists, is borrowed from the memory
71  * storage, and returned after use. If the chunk is too small, or
72  * too large, or if all chunks in the appropriate size range are
73  * in use, the memory is allocated dynamically and freed after use.
74  *
75  * There are four parameters that determine the use of pre-allocated memory:
76  *
77  * minsize: any requested chunk smaller than this is allocated
78  * dynamically and destroyed after use. No preallocated
79  * memory is used.
80  * smallest: the size of the smallest pre-allocated memory chunk.
81  * nlevels: the number of different sizes of data chunks, each a
82  * power of 2 larger than 'smallest'.
83  * numalloc: a Numa of size 'nlevels' containing the number of data
84  * chunks for each size that are in the memory store.
85  *
86  * As an example, suppose:
87  * minsize = 0.5MB
88  * smallest = 1.0MB
89  * nlevels = 4
90  * numalloc = {10, 5, 5, 5}
91  * Then the total amount of allocated memory (in MB) is
92  * 10 * 1 + 5 * 2 + 5 * 4 + 5 * 8 = 80 MB
93  * Any pix requiring less than 0.5 MB or more than 8 MB of memory will
94  * not come from the memory store. Instead, it will be dynamically
95  * allocated and freed after use.
96  *
97  * How is this implemented?
98  *
99  * At setup, the full data block size is computed and allocated.
100  * The addresses of the individual chunks are found, and the pointers
101  * are stored in a set of Ptra (generic pointer arrays), using one Ptra
102  * for each of the sizes of the chunks. When returning a chunk after
103  * use, it is necessary to determine from the address which size level
104  * (ptra) the chunk belongs to. This is done by comparing the address
105  * of the associated chunk.
106  *
107  * In the event that memory chunks need to be dynamically allocated,
108  * either (1) because they are too small or too large for the memory
109  * store or (2) because all the pix of that size (i.e., in the
110  * appropriate level) in the memory store are in use, the
111  * addresses generated will be outside the pre-allocated block.
112  * After use they won't be returned to a ptra; instead the deallocator
113  * will free them.
114  */
115 
118 {
119  struct L_Ptraa *paa;
120  size_t minsize;
122  size_t smallest;
123  size_t largest;
124  size_t nbytes;
125  l_int32 nlevels;
126  size_t *sizes;
127  l_int32 *allocarray;
128  l_uint32 *baseptr;
129  l_uint32 *maxptr;
130  l_uint32 **firstptr;
131  l_int32 *memused;
132  l_int32 *meminuse;
133  l_int32 *memmax;
134  l_int32 *memempty;
136  char *logfile;
137 };
138 typedef struct PixMemoryStore L_PIX_MEM_STORE;
139 
140 static L_PIX_MEM_STORE *CustomPMS = NULL;
141 
142 
171 l_ok
173  size_t smallest,
174  NUMA *numalloc,
175  const char *logfile)
176 {
177 l_int32 nlevels, i, j, nbytes;
178 l_int32 *alloca;
179 l_float32 nchunks;
180 l_uint32 *baseptr, *data;
181 l_uint32 **firstptr;
182 size_t *sizes;
183 L_PIX_MEM_STORE *pms;
184 L_PTRA *pa;
185 L_PTRAA *paa;
186 
187  if (!numalloc)
188  return ERROR_INT("numalloc not defined", __func__, 1);
189  numaGetSum(numalloc, &nchunks);
190  if (nchunks > 1000.0)
191  L_WARNING("There are %.0f chunks\n", __func__, nchunks);
192 
193  pms = (L_PIX_MEM_STORE *)LEPT_CALLOC(1, sizeof(L_PIX_MEM_STORE));
194  CustomPMS = pms;
195 
196  /* Make sure that minsize and smallest are multiples of 32 bit words */
197  if (minsize % 4 != 0)
198  minsize -= minsize % 4;
199  pms->minsize = minsize;
200  nlevels = numaGetCount(numalloc);
201  pms->nlevels = nlevels;
202 
203  if ((sizes = (size_t *)LEPT_CALLOC(nlevels, sizeof(size_t))) == NULL)
204  return ERROR_INT("sizes not made", __func__, 1);
205  pms->sizes = sizes;
206  if (smallest % 4 != 0)
207  smallest += 4 - (smallest % 4);
208  pms->smallest = smallest;
209  for (i = 0; i < nlevels; i++)
210  sizes[i] = smallest * (1 << i);
211  pms->largest = sizes[nlevels - 1];
212 
213  alloca = numaGetIArray(numalloc);
214  pms->allocarray = alloca;
215  if ((paa = ptraaCreate(nlevels)) == NULL)
216  return ERROR_INT("paa not made", __func__, 1);
217  pms->paa = paa;
218 
219  for (i = 0, nbytes = 0; i < nlevels; i++)
220  nbytes += alloca[i] * sizes[i];
221  pms->nbytes = nbytes;
222 
223  if ((baseptr = (l_uint32 *)LEPT_CALLOC(nbytes / 4, sizeof(l_uint32)))
224  == NULL)
225  return ERROR_INT("calloc fail for baseptr", __func__, 1);
226  pms->baseptr = baseptr;
227  pms->maxptr = baseptr + nbytes / 4; /* just beyond the memory store */
228  if ((firstptr = (l_uint32 **)LEPT_CALLOC(nlevels, sizeof(l_uint32 *)))
229  == NULL)
230  return ERROR_INT("calloc fail for firstptr", __func__, 1);
231  pms->firstptr = firstptr;
232 
233  data = baseptr;
234  for (i = 0; i < nlevels; i++) {
235  if ((pa = ptraCreate(alloca[i])) == NULL)
236  return ERROR_INT("pa not made", __func__, 1);
237  ptraaInsertPtra(paa, i, pa);
238  firstptr[i] = data;
239  for (j = 0; j < alloca[i]; j++) {
240  ptraAdd(pa, data);
241  data += sizes[i] / 4;
242  }
243  }
244 
245  if (logfile) {
246  pms->memused = (l_int32 *)LEPT_CALLOC(nlevels, sizeof(l_int32));
247  pms->meminuse = (l_int32 *)LEPT_CALLOC(nlevels, sizeof(l_int32));
248  pms->memmax = (l_int32 *)LEPT_CALLOC(nlevels, sizeof(l_int32));
249  pms->memempty = (l_int32 *)LEPT_CALLOC(nlevels, sizeof(l_int32));
250  pms->logfile = stringNew(logfile);
251  }
252 
253  return 0;
254 }
255 
256 
266 void
268 {
269 L_PIX_MEM_STORE *pms;
270 
271  if ((pms = CustomPMS) == NULL)
272  return;
273 
274  ptraaDestroy(&pms->paa, FALSE, FALSE); /* don't touch the ptrs */
275  LEPT_FREE(pms->baseptr); /* free the memory */
276 
277  if (pms->logfile) {
278  pmsLogInfo();
279  LEPT_FREE(pms->logfile);
280  LEPT_FREE(pms->memused);
281  LEPT_FREE(pms->meminuse);
282  LEPT_FREE(pms->memmax);
283  LEPT_FREE(pms->memempty);
284  }
285 
286  LEPT_FREE(pms->sizes);
287  LEPT_FREE(pms->allocarray);
288  LEPT_FREE(pms->firstptr);
289  LEPT_FREE(pms);
290  CustomPMS = NULL;
291 }
292 
293 
309 void *
311 {
312 l_int32 level;
313 void *data;
314 L_PIX_MEM_STORE *pms;
315 L_PTRA *pa;
316 
317  if ((pms = CustomPMS) == NULL)
318  return (void *)ERROR_PTR("pms not defined", __func__, NULL);
319 
320  pmsGetLevelForAlloc(nbytes, &level);
321 
322  if (level < 0) { /* size range invalid; must alloc */
323  if ((data = pmsGetAlloc(nbytes)) == NULL)
324  return (void *)ERROR_PTR("data not made", __func__, NULL);
325  } else { /* get from store */
326  pa = ptraaGetPtra(pms->paa, level, L_HANDLE_ONLY);
327  data = ptraRemoveLast(pa);
328  if (data && pms->logfile) {
329  pms->memused[level]++;
330  pms->meminuse[level]++;
331  if (pms->meminuse[level] > pms->memmax[level])
332  pms->memmax[level]++;
333  }
334  if (!data) { /* none left at this level */
335  data = pmsGetAlloc(nbytes);
336  if (pms->logfile)
337  pms->memempty[level]++;
338  }
339  }
340 
341  return data;
342 }
343 
344 
351 void
352 pmsCustomDealloc(void *data)
353 {
354 l_int32 level;
355 L_PIX_MEM_STORE *pms;
356 L_PTRA *pa;
357 
358  if ((pms = CustomPMS) == NULL) {
359  L_ERROR("pms not defined\n", __func__);
360  return;
361  }
362 
363  if (pmsGetLevelForDealloc(data, &level) == 1) {
364  L_ERROR("level not found\n", __func__);
365  return;
366  }
367 
368  if (level < 0) { /* no logging; just free the data */
369  LEPT_FREE(data);
370  } else { /* return the data to the store */
371  pa = ptraaGetPtra(pms->paa, level, L_HANDLE_ONLY);
372  ptraAdd(pa, data);
373  if (pms->logfile)
374  pms->meminuse[level]--;
375  }
376 }
377 
378 
396 void *
398 {
399 void *data;
400 FILE *fp;
401 L_PIX_MEM_STORE *pms;
402 
403  if ((pms = CustomPMS) == NULL)
404  return (void *)ERROR_PTR("pms not defined", __func__, NULL);
405 
406  if ((data = (void *)LEPT_CALLOC(nbytes, sizeof(char))) == NULL)
407  return (void *)ERROR_PTR("data not made", __func__, NULL);
408  if (pms->logfile && nbytes >= pms->smallest) {
409  if ((fp = fopenWriteStream(pms->logfile, "a")) != NULL) {
410  fprintf(fp, "Alloc %zu bytes at %p\n", nbytes, data);
411  fclose(fp);
412  } else {
413  L_ERROR("failed to open stream for %s\n", __func__, pms->logfile);
414  }
415  }
416  return data;
417 }
418 
419 
427 l_ok
429  l_int32 *plevel)
430 {
431 l_int32 i;
432 l_float64 ratio;
433 L_PIX_MEM_STORE *pms;
434 
435  if (!plevel)
436  return ERROR_INT("&level not defined", __func__, 1);
437  *plevel = -1;
438  if ((pms = CustomPMS) == NULL)
439  return ERROR_INT("pms not defined", __func__, 1);
440 
441  if (nbytes < pms->minsize || nbytes > pms->largest)
442  return 0; /* -1 */
443 
444  ratio = (l_float64)nbytes / (l_float64)(pms->smallest);
445  for (i = 0; i < pms->nlevels; i++) {
446  if (ratio <= 1.0)
447  break;
448  ratio /= 2.0;
449  }
450  *plevel = i;
451 
452  return 0;
453 }
454 
455 
464 l_ok
466  l_int32 *plevel)
467 {
468 l_int32 i;
469 l_uint32 *first;
470 L_PIX_MEM_STORE *pms;
471 
472  if (!plevel)
473  return ERROR_INT("&level not defined", __func__, 1);
474  *plevel = -1;
475  if (!data)
476  return ERROR_INT("data not defined", __func__, 1);
477  if ((pms = CustomPMS) == NULL)
478  return ERROR_INT("pms not defined", __func__, 1);
479 
480  if (data < (void *)pms->baseptr || data >= (void *)pms->maxptr)
481  return 0; /* -1 */
482 
483  for (i = 1; i < pms->nlevels; i++) {
484  first = pms->firstptr[i];
485  if (data < (void *)first)
486  break;
487  }
488  *plevel = i - 1;
489 
490  return 0;
491 }
492 
493 
497 void
499 {
500 l_int32 i;
501 L_PIX_MEM_STORE *pms;
502 
503  if ((pms = CustomPMS) == NULL)
504  return;
505 
506  lept_stderr("Total number of pix used at each level\n");
507  for (i = 0; i < pms->nlevels; i++)
508  lept_stderr(" Level %d (%zu bytes): %d\n", i,
509  pms->sizes[i], pms->memused[i]);
510 
511  lept_stderr("Max number of pix in use at any time in each level\n");
512  for (i = 0; i < pms->nlevels; i++)
513  lept_stderr(" Level %d (%zu bytes): %d\n", i,
514  pms->sizes[i], pms->memmax[i]);
515 
516  lept_stderr("Number of pix alloc'd because none were available\n");
517  for (i = 0; i < pms->nlevels; i++)
518  lept_stderr(" Level %d (%zu bytes): %d\n", i,
519  pms->sizes[i], pms->memempty[i]);
520 }
l_int32 numaGetCount(NUMA *na)
numaGetCount()
Definition: numabasic.c:630
l_int32 * numaGetIArray(NUMA *na)
numaGetIArray()
Definition: numabasic.c:807
l_ok numaGetSum(NUMA *na, l_float32 *psum)
numaGetSum()
Definition: numafunc1.c:525
l_ok pmsCreate(size_t minsize, size_t smallest, NUMA *numalloc, const char *logfile)
pmsCreate()
Definition: pixalloc.c:172
void pmsLogInfo(void)
pmsLogInfo()
Definition: pixalloc.c:498
l_ok pmsGetLevelForAlloc(size_t nbytes, l_int32 *plevel)
pmsGetLevelForAlloc()
Definition: pixalloc.c:428
void pmsDestroy(void)
pmsDestroy()
Definition: pixalloc.c:267
void pmsCustomDealloc(void *data)
pmsCustomDealloc()
Definition: pixalloc.c:352
l_ok pmsGetLevelForDealloc(void *data, l_int32 *plevel)
pmsGetLevelForDealloc()
Definition: pixalloc.c:465
void * pmsCustomAlloc(size_t nbytes)
pmsCustomAlloc()
Definition: pixalloc.c:310
void * pmsGetAlloc(size_t nbytes)
pmsGetAlloc()
Definition: pixalloc.c:397
L_PTRAA * ptraaCreate(l_int32 n)
ptraaCreate()
Definition: ptra.c:768
L_PTRA * ptraCreate(l_int32 n)
ptraCreate()
Definition: ptra.c:144
l_ok ptraAdd(L_PTRA *pa, void *item)
ptraAdd()
Definition: ptra.c:246
void ptraaDestroy(L_PTRAA **ppaa, l_int32 freeflag, l_int32 warnflag)
ptraaDestroy()
Definition: ptra.c:802
L_PTRA * ptraaGetPtra(L_PTRAA *paa, l_int32 index, l_int32 accessflag)
ptraaGetPtra()
Definition: ptra.c:910
l_ok ptraaInsertPtra(L_PTRAA *paa, l_int32 index, L_PTRA *pa)
ptraaInsertPtra()
Definition: ptra.c:869
void * ptraRemoveLast(L_PTRA *pa)
ptraRemoveLast()
Definition: ptra.c:479
@ L_HANDLE_ONLY
Definition: ptra.h:92
Definition: ptra.h:54
Definition: ptra.h:65
l_uint32 ** firstptr
Definition: pixalloc.c:130
l_int32 * memempty
Definition: pixalloc.c:134
l_uint32 * maxptr
Definition: pixalloc.c:129
size_t nbytes
Definition: pixalloc.c:124
size_t largest
Definition: pixalloc.c:123
l_int32 * allocarray
Definition: pixalloc.c:127
l_int32 * memused
Definition: pixalloc.c:131
l_int32 * memmax
Definition: pixalloc.c:133
l_uint32 * baseptr
Definition: pixalloc.c:128
struct L_Ptraa * paa
Definition: pixalloc.c:119
char * logfile
Definition: pixalloc.c:136
l_int32 nlevels
Definition: pixalloc.c:125
size_t * sizes
Definition: pixalloc.c:126
size_t smallest
Definition: pixalloc.c:122
size_t minsize
Definition: pixalloc.c:120
l_int32 * meminuse
Definition: pixalloc.c:132
void lept_stderr(const char *fmt,...)
lept_stderr()
Definition: utils1.c:306
char * stringNew(const char *src)
stringNew()
Definition: utils2.c:223
FILE * fopenWriteStream(const char *filename, const char *modestring)
fopenWriteStream()
Definition: utils2.c:1905