0
|
1 /* -*- Mode: C; tab-width: 8; c-basic-offset: 8 -*- */
|
|
2 /* vim:set softtabstop=8 shiftwidth=8: */
|
|
3 /*-
|
|
4 * Copyright (C) 2006-2008 Jason Evans <jasone@FreeBSD.org>.
|
|
5 * All rights reserved.
|
|
6 *
|
|
7 * Redistribution and use in source and binary forms, with or without
|
|
8 * modification, are permitted provided that the following conditions
|
|
9 * are met:
|
|
10 * 1. Redistributions of source code must retain the above copyright
|
|
11 * notice(s), this list of conditions and the following disclaimer as
|
|
12 * the first lines of this file unmodified other than the possible
|
|
13 * addition of one or more copyright notices.
|
|
14 * 2. Redistributions in binary form must reproduce the above copyright
|
|
15 * notice(s), this list of conditions and the following disclaimer in
|
|
16 * the documentation and/or other materials provided with the
|
|
17 * distribution.
|
|
18 *
|
|
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
|
|
20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
|
|
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
|
|
26 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
|
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
|
28 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
|
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
30 *
|
|
31 *******************************************************************************
|
|
32 *
|
|
33 * This allocator implementation is designed to provide scalable performance
|
|
34 * for multi-threaded programs on multi-processor systems. The following
|
|
35 * features are included for this purpose:
|
|
36 *
|
|
37 * + Multiple arenas are used if there are multiple CPUs, which reduces lock
|
|
38 * contention and cache sloshing.
|
|
39 *
|
|
40 * + Cache line sharing between arenas is avoided for internal data
|
|
41 * structures.
|
|
42 *
|
|
43 * + Memory is managed in chunks and runs (chunks can be split into runs),
|
|
44 * rather than as individual pages. This provides a constant-time
|
|
45 * mechanism for associating allocations with particular arenas.
|
|
46 *
|
|
47 * Allocation requests are rounded up to the nearest size class, and no record
|
|
48 * of the original request size is maintained. Allocations are broken into
|
|
49 * categories according to size class. Assuming runtime defaults, 4 kB pages
|
|
50 * and a 16 byte quantum on a 32-bit system, the size classes in each category
|
|
51 * are as follows:
|
|
52 *
|
|
53 * |=====================================|
|
|
54 * | Category | Subcategory | Size |
|
|
55 * |=====================================|
|
|
56 * | Small | Tiny | 2 |
|
|
57 * | | | 4 |
|
|
58 * | | | 8 |
|
|
59 * | |----------------+---------|
|
|
60 * | | Quantum-spaced | 16 |
|
|
61 * | | | 32 |
|
|
62 * | | | 48 |
|
|
63 * | | | ... |
|
|
64 * | | | 480 |
|
|
65 * | | | 496 |
|
|
66 * | | | 512 |
|
|
67 * | |----------------+---------|
|
|
68 * | | Sub-page | 1 kB |
|
|
69 * | | | 2 kB |
|
|
70 * |=====================================|
|
|
71 * | Large | 4 kB |
|
|
72 * | | 8 kB |
|
|
73 * | | 12 kB |
|
|
74 * | | ... |
|
|
75 * | | 1012 kB |
|
|
76 * | | 1016 kB |
|
|
77 * | | 1020 kB |
|
|
78 * |=====================================|
|
|
79 * | Huge | 1 MB |
|
|
80 * | | 2 MB |
|
|
81 * | | 3 MB |
|
|
82 * | | ... |
|
|
83 * |=====================================|
|
|
84 *
|
|
85 * A different mechanism is used for each category:
|
|
86 *
|
|
87 * Small : Each size class is segregated into its own set of runs. Each run
|
|
88 * maintains a bitmap of which regions are free/allocated.
|
|
89 *
|
|
90 * Large : Each allocation is backed by a dedicated run. Metadata are stored
|
|
91 * in the associated arena chunk header maps.
|
|
92 *
|
|
93 * Huge : Each allocation is backed by a dedicated contiguous set of chunks.
|
|
94 * Metadata are stored in a separate red-black tree.
|
|
95 *
|
|
96 *******************************************************************************
|
|
97 */
|
|
98
|
|
99 /*
|
|
100 * MALLOC_PRODUCTION disables assertions and statistics gathering. It also
|
|
101 * defaults the A and J runtime options to off. These settings are appropriate
|
|
102 * for production systems.
|
|
103 */
|
|
104 #ifndef MOZ_MEMORY_DEBUG
|
|
105 # define MALLOC_PRODUCTION
|
|
106 #endif
|
|
107
|
|
108 /*
|
|
109 * Use only one arena by default. Mozilla does not currently make extensive
|
|
110 * use of concurrent allocation, so the increased fragmentation associated with
|
|
111 * multiple arenas is not warranted.
|
|
112 */
|
|
113 #define MOZ_MEMORY_NARENAS_DEFAULT_ONE
|
|
114
|
|
115 /*
|
|
116 * MALLOC_STATS enables statistics calculation, and is required for
|
|
117 * jemalloc_stats().
|
|
118 */
|
|
119 #define MALLOC_STATS
|
|
120
|
|
121 #ifndef MALLOC_PRODUCTION
|
|
122 /*
|
|
123 * MALLOC_DEBUG enables assertions and other sanity checks, and disables
|
|
124 * inline functions.
|
|
125 */
|
|
126 # define MALLOC_DEBUG
|
|
127
|
|
128 /* Memory filling (junk/zero). */
|
|
129 # define MALLOC_FILL
|
|
130
|
|
131 /* Allocation tracing. */
|
|
132 # ifndef MOZ_MEMORY_WINDOWS
|
|
133 # define MALLOC_UTRACE
|
|
134 # endif
|
|
135
|
|
136 /* Support optional abort() on OOM. */
|
|
137 # define MALLOC_XMALLOC
|
|
138
|
|
139 /* Support SYSV semantics. */
|
|
140 # define MALLOC_SYSV
|
|
141 #endif
|
|
142
|
|
143 /*
|
|
144 * MALLOC_VALIDATE causes malloc_usable_size() to perform some pointer
|
|
145 * validation. There are many possible errors that validation does not even
|
|
146 * attempt to detect.
|
|
147 */
|
|
148 #define MALLOC_VALIDATE
|
|
149
|
|
150 /* Embed no-op macros that support memory allocation tracking via valgrind. */
|
|
151 #ifdef MOZ_VALGRIND
|
|
152 # define MALLOC_VALGRIND
|
|
153 #endif
|
|
154 #ifdef MALLOC_VALGRIND
|
|
155 # include <valgrind/valgrind.h>
|
|
156 #else
|
|
157 # define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
|
|
158 # define VALGRIND_FREELIKE_BLOCK(addr, rzB)
|
|
159 #endif
|
|
160
|
|
161 /*
|
|
162 * MALLOC_BALANCE enables monitoring of arena lock contention and dynamically
|
|
163 * re-balances arena load if exponentially averaged contention exceeds a
|
|
164 * certain threshold.
|
|
165 */
|
|
166 /* #define MALLOC_BALANCE */
|
|
167
|
|
168 #if (!defined(MOZ_MEMORY_WINDOWS) && !defined(MOZ_MEMORY_DARWIN))
|
|
169 /*
|
|
170 * MALLOC_PAGEFILE causes all mmap()ed memory to be backed by temporary
|
|
171 * files, so that if a chunk is mapped, it is guaranteed to be swappable.
|
|
172 * This avoids asynchronous OOM failures that are due to VM over-commit.
|
|
173 *
|
|
174 * XXX OS X over-commits, so we should probably use mmap() instead of
|
|
175 * vm_allocate(), so that MALLOC_PAGEFILE works.
|
|
176 */
|
|
177 #define MALLOC_PAGEFILE
|
|
178 #endif
|
|
179
|
|
180 #ifdef MALLOC_PAGEFILE
|
|
181 /* Write size when initializing a page file. */
|
|
182 # define MALLOC_PAGEFILE_WRITE_SIZE 512
|
|
183 #endif
|
|
184
|
|
185 #ifdef MOZ_MEMORY_LINUX
|
|
186 #define _GNU_SOURCE /* For mremap(2). */
|
|
187 #define issetugid() 0
|
|
188 #if 0 /* Enable in order to test decommit code on Linux. */
|
|
189 # define MALLOC_DECOMMIT
|
|
190 #endif
|
|
191 #endif
|
|
192
|
|
193 #ifndef MOZ_MEMORY_WINCE
|
|
194 #include <sys/types.h>
|
|
195
|
|
196 #include <errno.h>
|
|
197 #include <stdlib.h>
|
|
198 #endif
|
|
199 #include <limits.h>
|
|
200 #include <stdarg.h>
|
|
201 #include <stdio.h>
|
|
202 #include <string.h>
|
|
203
|
|
204 #ifdef MOZ_MEMORY_WINDOWS
|
|
205 #ifndef MOZ_MEMORY_WINCE
|
|
206 #include <cruntime.h>
|
|
207 #include <internal.h>
|
|
208 #include <io.h>
|
|
209 #else
|
|
210 #include <crtdefs.h>
|
|
211 #define SIZE_MAX UINT_MAX
|
|
212 #endif
|
|
213 #include <windows.h>
|
|
214
|
|
215 #pragma warning( disable: 4267 4996 4146 )
|
|
216
|
|
217 #define false FALSE
|
|
218 #define true TRUE
|
|
219 #define inline __inline
|
|
220 #define SIZE_T_MAX SIZE_MAX
|
|
221 #define STDERR_FILENO 2
|
|
222 #define PATH_MAX MAX_PATH
|
|
223 #define vsnprintf _vsnprintf
|
|
224
|
|
225 #ifndef NO_TLS
|
|
226 static unsigned long tlsIndex = 0xffffffff;
|
|
227 #endif
|
|
228
|
|
229 #define __thread
|
|
230 #ifdef MOZ_MEMORY_WINCE
|
|
231 #define _pthread_self() GetCurrentThreadId()
|
|
232 #else
|
|
233 #define _pthread_self() __threadid()
|
|
234 #endif
|
|
235 #define issetugid() 0
|
|
236
|
|
237 #ifndef MOZ_MEMORY_WINCE
|
|
238 /* use MSVC intrinsics */
|
|
239 #pragma intrinsic(_BitScanForward)
|
|
240 static __forceinline int
|
|
241 ffs(int x)
|
|
242 {
|
|
243 unsigned long i;
|
|
244
|
|
245 if (_BitScanForward(&i, x) != 0)
|
|
246 return (i + 1);
|
|
247
|
|
248 return (0);
|
|
249 }
|
|
250
|
|
251 /* Implement getenv without using malloc */
|
|
252 static char mozillaMallocOptionsBuf[64];
|
|
253
|
|
254 #define getenv xgetenv
|
|
255 static char *
|
|
256 getenv(const char *name)
|
|
257 {
|
|
258
|
|
259 if (GetEnvironmentVariableA(name, (LPSTR)&mozillaMallocOptionsBuf,
|
|
260 sizeof(mozillaMallocOptionsBuf)) > 0)
|
|
261 return (mozillaMallocOptionsBuf);
|
|
262
|
|
263 return (NULL);
|
|
264 }
|
|
265 #else
|
|
266
|
|
267 static void abort() {
|
|
268 DebugBreak();
|
|
269 exit(-3);
|
|
270 }
|
|
271
|
|
272 static int errno = 0;
|
|
273 #define ENOMEM 12
|
|
274 #define EINVAL 22
|
|
275
|
|
276 static char *
|
|
277 getenv(const char *name)
|
|
278 {
|
|
279 return (NULL);
|
|
280 }
|
|
281
|
|
282 static int
|
|
283 ffs(int x)
|
|
284 {
|
|
285 int ret;
|
|
286
|
|
287 if (x == 0)
|
|
288 return 0;
|
|
289 ret = 2;
|
|
290 if ((x & 0x0000ffff) == 0) { ret += 16; x >>= 16;}
|
|
291 if ((x & 0x000000ff) == 0) { ret += 8; x >>= 8;}
|
|
292 if ((x & 0x0000000f) == 0) { ret += 4; x >>= 4;}
|
|
293 if ((x & 0x00000003) == 0) { ret += 2; x >>= 2;}
|
|
294 ret -= (x & 1);
|
|
295
|
|
296 return (ret);
|
|
297 }
|
|
298 #endif
|
|
299
|
|
300 typedef unsigned char uint8_t;
|
|
301 typedef unsigned uint32_t;
|
|
302 typedef unsigned long long uint64_t;
|
|
303 typedef unsigned long long uintmax_t;
|
|
304 typedef long ssize_t;
|
|
305
|
|
306 #define MALLOC_DECOMMIT
|
|
307 #endif
|
|
308
|
|
309 #ifndef MOZ_MEMORY_WINDOWS
|
|
310 #ifndef MOZ_MEMORY_SOLARIS
|
|
311 #include <sys/cdefs.h>
|
|
312 #endif
|
|
313 #ifndef __DECONST
|
|
314 # define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
|
|
315 #endif
|
|
316 #ifndef MOZ_MEMORY
|
|
317 __FBSDID("$FreeBSD: head/lib/libc/stdlib/malloc.c 180599 2008-07-18 19:35:44Z jasone $");
|
|
318 #include "libc_private.h"
|
|
319 #ifdef MALLOC_DEBUG
|
|
320 # define _LOCK_DEBUG
|
|
321 #endif
|
|
322 #include "spinlock.h"
|
|
323 #include "namespace.h"
|
|
324 #endif
|
|
325 #include <sys/mman.h>
|
|
326 #ifndef MADV_FREE
|
|
327 # define MADV_FREE MADV_DONTNEED
|
|
328 #endif
|
|
329 #ifndef MAP_NOSYNC
|
|
330 # define MAP_NOSYNC 0
|
|
331 #endif
|
|
332 #include <sys/param.h>
|
|
333 #ifndef MOZ_MEMORY
|
|
334 #include <sys/stddef.h>
|
|
335 #endif
|
|
336 #include <sys/time.h>
|
|
337 #include <sys/types.h>
|
|
338 #ifndef MOZ_MEMORY_SOLARIS
|
|
339 #include <sys/sysctl.h>
|
|
340 #endif
|
|
341 #include <sys/uio.h>
|
|
342 #ifndef MOZ_MEMORY
|
|
343 #include <sys/ktrace.h> /* Must come after several other sys/ includes. */
|
|
344
|
|
345 #include <machine/atomic.h>
|
|
346 #include <machine/cpufunc.h>
|
|
347 #include <machine/vmparam.h>
|
|
348 #endif
|
|
349
|
|
350 #include <errno.h>
|
|
351 #include <limits.h>
|
|
352 #ifndef SIZE_T_MAX
|
|
353 # define SIZE_T_MAX SIZE_MAX
|
|
354 #endif
|
|
355 #include <pthread.h>
|
|
356 #ifdef MOZ_MEMORY_DARWIN
|
|
357 #define _pthread_self pthread_self
|
|
358 #define _pthread_mutex_init pthread_mutex_init
|
|
359 #define _pthread_mutex_trylock pthread_mutex_trylock
|
|
360 #define _pthread_mutex_lock pthread_mutex_lock
|
|
361 #define _pthread_mutex_unlock pthread_mutex_unlock
|
|
362 #endif
|
|
363 #include <sched.h>
|
|
364 #include <stdarg.h>
|
|
365 #include <stdbool.h>
|
|
366 #include <stdio.h>
|
|
367 #include <stdint.h>
|
|
368 #include <stdlib.h>
|
|
369 #include <string.h>
|
|
370 #ifndef MOZ_MEMORY_DARWIN
|
|
371 #include <strings.h>
|
|
372 #endif
|
|
373 #include <unistd.h>
|
|
374
|
|
375 #ifdef MOZ_MEMORY_DARWIN
|
|
376 #include <libkern/OSAtomic.h>
|
|
377 #include <mach/mach_error.h>
|
|
378 #include <mach/mach_init.h>
|
|
379 #include <mach/vm_map.h>
|
|
380 #include <malloc/malloc.h>
|
|
381 #endif
|
|
382
|
|
383 #ifndef MOZ_MEMORY
|
|
384 #include "un-namespace.h"
|
|
385 #endif
|
|
386
|
|
387 #endif
|
|
388
|
|
389 #include "jemalloc.h"
|
|
390
|
|
391 #ifdef MOZ_MEMORY_DARWIN
|
|
392 static const bool __isthreaded = true;
|
|
393 #endif
|
|
394
|
|
395 #if defined(MOZ_MEMORY_SOLARIS) && defined(MAP_ALIGN) && !defined(JEMALLOC_NEVER_USES_MAP_ALIGN)
|
|
396 #define JEMALLOC_USES_MAP_ALIGN /* Required on Solaris 10. Might improve performance elsewhere. */
|
|
397 #endif
|
|
398
|
|
399 #if defined(MOZ_MEMORY_WINCE)
|
|
400 #define JEMALLOC_USES_MAP_ALIGN /* Required for Windows CE */
|
|
401 #endif
|
|
402
|
|
403 #define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
|
|
404
|
|
405 #include "qr.h"
|
|
406 #include "ql.h"
|
|
407 #ifdef MOZ_MEMORY_WINDOWS
|
|
408 /* MSVC++ does not support C99 variable-length arrays. */
|
|
409 # define RB_NO_C99_VARARRAYS
|
|
410 #endif
|
|
411 #include "rb.h"
|
|
412
|
|
413 #ifdef MALLOC_DEBUG
|
|
414 /* Disable inlining to make debugging easier. */
|
|
415 #ifdef inline
|
|
416 #undef inline
|
|
417 #endif
|
|
418
|
|
419 # define inline
|
|
420 #endif
|
|
421
|
|
422 /* Size of stack-allocated buffer passed to strerror_r(). */
|
|
423 #define STRERROR_BUF 64
|
|
424
|
|
425 /* Minimum alignment of allocations is 2^QUANTUM_2POW_MIN bytes. */
|
|
426 # define QUANTUM_2POW_MIN 4
|
|
427 #ifdef MOZ_MEMORY_SIZEOF_PTR_2POW
|
|
428 # define SIZEOF_PTR_2POW MOZ_MEMORY_SIZEOF_PTR_2POW
|
|
429 #else
|
|
430 # define SIZEOF_PTR_2POW 2
|
|
431 #endif
|
|
432 #define PIC
|
|
433 #ifndef MOZ_MEMORY_DARWIN
|
|
434 static const bool __isthreaded = true;
|
|
435 #else
|
|
436 # define NO_TLS
|
|
437 #endif
|
|
438 #if 0
|
|
439 #ifdef __i386__
|
|
440 # define QUANTUM_2POW_MIN 4
|
|
441 # define SIZEOF_PTR_2POW 2
|
|
442 # define CPU_SPINWAIT __asm__ volatile("pause")
|
|
443 #endif
|
|
444 #ifdef __ia64__
|
|
445 # define QUANTUM_2POW_MIN 4
|
|
446 # define SIZEOF_PTR_2POW 3
|
|
447 #endif
|
|
448 #ifdef __alpha__
|
|
449 # define QUANTUM_2POW_MIN 4
|
|
450 # define SIZEOF_PTR_2POW 3
|
|
451 # define NO_TLS
|
|
452 #endif
|
|
453 #ifdef __sparc64__
|
|
454 # define QUANTUM_2POW_MIN 4
|
|
455 # define SIZEOF_PTR_2POW 3
|
|
456 # define NO_TLS
|
|
457 #endif
|
|
458 #ifdef __amd64__
|
|
459 # define QUANTUM_2POW_MIN 4
|
|
460 # define SIZEOF_PTR_2POW 3
|
|
461 # define CPU_SPINWAIT __asm__ volatile("pause")
|
|
462 #endif
|
|
463 #ifdef __arm__
|
|
464 # define QUANTUM_2POW_MIN 3
|
|
465 # define SIZEOF_PTR_2POW 2
|
|
466 # define NO_TLS
|
|
467 #endif
|
|
468 #ifdef __mips__
|
|
469 # define QUANTUM_2POW_MIN 3
|
|
470 # define SIZEOF_PTR_2POW 2
|
|
471 # define NO_TLS
|
|
472 #endif
|
|
473 #ifdef __powerpc__
|
|
474 # define QUANTUM_2POW_MIN 4
|
|
475 # define SIZEOF_PTR_2POW 2
|
|
476 #endif
|
|
477 #endif
|
|
478
|
|
479 #define SIZEOF_PTR (1U << SIZEOF_PTR_2POW)
|
|
480
|
|
481 /* sizeof(int) == (1U << SIZEOF_INT_2POW). */
|
|
482 #ifndef SIZEOF_INT_2POW
|
|
483 # define SIZEOF_INT_2POW 2
|
|
484 #endif
|
|
485
|
|
486 /* We can't use TLS in non-PIC programs, since TLS relies on loader magic. */
|
|
487 #if (!defined(PIC) && !defined(NO_TLS))
|
|
488 # define NO_TLS
|
|
489 #endif
|
|
490
|
|
491 #ifdef NO_TLS
|
|
492 /* MALLOC_BALANCE requires TLS. */
|
|
493 # ifdef MALLOC_BALANCE
|
|
494 # undef MALLOC_BALANCE
|
|
495 # endif
|
|
496 #endif
|
|
497
|
|
498 /*
|
|
499 * Size and alignment of memory chunks that are allocated by the OS's virtual
|
|
500 * memory system.
|
|
501 */
|
|
502 #ifdef MOZ_MEMORY_WINCE
|
|
503 #define CHUNK_2POW_DEFAULT 21
|
|
504 #else
|
|
505 #define CHUNK_2POW_DEFAULT 20
|
|
506 #endif
|
|
507 /* Maximum number of dirty pages per arena. */
|
|
508 #define DIRTY_MAX_DEFAULT (1U << 10)
|
|
509
|
|
510 /* Default reserve chunks. */
|
|
511 #define RESERVE_MIN_2POW_DEFAULT 1
|
|
512 /*
|
|
513 * Default range (in chunks) between reserve_min and reserve_max, in addition
|
|
514 * to the mandatory one chunk per arena.
|
|
515 */
|
|
516 #ifdef MALLOC_PAGEFILE
|
|
517 # define RESERVE_RANGE_2POW_DEFAULT 5
|
|
518 #else
|
|
519 # define RESERVE_RANGE_2POW_DEFAULT 0
|
|
520 #endif
|
|
521
|
|
522 /*
|
|
523 * Maximum size of L1 cache line. This is used to avoid cache line aliasing,
|
|
524 * so over-estimates are okay (up to a point), but under-estimates will
|
|
525 * negatively affect performance.
|
|
526 */
|
|
527 #define CACHELINE_2POW 6
|
|
528 #define CACHELINE ((size_t)(1U << CACHELINE_2POW))
|
|
529
|
|
530 /* Smallest size class to support. */
|
|
531 #define TINY_MIN_2POW 1
|
|
532
|
|
533 /*
|
|
534 * Maximum size class that is a multiple of the quantum, but not (necessarily)
|
|
535 * a power of 2. Above this size, allocations are rounded up to the nearest
|
|
536 * power of 2.
|
|
537 */
|
|
538 #define SMALL_MAX_2POW_DEFAULT 9
|
|
539 #define SMALL_MAX_DEFAULT (1U << SMALL_MAX_2POW_DEFAULT)
|
|
540
|
|
541 /*
|
|
542 * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized
|
|
543 * as small as possible such that this setting is still honored, without
|
|
544 * violating other constraints. The goal is to make runs as small as possible
|
|
545 * without exceeding a per run external fragmentation threshold.
|
|
546 *
|
|
547 * We use binary fixed point math for overhead computations, where the binary
|
|
548 * point is implicitly RUN_BFP bits to the left.
|
|
549 *
|
|
550 * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be
|
|
551 * honored for some/all object sizes, since there is one bit of header overhead
|
|
552 * per object (plus a constant). This constraint is relaxed (ignored) for runs
|
|
553 * that are so small that the per-region overhead is greater than:
|
|
554 *
|
|
555 * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
|
|
556 */
|
|
557 #define RUN_BFP 12
|
|
558 /* \/ Implicit binary fixed point. */
|
|
559 #define RUN_MAX_OVRHD 0x0000003dU
|
|
560 #define RUN_MAX_OVRHD_RELAX 0x00001800U
|
|
561
|
|
562 /* Put a cap on small object run size. This overrides RUN_MAX_OVRHD. */
|
|
563 #define RUN_MAX_SMALL_2POW 15
|
|
564 #define RUN_MAX_SMALL (1U << RUN_MAX_SMALL_2POW)
|
|
565
|
|
566 /*
|
|
567 * Hyper-threaded CPUs may need a special instruction inside spin loops in
|
|
568 * order to yield to another virtual CPU. If no such instruction is defined
|
|
569 * above, make CPU_SPINWAIT a no-op.
|
|
570 */
|
|
571 #ifndef CPU_SPINWAIT
|
|
572 # define CPU_SPINWAIT
|
|
573 #endif
|
|
574
|
|
575 /*
|
|
576 * Adaptive spinning must eventually switch to blocking, in order to avoid the
|
|
577 * potential for priority inversion deadlock. Backing off past a certain point
|
|
578 * can actually waste time.
|
|
579 */
|
|
580 #define SPIN_LIMIT_2POW 11
|
|
581
|
|
582 /*
|
|
583 * Conversion from spinning to blocking is expensive; we use (1U <<
|
|
584 * BLOCK_COST_2POW) to estimate how many more times costly blocking is than
|
|
585 * worst-case spinning.
|
|
586 */
|
|
587 #define BLOCK_COST_2POW 4
|
|
588
|
|
589 #ifdef MALLOC_BALANCE
|
|
590 /*
|
|
591 * We use an exponential moving average to track recent lock contention,
|
|
592 * where the size of the history window is N, and alpha=2/(N+1).
|
|
593 *
|
|
594 * Due to integer math rounding, very small values here can cause
|
|
595 * substantial degradation in accuracy, thus making the moving average decay
|
|
596 * faster than it would with precise calculation.
|
|
597 */
|
|
598 # define BALANCE_ALPHA_INV_2POW 9
|
|
599
|
|
600 /*
|
|
601 * Threshold value for the exponential moving contention average at which to
|
|
602 * re-assign a thread.
|
|
603 */
|
|
604 # define BALANCE_THRESHOLD_DEFAULT (1U << (SPIN_LIMIT_2POW-4))
|
|
605 #endif
|
|
606
|
|
607 /******************************************************************************/
|
|
608
|
|
609 /*
|
|
610 * Mutexes based on spinlocks. We can't use normal pthread spinlocks in all
|
|
611 * places, because they require malloc()ed memory, which causes bootstrapping
|
|
612 * issues in some cases.
|
|
613 */
|
|
614 #if defined(MOZ_MEMORY_WINDOWS)
|
|
615 #define malloc_mutex_t CRITICAL_SECTION
|
|
616 #define malloc_spinlock_t CRITICAL_SECTION
|
|
617 #elif defined(MOZ_MEMORY_DARWIN)
|
|
618 typedef struct {
|
|
619 OSSpinLock lock;
|
|
620 } malloc_mutex_t;
|
|
621 typedef struct {
|
|
622 OSSpinLock lock;
|
|
623 } malloc_spinlock_t;
|
|
624 #elif defined(MOZ_MEMORY)
|
|
625 typedef pthread_mutex_t malloc_mutex_t;
|
|
626 typedef pthread_mutex_t malloc_spinlock_t;
|
|
627 #else
|
|
628 /* XXX these should #ifdef these for freebsd (and linux?) only */
|
|
629 typedef struct {
|
|
630 spinlock_t lock;
|
|
631 } malloc_mutex_t;
|
|
632 typedef malloc_spinlock_t malloc_mutex_t;
|
|
633 #endif
|
|
634
|
|
635 /* Set to true once the allocator has been initialized. */
|
|
636 static bool malloc_initialized = false;
|
|
637
|
|
638 #if defined(MOZ_MEMORY_WINDOWS)
|
|
639 /* No init lock for Windows. */
|
|
640 #elif defined(MOZ_MEMORY_DARWIN)
|
|
641 static malloc_mutex_t init_lock = {OS_SPINLOCK_INIT};
|
|
642 #elif defined(MOZ_MEMORY_LINUX)
|
|
643 static malloc_mutex_t init_lock = PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP;
|
|
644 #elif defined(MOZ_MEMORY)
|
|
645 static malloc_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
|
|
646 #else
|
|
647 static malloc_mutex_t init_lock = {_SPINLOCK_INITIALIZER};
|
|
648 #endif
|
|
649
|
|
650 /******************************************************************************/
|
|
651 /*
|
|
652 * Statistics data structures.
|
|
653 */
|
|
654
|
|
655 #ifdef MALLOC_STATS
|
|
656
|
|
657 typedef struct malloc_bin_stats_s malloc_bin_stats_t;
|
|
658 struct malloc_bin_stats_s {
|
|
659 /*
|
|
660 * Number of allocation requests that corresponded to the size of this
|
|
661 * bin.
|
|
662 */
|
|
663 uint64_t nrequests;
|
|
664
|
|
665 /* Total number of runs created for this bin's size class. */
|
|
666 uint64_t nruns;
|
|
667
|
|
668 /*
|
|
669 * Total number of runs reused by extracting them from the runs tree for
|
|
670 * this bin's size class.
|
|
671 */
|
|
672 uint64_t reruns;
|
|
673
|
|
674 /* High-water mark for this bin. */
|
|
675 unsigned long highruns;
|
|
676
|
|
677 /* Current number of runs in this bin. */
|
|
678 unsigned long curruns;
|
|
679 };
|
|
680
|
|
681 typedef struct arena_stats_s arena_stats_t;
|
|
682 struct arena_stats_s {
|
|
683 /* Number of bytes currently mapped. */
|
|
684 size_t mapped;
|
|
685
|
|
686 /*
|
|
687 * Total number of purge sweeps, total number of madvise calls made,
|
|
688 * and total pages purged in order to keep dirty unused memory under
|
|
689 * control.
|
|
690 */
|
|
691 uint64_t npurge;
|
|
692 uint64_t nmadvise;
|
|
693 uint64_t purged;
|
|
694 #ifdef MALLOC_DECOMMIT
|
|
695 /*
|
|
696 * Total number of decommit/commit operations, and total number of
|
|
697 * pages decommitted.
|
|
698 */
|
|
699 uint64_t ndecommit;
|
|
700 uint64_t ncommit;
|
|
701 uint64_t decommitted;
|
|
702 #endif
|
|
703
|
|
704 /* Per-size-category statistics. */
|
|
705 size_t allocated_small;
|
|
706 uint64_t nmalloc_small;
|
|
707 uint64_t ndalloc_small;
|
|
708
|
|
709 size_t allocated_large;
|
|
710 uint64_t nmalloc_large;
|
|
711 uint64_t ndalloc_large;
|
|
712
|
|
713 #ifdef MALLOC_BALANCE
|
|
714 /* Number of times this arena reassigned a thread due to contention. */
|
|
715 uint64_t nbalance;
|
|
716 #endif
|
|
717 };
|
|
718
|
|
719 typedef struct chunk_stats_s chunk_stats_t;
|
|
720 struct chunk_stats_s {
|
|
721 /* Number of chunks that were allocated. */
|
|
722 uint64_t nchunks;
|
|
723
|
|
724 /* High-water mark for number of chunks allocated. */
|
|
725 unsigned long highchunks;
|
|
726
|
|
727 /*
|
|
728 * Current number of chunks allocated. This value isn't maintained for
|
|
729 * any other purpose, so keep track of it in order to be able to set
|
|
730 * highchunks.
|
|
731 */
|
|
732 unsigned long curchunks;
|
|
733 };
|
|
734
|
|
735 #endif /* #ifdef MALLOC_STATS */
|
|
736
|
|
737 /******************************************************************************/
|
|
738 /*
|
|
739 * Extent data structures.
|
|
740 */
|
|
741
|
|
742 /* Tree of extents. */
|
|
743 typedef struct extent_node_s extent_node_t;
|
|
744 struct extent_node_s {
|
|
745 /* Linkage for the size/address-ordered tree. */
|
|
746 rb_node(extent_node_t) link_szad;
|
|
747
|
|
748 /* Linkage for the address-ordered tree. */
|
|
749 rb_node(extent_node_t) link_ad;
|
|
750
|
|
751 /* Pointer to the extent that this tree node is responsible for. */
|
|
752 void *addr;
|
|
753
|
|
754 /* Total region size. */
|
|
755 size_t size;
|
|
756 };
|
|
757 typedef rb_tree(extent_node_t) extent_tree_t;
|
|
758
|
|
759 /******************************************************************************/
|
|
760 /*
|
|
761 * Radix tree data structures.
|
|
762 */
|
|
763
|
|
764 #ifdef MALLOC_VALIDATE
|
|
765 /*
|
|
766 * Size of each radix tree node (must be a power of 2). This impacts tree
|
|
767 * depth.
|
|
768 */
|
|
769 # if (SIZEOF_PTR == 4)
|
|
770 # define MALLOC_RTREE_NODESIZE (1U << 14)
|
|
771 # else
|
|
772 # define MALLOC_RTREE_NODESIZE CACHELINE
|
|
773 # endif
|
|
774
|
|
775 typedef struct malloc_rtree_s malloc_rtree_t;
|
|
776 struct malloc_rtree_s {
|
|
777 malloc_spinlock_t lock;
|
|
778 void **root;
|
|
779 unsigned height;
|
|
780 unsigned level2bits[1]; /* Dynamically sized. */
|
|
781 };
|
|
782 #endif
|
|
783
|
|
784 /******************************************************************************/
|
|
785 /*
|
|
786 * Reserve data structures.
|
|
787 */
|
|
788
|
|
789 /* Callback registration. */
|
|
790 typedef struct reserve_reg_s reserve_reg_t;
|
|
791 struct reserve_reg_s {
|
|
792 /* Linkage for list of all registered callbacks. */
|
|
793 ql_elm(reserve_reg_t) link;
|
|
794
|
|
795 /* Callback function pointer. */
|
|
796 reserve_cb_t *cb;
|
|
797
|
|
798 /* Opaque application data pointer. */
|
|
799 void *ctx;
|
|
800
|
|
801 /*
|
|
802 * Sequence number of condition notification most recently sent to this
|
|
803 * callback.
|
|
804 */
|
|
805 uint64_t seq;
|
|
806 };
|
|
807
|
|
808 /******************************************************************************/
|
|
809 /*
|
|
810 * Arena data structures.
|
|
811 */
|
|
812
|
|
813 typedef struct arena_s arena_t;
|
|
814 typedef struct arena_bin_s arena_bin_t;
|
|
815
|
|
816 /* Each element of the chunk map corresponds to one page within the chunk. */
|
|
817 typedef struct arena_chunk_map_s arena_chunk_map_t;
|
|
818 struct arena_chunk_map_s {
|
|
819 /*
|
|
820 * Linkage for run trees. There are two disjoint uses:
|
|
821 *
|
|
822 * 1) arena_t's runs_avail tree.
|
|
823 * 2) arena_run_t conceptually uses this linkage for in-use non-full
|
|
824 * runs, rather than directly embedding linkage.
|
|
825 */
|
|
826 rb_node(arena_chunk_map_t) link;
|
|
827
|
|
828 /*
|
|
829 * Run address (or size) and various flags are stored together. The bit
|
|
830 * layout looks like (assuming 32-bit system):
|
|
831 *
|
|
832 * ???????? ???????? ????---- --ckdzla
|
|
833 *
|
|
834 * ? : Unallocated: Run address for first/last pages, unset for internal
|
|
835 * pages.
|
|
836 * Small: Run address.
|
|
837 * Large: Run size for first page, unset for trailing pages.
|
|
838 * - : Unused.
|
|
839 * c : decommitted?
|
|
840 * k : key?
|
|
841 * d : dirty?
|
|
842 * z : zeroed?
|
|
843 * l : large?
|
|
844 * a : allocated?
|
|
845 *
|
|
846 * Following are example bit patterns for the three types of runs.
|
|
847 *
|
|
848 * r : run address
|
|
849 * s : run size
|
|
850 * x : don't care
|
|
851 * - : 0
|
|
852 * [cdzla] : bit set
|
|
853 *
|
|
854 * Unallocated:
|
|
855 * ssssssss ssssssss ssss---- --c-----
|
|
856 * xxxxxxxx xxxxxxxx xxxx---- ----d---
|
|
857 * ssssssss ssssssss ssss---- -----z--
|
|
858 *
|
|
859 * Small:
|
|
860 * rrrrrrrr rrrrrrrr rrrr---- -------a
|
|
861 * rrrrrrrr rrrrrrrr rrrr---- -------a
|
|
862 * rrrrrrrr rrrrrrrr rrrr---- -------a
|
|
863 *
|
|
864 * Large:
|
|
865 * ssssssss ssssssss ssss---- ------la
|
|
866 * -------- -------- -------- ------la
|
|
867 * -------- -------- -------- ------la
|
|
868 */
|
|
869 size_t bits;
|
|
870 #ifdef MALLOC_DECOMMIT
|
|
871 #define CHUNK_MAP_DECOMMITTED ((size_t)0x20U)
|
|
872 #endif
|
|
873 #define CHUNK_MAP_KEY ((size_t)0x10U)
|
|
874 #define CHUNK_MAP_DIRTY ((size_t)0x08U)
|
|
875 #define CHUNK_MAP_ZEROED ((size_t)0x04U)
|
|
876 #define CHUNK_MAP_LARGE ((size_t)0x02U)
|
|
877 #define CHUNK_MAP_ALLOCATED ((size_t)0x01U)
|
|
878 };
|
|
879 typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
|
|
880 typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
|
|
881
|
|
882 /* Arena chunk header. */
|
|
883 typedef struct arena_chunk_s arena_chunk_t;
|
|
884 struct arena_chunk_s {
|
|
885 /* Arena that owns the chunk. */
|
|
886 arena_t *arena;
|
|
887
|
|
888 /* Linkage for the arena's chunks_dirty tree. */
|
|
889 rb_node(arena_chunk_t) link_dirty;
|
|
890
|
|
891 /* Number of dirty pages. */
|
|
892 size_t ndirty;
|
|
893
|
|
894 /* Map of pages within chunk that keeps track of free/large/small. */
|
|
895 arena_chunk_map_t map[1]; /* Dynamically sized. */
|
|
896 };
|
|
897 typedef rb_tree(arena_chunk_t) arena_chunk_tree_t;
|
|
898
|
|
899 typedef struct arena_run_s arena_run_t;
|
|
900 struct arena_run_s {
|
|
901 #ifdef MALLOC_DEBUG
|
|
902 uint32_t magic;
|
|
903 # define ARENA_RUN_MAGIC 0x384adf93
|
|
904 #endif
|
|
905
|
|
906 /* Bin this run is associated with. */
|
|
907 arena_bin_t *bin;
|
|
908
|
|
909 /* Index of first element that might have a free region. */
|
|
910 unsigned regs_minelm;
|
|
911
|
|
912 /* Number of free regions in run. */
|
|
913 unsigned nfree;
|
|
914
|
|
915 /* Bitmask of in-use regions (0: in use, 1: free). */
|
|
916 unsigned regs_mask[1]; /* Dynamically sized. */
|
|
917 };
|
|
918
|
|
919 struct arena_bin_s {
|
|
920 /*
|
|
921 * Current run being used to service allocations of this bin's size
|
|
922 * class.
|
|
923 */
|
|
924 arena_run_t *runcur;
|
|
925
|
|
926 /*
|
|
927 * Tree of non-full runs. This tree is used when looking for an
|
|
928 * existing run when runcur is no longer usable. We choose the
|
|
929 * non-full run that is lowest in memory; this policy tends to keep
|
|
930 * objects packed well, and it can also help reduce the number of
|
|
931 * almost-empty chunks.
|
|
932 */
|
|
933 arena_run_tree_t runs;
|
|
934
|
|
935 /* Size of regions in a run for this bin's size class. */
|
|
936 size_t reg_size;
|
|
937
|
|
938 /* Total size of a run for this bin's size class. */
|
|
939 size_t run_size;
|
|
940
|
|
941 /* Total number of regions in a run for this bin's size class. */
|
|
942 uint32_t nregs;
|
|
943
|
|
944 /* Number of elements in a run's regs_mask for this bin's size class. */
|
|
945 uint32_t regs_mask_nelms;
|
|
946
|
|
947 /* Offset of first region in a run for this bin's size class. */
|
|
948 uint32_t reg0_offset;
|
|
949
|
|
950 #ifdef MALLOC_STATS
|
|
951 /* Bin statistics. */
|
|
952 malloc_bin_stats_t stats;
|
|
953 #endif
|
|
954 };
|
|
955
|
|
956 struct arena_s {
|
|
957 #ifdef MALLOC_DEBUG
|
|
958 uint32_t magic;
|
|
959 # define ARENA_MAGIC 0x947d3d24
|
|
960 #endif
|
|
961
|
|
962 /* All operations on this arena require that lock be locked. */
|
|
963 #ifdef MOZ_MEMORY
|
|
964 malloc_spinlock_t lock;
|
|
965 #else
|
|
966 pthread_mutex_t lock;
|
|
967 #endif
|
|
968
|
|
969 #ifdef MALLOC_STATS
|
|
970 arena_stats_t stats;
|
|
971 #endif
|
|
972
|
|
973 /*
|
|
974 * Chunk allocation sequence number, used to detect races with other
|
|
975 * threads during chunk allocation, and then discard unnecessary chunks.
|
|
976 */
|
|
977 uint64_t chunk_seq;
|
|
978
|
|
979 /* Tree of dirty-page-containing chunks this arena manages. */
|
|
980 arena_chunk_tree_t chunks_dirty;
|
|
981
|
|
982 /*
|
|
983 * In order to avoid rapid chunk allocation/deallocation when an arena
|
|
984 * oscillates right on the cusp of needing a new chunk, cache the most
|
|
985 * recently freed chunk. The spare is left in the arena's chunk trees
|
|
986 * until it is deleted.
|
|
987 *
|
|
988 * There is one spare chunk per arena, rather than one spare total, in
|
|
989 * order to avoid interactions between multiple threads that could make
|
|
990 * a single spare inadequate.
|
|
991 */
|
|
992 arena_chunk_t *spare;
|
|
993
|
|
994 /*
|
|
995 * Current count of pages within unused runs that are potentially
|
|
996 * dirty, and for which madvise(... MADV_FREE) has not been called. By
|
|
997 * tracking this, we can institute a limit on how much dirty unused
|
|
998 * memory is mapped for each arena.
|
|
999 */
|
|
1000 size_t ndirty;
|
|
1001
|
|
1002 /*
|
|
1003 * Size/address-ordered tree of this arena's available runs. This tree
|
|
1004 * is used for first-best-fit run allocation.
|
|
1005 */
|
|
1006 arena_avail_tree_t runs_avail;
|
|
1007
|
|
1008 #ifdef MALLOC_BALANCE
|
|
1009 /*
|
|
1010 * The arena load balancing machinery needs to keep track of how much
|
|
1011 * lock contention there is. This value is exponentially averaged.
|
|
1012 */
|
|
1013 uint32_t contention;
|
|
1014 #endif
|
|
1015
|
|
1016 /*
|
|
1017 * bins is used to store rings of free regions of the following sizes,
|
|
1018 * assuming a 16-byte quantum, 4kB pagesize, and default MALLOC_OPTIONS.
|
|
1019 *
|
|
1020 * bins[i] | size |
|
|
1021 * --------+------+
|
|
1022 * 0 | 2 |
|
|
1023 * 1 | 4 |
|
|
1024 * 2 | 8 |
|
|
1025 * --------+------+
|
|
1026 * 3 | 16 |
|
|
1027 * 4 | 32 |
|
|
1028 * 5 | 48 |
|
|
1029 * 6 | 64 |
|
|
1030 * : :
|
|
1031 * : :
|
|
1032 * 33 | 496 |
|
|
1033 * 34 | 512 |
|
|
1034 * --------+------+
|
|
1035 * 35 | 1024 |
|
|
1036 * 36 | 2048 |
|
|
1037 * --------+------+
|
|
1038 */
|
|
1039 arena_bin_t bins[1]; /* Dynamically sized. */
|
|
1040 };
|
|
1041
|
|
1042 /******************************************************************************/
|
|
1043 /*
|
|
1044 * Data.
|
|
1045 */
|
|
1046
|
|
1047 /* Number of CPUs. */
|
|
1048 static unsigned ncpus;
|
|
1049
|
|
1050 /* VM page size. */
|
|
1051 static size_t pagesize;
|
|
1052 static size_t pagesize_mask;
|
|
1053 static size_t pagesize_2pow;
|
|
1054
|
|
1055 /* Various bin-related settings. */
|
|
1056 static size_t bin_maxclass; /* Max size class for bins. */
|
|
1057 static unsigned ntbins; /* Number of (2^n)-spaced tiny bins. */
|
|
1058 static unsigned nqbins; /* Number of quantum-spaced bins. */
|
|
1059 static unsigned nsbins; /* Number of (2^n)-spaced sub-page bins. */
|
|
1060 static size_t small_min;
|
|
1061 static size_t small_max;
|
|
1062
|
|
1063 /* Various quantum-related settings. */
|
|
1064 static size_t quantum;
|
|
1065 static size_t quantum_mask; /* (quantum - 1). */
|
|
1066
|
|
1067 /* Various chunk-related settings. */
|
|
1068 static size_t chunksize;
|
|
1069 static size_t chunksize_mask; /* (chunksize - 1). */
|
|
1070 static size_t chunk_npages;
|
|
1071 static size_t arena_chunk_header_npages;
|
|
1072 static size_t arena_maxclass; /* Max size class for arenas. */
|
|
1073
|
|
1074 /********/
|
|
1075 /*
|
|
1076 * Chunks.
|
|
1077 */
|
|
1078
|
|
1079 #ifdef MALLOC_VALIDATE
|
|
1080 static malloc_rtree_t *chunk_rtree;
|
|
1081 #endif
|
|
1082
|
|
1083 /* Protects chunk-related data structures. */
|
|
1084 static malloc_mutex_t huge_mtx;
|
|
1085
|
|
1086 /* Tree of chunks that are stand-alone huge allocations. */
|
|
1087 static extent_tree_t huge;
|
|
1088
|
|
1089 #ifdef MALLOC_STATS
|
|
1090 /* Huge allocation statistics. */
|
|
1091 static uint64_t huge_nmalloc;
|
|
1092 static uint64_t huge_ndalloc;
|
|
1093 static size_t huge_allocated;
|
|
1094 #endif
|
|
1095
|
|
1096 /****************/
|
|
1097 /*
|
|
1098 * Memory reserve.
|
|
1099 */
|
|
1100
|
|
1101 #ifdef MALLOC_PAGEFILE
|
|
1102 static char pagefile_templ[PATH_MAX];
|
|
1103 #endif
|
|
1104
|
|
1105 /* Protects reserve-related data structures. */
|
|
1106 static malloc_mutex_t reserve_mtx;
|
|
1107
|
|
1108 /*
|
|
1109 * Bounds on acceptable reserve size, and current reserve size. Reserve
|
|
1110 * depletion may cause (reserve_cur < reserve_min).
|
|
1111 */
|
|
1112 static size_t reserve_min;
|
|
1113 static size_t reserve_cur;
|
|
1114 static size_t reserve_max;
|
|
1115
|
|
1116 /* List of registered callbacks. */
|
|
1117 static ql_head(reserve_reg_t) reserve_regs;
|
|
1118
|
|
1119 /*
|
|
1120 * Condition notification sequence number, used to determine whether all
|
|
1121 * registered callbacks have been notified of the most current condition.
|
|
1122 */
|
|
1123 static uint64_t reserve_seq;
|
|
1124
|
|
1125 /*
|
|
1126 * Trees of chunks currently in the memory reserve. Depending on function,
|
|
1127 * different tree orderings are needed, which is why there are two trees with
|
|
1128 * the same contents.
|
|
1129 */
|
|
1130 static extent_tree_t reserve_chunks_szad;
|
|
1131 static extent_tree_t reserve_chunks_ad;
|
|
1132
|
|
1133 /****************************/
|
|
1134 /*
|
|
1135 * base (internal allocation).
|
|
1136 */
|
|
1137
|
|
1138 /*
|
|
1139 * Current pages that are being used for internal memory allocations. These
|
|
1140 * pages are carved up in cacheline-size quanta, so that there is no chance of
|
|
1141 * false cache line sharing.
|
|
1142 */
|
|
1143 static void *base_pages;
|
|
1144 static void *base_next_addr;
|
|
1145 #ifdef MALLOC_DECOMMIT
|
|
1146 static void *base_next_decommitted;
|
|
1147 #endif
|
|
1148 static void *base_past_addr; /* Addr immediately past base_pages. */
|
|
1149 static extent_node_t *base_nodes;
|
|
1150 static reserve_reg_t *base_reserve_regs;
|
|
1151 static malloc_mutex_t base_mtx;
|
|
1152 #ifdef MALLOC_STATS
|
|
1153 static size_t base_mapped;
|
|
1154 #endif
|
|
1155
|
|
1156 /********/
|
|
1157 /*
|
|
1158 * Arenas.
|
|
1159 */
|
|
1160
|
|
1161 /*
|
|
1162 * Arenas that are used to service external requests. Not all elements of the
|
|
1163 * arenas array are necessarily used; arenas are created lazily as needed.
|
|
1164 */
|
|
1165 static arena_t **arenas;
|
|
1166 static unsigned narenas;
|
|
1167 static unsigned narenas_2pow;
|
|
1168 #ifndef NO_TLS
|
|
1169 # ifdef MALLOC_BALANCE
|
|
1170 static unsigned narenas_2pow;
|
|
1171 # else
|
|
1172 static unsigned next_arena;
|
|
1173 # endif
|
|
1174 #endif
|
|
1175 #ifdef MOZ_MEMORY
|
|
1176 static malloc_spinlock_t arenas_lock; /* Protects arenas initialization. */
|
|
1177 #else
|
|
1178 static pthread_mutex_t arenas_lock; /* Protects arenas initialization. */
|
|
1179 #endif
|
|
1180
|
|
1181 #ifndef NO_TLS
|
|
1182 /*
|
|
1183 * Map of pthread_self() --> arenas[???], used for selecting an arena to use
|
|
1184 * for allocations.
|
|
1185 */
|
|
1186 #ifndef MOZ_MEMORY_WINDOWS
|
|
1187 static __thread arena_t *arenas_map;
|
|
1188 #endif
|
|
1189 #endif
|
|
1190
|
|
1191 #ifdef MALLOC_STATS
|
|
1192 /* Chunk statistics. */
|
|
1193 static chunk_stats_t stats_chunks;
|
|
1194 #endif
|
|
1195
|
|
1196 /*******************************/
|
|
1197 /*
|
|
1198 * Runtime configuration options.
|
|
1199 */
|
|
1200 const char *_malloc_options;
|
|
1201
|
|
1202 #ifndef MALLOC_PRODUCTION
|
|
1203 static bool opt_abort = true;
|
|
1204 #ifdef MALLOC_FILL
|
|
1205 static bool opt_junk = true;
|
|
1206 #endif
|
|
1207 #else
|
|
1208 static bool opt_abort = false;
|
|
1209 #ifdef MALLOC_FILL
|
|
1210 static bool opt_junk = false;
|
|
1211 #endif
|
|
1212 #endif
|
|
1213 static size_t opt_dirty_max = DIRTY_MAX_DEFAULT;
|
|
1214 #ifdef MALLOC_BALANCE
|
|
1215 static uint64_t opt_balance_threshold = BALANCE_THRESHOLD_DEFAULT;
|
|
1216 #endif
|
|
1217 static bool opt_print_stats = false;
|
|
1218 static size_t opt_quantum_2pow = QUANTUM_2POW_MIN;
|
|
1219 static size_t opt_small_max_2pow = SMALL_MAX_2POW_DEFAULT;
|
|
1220 static size_t opt_chunk_2pow = CHUNK_2POW_DEFAULT;
|
|
1221 static int opt_reserve_min_lshift = 0;
|
|
1222 static int opt_reserve_range_lshift = 0;
|
|
1223 #ifdef MALLOC_PAGEFILE
|
|
1224 static bool opt_pagefile = false;
|
|
1225 #endif
|
|
1226 #ifdef MALLOC_UTRACE
|
|
1227 static bool opt_utrace = false;
|
|
1228 #endif
|
|
1229 #ifdef MALLOC_SYSV
|
|
1230 static bool opt_sysv = false;
|
|
1231 #endif
|
|
1232 #ifdef MALLOC_XMALLOC
|
|
1233 static bool opt_xmalloc = false;
|
|
1234 #endif
|
|
1235 #ifdef MALLOC_FILL
|
|
1236 static bool opt_zero = false;
|
|
1237 #endif
|
|
1238 static int opt_narenas_lshift = 0;
|
|
1239
|
|
1240 #ifdef MALLOC_UTRACE
|
|
1241 typedef struct {
|
|
1242 void *p;
|
|
1243 size_t s;
|
|
1244 void *r;
|
|
1245 } malloc_utrace_t;
|
|
1246
|
|
1247 #define UTRACE(a, b, c) \
|
|
1248 if (opt_utrace) { \
|
|
1249 malloc_utrace_t ut; \
|
|
1250 ut.p = (a); \
|
|
1251 ut.s = (b); \
|
|
1252 ut.r = (c); \
|
|
1253 utrace(&ut, sizeof(ut)); \
|
|
1254 }
|
|
1255 #else
|
|
1256 #define UTRACE(a, b, c)
|
|
1257 #endif
|
|
1258
|
|
1259 /******************************************************************************/
|
|
1260 /*
|
|
1261 * Begin function prototypes for non-inline static functions.
|
|
1262 */
|
|
1263
|
|
1264 static char *umax2s(uintmax_t x, char *s);
|
|
1265 static bool malloc_mutex_init(malloc_mutex_t *mutex);
|
|
1266 static bool malloc_spin_init(malloc_spinlock_t *lock);
|
|
1267 static void wrtmessage(const char *p1, const char *p2, const char *p3,
|
|
1268 const char *p4);
|
|
1269 #ifdef MALLOC_STATS
|
|
1270 #ifdef MOZ_MEMORY_DARWIN
|
|
1271 /* Avoid namespace collision with OS X's malloc APIs. */
|
|
1272 #define malloc_printf moz_malloc_printf
|
|
1273 #endif
|
|
1274 static void malloc_printf(const char *format, ...);
|
|
1275 #endif
|
|
1276 static bool base_pages_alloc_mmap(size_t minsize);
|
|
1277 static bool base_pages_alloc(size_t minsize);
|
|
1278 static void *base_alloc(size_t size);
|
|
1279 static void *base_calloc(size_t number, size_t size);
|
|
1280 static extent_node_t *base_node_alloc(void);
|
|
1281 static void base_node_dealloc(extent_node_t *node);
|
|
1282 static reserve_reg_t *base_reserve_reg_alloc(void);
|
|
1283 static void base_reserve_reg_dealloc(reserve_reg_t *reg);
|
|
1284 #ifdef MALLOC_STATS
|
|
1285 static void stats_print(arena_t *arena);
|
|
1286 #endif
|
|
1287 static void *pages_map(void *addr, size_t size, int pfd);
|
|
1288 static void pages_unmap(void *addr, size_t size);
|
|
1289 static void *chunk_alloc_mmap(size_t size, bool pagefile);
|
|
1290 #ifdef MALLOC_PAGEFILE
|
|
1291 static int pagefile_init(size_t size);
|
|
1292 static void pagefile_close(int pfd);
|
|
1293 #endif
|
|
1294 static void *chunk_recycle_reserve(size_t size, bool zero);
|
|
1295 static void *chunk_alloc(size_t size, bool zero, bool pagefile);
|
|
1296 static extent_node_t *chunk_dealloc_reserve(void *chunk, size_t size);
|
|
1297 static void chunk_dealloc_mmap(void *chunk, size_t size);
|
|
1298 static void chunk_dealloc(void *chunk, size_t size);
|
|
1299 #ifndef NO_TLS
|
|
1300 static arena_t *choose_arena_hard(void);
|
|
1301 #endif
|
|
1302 static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size,
|
|
1303 bool large, bool zero);
|
|
1304 static void arena_chunk_init(arena_t *arena, arena_chunk_t *chunk);
|
|
1305 static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk);
|
|
1306 static arena_run_t *arena_run_alloc(arena_t *arena, arena_bin_t *bin,
|
|
1307 size_t size, bool large, bool zero);
|
|
1308 static void arena_purge(arena_t *arena);
|
|
1309 static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty);
|
|
1310 static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk,
|
|
1311 arena_run_t *run, size_t oldsize, size_t newsize);
|
|
1312 static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk,
|
|
1313 arena_run_t *run, size_t oldsize, size_t newsize, bool dirty);
|
|
1314 static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin);
|
|
1315 static void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin);
|
|
1316 static size_t arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size);
|
|
1317 #ifdef MALLOC_BALANCE
|
|
1318 static void arena_lock_balance_hard(arena_t *arena);
|
|
1319 #endif
|
|
1320 static void *arena_malloc_large(arena_t *arena, size_t size, bool zero);
|
|
1321 static void *arena_palloc(arena_t *arena, size_t alignment, size_t size,
|
|
1322 size_t alloc_size);
|
|
1323 static size_t arena_salloc(const void *ptr);
|
|
1324 static void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk,
|
|
1325 void *ptr);
|
|
1326 static void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk,
|
|
1327 void *ptr, size_t size, size_t oldsize);
|
|
1328 static bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk,
|
|
1329 void *ptr, size_t size, size_t oldsize);
|
|
1330 static bool arena_ralloc_large(void *ptr, size_t size, size_t oldsize);
|
|
1331 static void *arena_ralloc(void *ptr, size_t size, size_t oldsize);
|
|
1332 static bool arena_new(arena_t *arena);
|
|
1333 static arena_t *arenas_extend(unsigned ind);
|
|
1334 static void *huge_malloc(size_t size, bool zero);
|
|
1335 static void *huge_palloc(size_t alignment, size_t size);
|
|
1336 static void *huge_ralloc(void *ptr, size_t size, size_t oldsize);
|
|
1337 static void huge_dalloc(void *ptr);
|
|
1338 static void malloc_print_stats(void);
|
|
1339 #ifndef MOZ_MEMORY_WINDOWS
|
|
1340 static
|
|
1341 #endif
|
|
1342 bool malloc_init_hard(void);
|
|
1343 static void reserve_shrink(void);
|
|
1344 static uint64_t reserve_notify(reserve_cnd_t cnd, size_t size, uint64_t seq);
|
|
1345 static uint64_t reserve_crit(size_t size, const char *fname, uint64_t seq);
|
|
1346 static void reserve_fail(size_t size, const char *fname);
|
|
1347
|
|
1348 void _malloc_prefork(void);
|
|
1349 void _malloc_postfork(void);
|
|
1350
|
|
1351 /*
|
|
1352 * End function prototypes.
|
|
1353 */
|
|
1354 /******************************************************************************/
|
|
1355
|
|
1356 /*
|
|
1357 * umax2s() provides minimal integer printing functionality, which is
|
|
1358 * especially useful for situations where allocation in vsnprintf() calls would
|
|
1359 * potentially cause deadlock.
|
|
1360 */
|
|
1361 #define UMAX2S_BUFSIZE 21
|
|
1362 static char *
|
|
1363 umax2s(uintmax_t x, char *s)
|
|
1364 {
|
|
1365 unsigned i;
|
|
1366
|
|
1367 i = UMAX2S_BUFSIZE - 1;
|
|
1368 s[i] = '\0';
|
|
1369 do {
|
|
1370 i--;
|
|
1371 s[i] = "0123456789"[x % 10];
|
|
1372 x /= 10;
|
|
1373 } while (x > 0);
|
|
1374
|
|
1375 return (&s[i]);
|
|
1376 }
|
|
1377
|
|
1378 static void
|
|
1379 wrtmessage(const char *p1, const char *p2, const char *p3, const char *p4)
|
|
1380 {
|
|
1381 #ifdef MOZ_MEMORY_WINCE
|
|
1382 wchar_t buf[1024];
|
|
1383 #define WRT_PRINT(s) \
|
|
1384 MultiByteToWideChar(CP_ACP, 0, s, -1, buf, 1024); \
|
|
1385 OutputDebugStringW(buf)
|
|
1386
|
|
1387 WRT_PRINT(p1);
|
|
1388 WRT_PRINT(p2);
|
|
1389 WRT_PRINT(p3);
|
|
1390 WRT_PRINT(p4);
|
|
1391 #else
|
|
1392 #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_WINDOWS)
|
|
1393 #define _write write
|
|
1394 #endif
|
|
1395 _write(STDERR_FILENO, p1, (unsigned int) strlen(p1));
|
|
1396 _write(STDERR_FILENO, p2, (unsigned int) strlen(p2));
|
|
1397 _write(STDERR_FILENO, p3, (unsigned int) strlen(p3));
|
|
1398 _write(STDERR_FILENO, p4, (unsigned int) strlen(p4));
|
|
1399 #endif
|
|
1400
|
|
1401 }
|
|
1402
|
|
1403 #define _malloc_message malloc_message
|
|
1404
|
|
1405 void (*_malloc_message)(const char *p1, const char *p2, const char *p3,
|
|
1406 const char *p4) = wrtmessage;
|
|
1407
|
|
1408 #ifdef MALLOC_DEBUG
|
|
1409 # define assert(e) do { \
|
|
1410 if (!(e)) { \
|
|
1411 char line_buf[UMAX2S_BUFSIZE]; \
|
|
1412 _malloc_message(__FILE__, ":", umax2s(__LINE__, \
|
|
1413 line_buf), ": Failed assertion: "); \
|
|
1414 _malloc_message("\"", #e, "\"\n", ""); \
|
|
1415 abort(); \
|
|
1416 } \
|
|
1417 } while (0)
|
|
1418 #else
|
|
1419 #define assert(e)
|
|
1420 #endif
|
|
1421
|
|
1422 /******************************************************************************/
|
|
1423 /*
|
|
1424 * Begin mutex. We can't use normal pthread mutexes in all places, because
|
|
1425 * they require malloc()ed memory, which causes bootstrapping issues in some
|
|
1426 * cases.
|
|
1427 */
|
|
1428
|
|
1429 static bool
|
|
1430 malloc_mutex_init(malloc_mutex_t *mutex)
|
|
1431 {
|
|
1432 #if defined(MOZ_MEMORY_WINCE)
|
|
1433 InitializeCriticalSection(mutex);
|
|
1434 #elif defined(MOZ_MEMORY_WINDOWS)
|
|
1435 if (__isthreaded)
|
|
1436 if (! __crtInitCritSecAndSpinCount(mutex, _CRT_SPINCOUNT))
|
|
1437 return (true);
|
|
1438 #elif defined(MOZ_MEMORY_DARWIN)
|
|
1439 mutex->lock = OS_SPINLOCK_INIT;
|
|
1440 #elif defined(MOZ_MEMORY_LINUX)
|
|
1441 pthread_mutexattr_t attr;
|
|
1442 if (pthread_mutexattr_init(&attr) != 0)
|
|
1443 return (true);
|
|
1444 pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
|
|
1445 if (pthread_mutex_init(mutex, &attr) != 0) {
|
|
1446 pthread_mutexattr_destroy(&attr);
|
|
1447 return (true);
|
|
1448 }
|
|
1449 pthread_mutexattr_destroy(&attr);
|
|
1450 #elif defined(MOZ_MEMORY)
|
|
1451 if (pthread_mutex_init(mutex, NULL) != 0)
|
|
1452 return (true);
|
|
1453 #else
|
|
1454 static const spinlock_t lock = _SPINLOCK_INITIALIZER;
|
|
1455
|
|
1456 mutex->lock = lock;
|
|
1457 #endif
|
|
1458 return (false);
|
|
1459 }
|
|
1460
|
|
1461 static inline void
|
|
1462 malloc_mutex_lock(malloc_mutex_t *mutex)
|
|
1463 {
|
|
1464
|
|
1465 #if defined(MOZ_MEMORY_WINDOWS)
|
|
1466 EnterCriticalSection(mutex);
|
|
1467 #elif defined(MOZ_MEMORY_DARWIN)
|
|
1468 OSSpinLockLock(&mutex->lock);
|
|
1469 #elif defined(MOZ_MEMORY)
|
|
1470 pthread_mutex_lock(mutex);
|
|
1471 #else
|
|
1472 if (__isthreaded)
|
|
1473 _SPINLOCK(&mutex->lock);
|
|
1474 #endif
|
|
1475 }
|
|
1476
|
|
1477 static inline void
|
|
1478 malloc_mutex_unlock(malloc_mutex_t *mutex)
|
|
1479 {
|
|
1480
|
|
1481 #if defined(MOZ_MEMORY_WINDOWS)
|
|
1482 LeaveCriticalSection(mutex);
|
|
1483 #elif defined(MOZ_MEMORY_DARWIN)
|
|
1484 OSSpinLockUnlock(&mutex->lock);
|
|
1485 #elif defined(MOZ_MEMORY)
|
|
1486 pthread_mutex_unlock(mutex);
|
|
1487 #else
|
|
1488 if (__isthreaded)
|
|
1489 _SPINUNLOCK(&mutex->lock);
|
|
1490 #endif
|
|
1491 }
|
|
1492
|
|
1493 static bool
|
|
1494 malloc_spin_init(malloc_spinlock_t *lock)
|
|
1495 {
|
|
1496 #if defined(MOZ_MEMORY_WINCE)
|
|
1497 InitializeCriticalSection(lock);
|
|
1498 #elif defined(MOZ_MEMORY_WINDOWS)
|
|
1499 if (__isthreaded)
|
|
1500 if (! __crtInitCritSecAndSpinCount(lock, _CRT_SPINCOUNT))
|
|
1501 return (true);
|
|
1502 #elif defined(MOZ_MEMORY_DARWIN)
|
|
1503 lock->lock = OS_SPINLOCK_INIT;
|
|
1504 #elif defined(MOZ_MEMORY_LINUX)
|
|
1505 pthread_mutexattr_t attr;
|
|
1506 if (pthread_mutexattr_init(&attr) != 0)
|
|
1507 return (true);
|
|
1508 pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
|
|
1509 if (pthread_mutex_init(lock, &attr) != 0) {
|
|
1510 pthread_mutexattr_destroy(&attr);
|
|
1511 return (true);
|
|
1512 }
|
|
1513 pthread_mutexattr_destroy(&attr);
|
|
1514 #elif defined(MOZ_MEMORY)
|
|
1515 if (pthread_mutex_init(lock, NULL) != 0)
|
|
1516 return (true);
|
|
1517 #else
|
|
1518 lock->lock = _SPINLOCK_INITIALIZER;
|
|
1519 #endif
|
|
1520 return (false);
|
|
1521 }
|
|
1522
|
|
1523 static inline void
|
|
1524 malloc_spin_lock(malloc_spinlock_t *lock)
|
|
1525 {
|
|
1526
|
|
1527 #if defined(MOZ_MEMORY_WINDOWS)
|
|
1528 EnterCriticalSection(lock);
|
|
1529 #elif defined(MOZ_MEMORY_DARWIN)
|
|
1530 OSSpinLockLock(&lock->lock);
|
|
1531 #elif defined(MOZ_MEMORY)
|
|
1532 pthread_mutex_lock(lock);
|
|
1533 #else
|
|
1534 if (__isthreaded)
|
|
1535 _SPINLOCK(&lock->lock);
|
|
1536 #endif
|
|
1537 }
|
|
1538
|
|
1539 static inline void
|
|
1540 malloc_spin_unlock(malloc_spinlock_t *lock)
|
|
1541 {
|
|
1542 #if defined(MOZ_MEMORY_WINDOWS)
|
|
1543 LeaveCriticalSection(lock);
|
|
1544 #elif defined(MOZ_MEMORY_DARWIN)
|
|
1545 OSSpinLockUnlock(&lock->lock);
|
|
1546 #elif defined(MOZ_MEMORY)
|
|
1547 pthread_mutex_unlock(lock);
|
|
1548 #else
|
|
1549 if (__isthreaded)
|
|
1550 _SPINUNLOCK(&lock->lock);
|
|
1551 #endif
|
|
1552 }
|
|
1553
|
|
1554 /*
|
|
1555 * End mutex.
|
|
1556 */
|
|
1557 /******************************************************************************/
|
|
1558 /*
|
|
1559 * Begin spin lock. Spin locks here are actually adaptive mutexes that block
|
|
1560 * after a period of spinning, because unbounded spinning would allow for
|
|
1561 * priority inversion.
|
|
1562 */
|
|
1563
|
|
1564 #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_DARWIN)
|
|
1565 # define malloc_spin_init malloc_mutex_init
|
|
1566 # define malloc_spin_lock malloc_mutex_lock
|
|
1567 # define malloc_spin_unlock malloc_mutex_unlock
|
|
1568 #endif
|
|
1569
|
|
1570 #ifndef MOZ_MEMORY
|
|
1571 /*
|
|
1572 * We use an unpublished interface to initialize pthread mutexes with an
|
|
1573 * allocation callback, in order to avoid infinite recursion.
|
|
1574 */
|
|
1575 int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
|
|
1576 void *(calloc_cb)(size_t, size_t));
|
|
1577
|
|
1578 __weak_reference(_pthread_mutex_init_calloc_cb_stub,
|
|
1579 _pthread_mutex_init_calloc_cb);
|
|
1580
|
|
1581 int
|
|
1582 _pthread_mutex_init_calloc_cb_stub(pthread_mutex_t *mutex,
|
|
1583 void *(calloc_cb)(size_t, size_t))
|
|
1584 {
|
|
1585
|
|
1586 return (0);
|
|
1587 }
|
|
1588
|
|
1589 static bool
|
|
1590 malloc_spin_init(pthread_mutex_t *lock)
|
|
1591 {
|
|
1592
|
|
1593 if (_pthread_mutex_init_calloc_cb(lock, base_calloc) != 0)
|
|
1594 return (true);
|
|
1595
|
|
1596 return (false);
|
|
1597 }
|
|
1598
|
|
1599 static inline unsigned
|
|
1600 malloc_spin_lock(pthread_mutex_t *lock)
|
|
1601 {
|
|
1602 unsigned ret = 0;
|
|
1603
|
|
1604 if (__isthreaded) {
|
|
1605 if (_pthread_mutex_trylock(lock) != 0) {
|
|
1606 unsigned i;
|
|
1607 volatile unsigned j;
|
|
1608
|
|
1609 /* Exponentially back off. */
|
|
1610 for (i = 1; i <= SPIN_LIMIT_2POW; i++) {
|
|
1611 for (j = 0; j < (1U << i); j++)
|
|
1612 ret++;
|
|
1613
|
|
1614 CPU_SPINWAIT;
|
|
1615 if (_pthread_mutex_trylock(lock) == 0)
|
|
1616 return (ret);
|
|
1617 }
|
|
1618
|
|
1619 /*
|
|
1620 * Spinning failed. Block until the lock becomes
|
|
1621 * available, in order to avoid indefinite priority
|
|
1622 * inversion.
|
|
1623 */
|
|
1624 _pthread_mutex_lock(lock);
|
|
1625 assert((ret << BLOCK_COST_2POW) != 0);
|
|
1626 return (ret << BLOCK_COST_2POW);
|
|
1627 }
|
|
1628 }
|
|
1629
|
|
1630 return (ret);
|
|
1631 }
|
|
1632
|
|
1633 static inline void
|
|
1634 malloc_spin_unlock(pthread_mutex_t *lock)
|
|
1635 {
|
|
1636
|
|
1637 if (__isthreaded)
|
|
1638 _pthread_mutex_unlock(lock);
|
|
1639 }
|
|
1640 #endif
|
|
1641
|
|
1642 /*
|
|
1643 * End spin lock.
|
|
1644 */
|
|
1645 /******************************************************************************/
|
|
1646 /*
|
|
1647 * Begin Utility functions/macros.
|
|
1648 */
|
|
1649
|
|
1650 /* Return the chunk address for allocation address a. */
|
|
1651 #define CHUNK_ADDR2BASE(a) \
|
|
1652 ((void *)((uintptr_t)(a) & ~chunksize_mask))
|
|
1653
|
|
1654 /* Return the chunk offset of address a. */
|
|
1655 #define CHUNK_ADDR2OFFSET(a) \
|
|
1656 ((size_t)((uintptr_t)(a) & chunksize_mask))
|
|
1657
|
|
1658 /* Return the smallest chunk multiple that is >= s. */
|
|
1659 #define CHUNK_CEILING(s) \
|
|
1660 (((s) + chunksize_mask) & ~chunksize_mask)
|
|
1661
|
|
1662 /* Return the smallest cacheline multiple that is >= s. */
|
|
1663 #define CACHELINE_CEILING(s) \
|
|
1664 (((s) + (CACHELINE - 1)) & ~(CACHELINE - 1))
|
|
1665
|
|
1666 /* Return the smallest quantum multiple that is >= a. */
|
|
1667 #define QUANTUM_CEILING(a) \
|
|
1668 (((a) + quantum_mask) & ~quantum_mask)
|
|
1669
|
|
1670 /* Return the smallest pagesize multiple that is >= s. */
|
|
1671 #define PAGE_CEILING(s) \
|
|
1672 (((s) + pagesize_mask) & ~pagesize_mask)
|
|
1673
|
|
1674 /* Compute the smallest power of 2 that is >= x. */
|
|
1675 static inline size_t
|
|
1676 pow2_ceil(size_t x)
|
|
1677 {
|
|
1678
|
|
1679 x--;
|
|
1680 x |= x >> 1;
|
|
1681 x |= x >> 2;
|
|
1682 x |= x >> 4;
|
|
1683 x |= x >> 8;
|
|
1684 x |= x >> 16;
|
|
1685 #if (SIZEOF_PTR == 8)
|
|
1686 x |= x >> 32;
|
|
1687 #endif
|
|
1688 x++;
|
|
1689 return (x);
|
|
1690 }
|
|
1691
|
|
1692 #ifdef MALLOC_BALANCE
|
|
1693 /*
|
|
1694 * Use a simple linear congruential pseudo-random number generator:
|
|
1695 *
|
|
1696 * prn(y) = (a*x + c) % m
|
|
1697 *
|
|
1698 * where the following constants ensure maximal period:
|
|
1699 *
|
|
1700 * a == Odd number (relatively prime to 2^n), and (a-1) is a multiple of 4.
|
|
1701 * c == Odd number (relatively prime to 2^n).
|
|
1702 * m == 2^32
|
|
1703 *
|
|
1704 * See Knuth's TAOCP 3rd Ed., Vol. 2, pg. 17 for details on these constraints.
|
|
1705 *
|
|
1706 * This choice of m has the disadvantage that the quality of the bits is
|
|
1707 * proportional to bit position. For example. the lowest bit has a cycle of 2,
|
|
1708 * the next has a cycle of 4, etc. For this reason, we prefer to use the upper
|
|
1709 * bits.
|
|
1710 */
|
|
1711 # define PRN_DEFINE(suffix, var, a, c) \
|
|
1712 static inline void \
|
|
1713 sprn_##suffix(uint32_t seed) \
|
|
1714 { \
|
|
1715 var = seed; \
|
|
1716 } \
|
|
1717 \
|
|
1718 static inline uint32_t \
|
|
1719 prn_##suffix(uint32_t lg_range) \
|
|
1720 { \
|
|
1721 uint32_t ret, x; \
|
|
1722 \
|
|
1723 assert(lg_range > 0); \
|
|
1724 assert(lg_range <= 32); \
|
|
1725 \
|
|
1726 x = (var * (a)) + (c); \
|
|
1727 var = x; \
|
|
1728 ret = x >> (32 - lg_range); \
|
|
1729 \
|
|
1730 return (ret); \
|
|
1731 }
|
|
1732 # define SPRN(suffix, seed) sprn_##suffix(seed)
|
|
1733 # define PRN(suffix, lg_range) prn_##suffix(lg_range)
|
|
1734 #endif
|
|
1735
|
|
1736 #ifdef MALLOC_BALANCE
|
|
1737 /* Define the PRNG used for arena assignment. */
|
|
1738 static __thread uint32_t balance_x;
|
|
1739 PRN_DEFINE(balance, balance_x, 1297, 1301)
|
|
1740 #endif
|
|
1741
|
|
1742 #ifdef MALLOC_UTRACE
|
|
1743 static int
|
|
1744 utrace(const void *addr, size_t len)
|
|
1745 {
|
|
1746 malloc_utrace_t *ut = (malloc_utrace_t *)addr;
|
|
1747
|
|
1748 assert(len == sizeof(malloc_utrace_t));
|
|
1749
|
|
1750 if (ut->p == NULL && ut->s == 0 && ut->r == NULL)
|
|
1751 malloc_printf("%d x USER malloc_init()\n", getpid());
|
|
1752 else if (ut->p == NULL && ut->r != NULL) {
|
|
1753 malloc_printf("%d x USER %p = malloc(%zu)\n", getpid(), ut->r,
|
|
1754 ut->s);
|
|
1755 } else if (ut->p != NULL && ut->r != NULL) {
|
|
1756 malloc_printf("%d x USER %p = realloc(%p, %zu)\n", getpid(),
|
|
1757 ut->r, ut->p, ut->s);
|
|
1758 } else
|
|
1759 malloc_printf("%d x USER free(%p)\n", getpid(), ut->p);
|
|
1760
|
|
1761 return (0);
|
|
1762 }
|
|
1763 #endif
|
|
1764
|
|
1765 static inline const char *
|
|
1766 _getprogname(void)
|
|
1767 {
|
|
1768
|
|
1769 return ("<jemalloc>");
|
|
1770 }
|
|
1771
|
|
1772 #ifdef MALLOC_STATS
|
|
1773 /*
|
|
1774 * Print to stderr in such a way as to (hopefully) avoid memory allocation.
|
|
1775 */
|
|
1776 static void
|
|
1777 malloc_printf(const char *format, ...)
|
|
1778 {
|
|
1779 #ifndef WINCE
|
|
1780 char buf[4096];
|
|
1781 va_list ap;
|
|
1782
|
|
1783 va_start(ap, format);
|
|
1784 vsnprintf(buf, sizeof(buf), format, ap);
|
|
1785 va_end(ap);
|
|
1786 _malloc_message(buf, "", "", "");
|
|
1787 #endif
|
|
1788 }
|
|
1789 #endif
|
|
1790
|
|
1791 /******************************************************************************/
|
|
1792
|
|
1793 #ifdef MALLOC_DECOMMIT
|
|
1794 static inline void
|
|
1795 pages_decommit(void *addr, size_t size)
|
|
1796 {
|
|
1797
|
|
1798 #ifdef MOZ_MEMORY_WINDOWS
|
|
1799 VirtualFree(addr, size, MEM_DECOMMIT);
|
|
1800 #else
|
|
1801 if (mmap(addr, size, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1,
|
|
1802 0) == MAP_FAILED)
|
|
1803 abort();
|
|
1804 #endif
|
|
1805 }
|
|
1806
|
|
1807 static inline void
|
|
1808 pages_commit(void *addr, size_t size)
|
|
1809 {
|
|
1810
|
|
1811 # ifdef MOZ_MEMORY_WINDOWS
|
|
1812 VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
|
|
1813 # else
|
|
1814 if (mmap(addr, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE |
|
|
1815 MAP_ANON, -1, 0) == MAP_FAILED)
|
|
1816 abort();
|
|
1817 # endif
|
|
1818 }
|
|
1819 #endif
|
|
1820
|
|
1821 static bool
|
|
1822 base_pages_alloc_mmap(size_t minsize)
|
|
1823 {
|
|
1824 bool ret;
|
|
1825 size_t csize;
|
|
1826 #ifdef MALLOC_DECOMMIT
|
|
1827 size_t pminsize;
|
|
1828 #endif
|
|
1829 int pfd;
|
|
1830
|
|
1831 assert(minsize != 0);
|
|
1832 csize = CHUNK_CEILING(minsize);
|
|
1833 #ifdef MALLOC_PAGEFILE
|
|
1834 if (opt_pagefile) {
|
|
1835 pfd = pagefile_init(csize);
|
|
1836 if (pfd == -1)
|
|
1837 return (true);
|
|
1838 } else
|
|
1839 #endif
|
|
1840 pfd = -1;
|
|
1841 base_pages = pages_map(NULL, csize, pfd);
|
|
1842 if (base_pages == NULL) {
|
|
1843 ret = true;
|
|
1844 goto RETURN;
|
|
1845 }
|
|
1846 base_next_addr = base_pages;
|
|
1847 base_past_addr = (void *)((uintptr_t)base_pages + csize);
|
|
1848 #ifdef MALLOC_DECOMMIT
|
|
1849 /*
|
|
1850 * Leave enough pages for minsize committed, since otherwise they would
|
|
1851 * have to be immediately recommitted.
|
|
1852 */
|
|
1853 pminsize = PAGE_CEILING(minsize);
|
|
1854 base_next_decommitted = (void *)((uintptr_t)base_pages + pminsize);
|
|
1855 if (pminsize < csize)
|
|
1856 pages_decommit(base_next_decommitted, csize - pminsize);
|
|
1857 #endif
|
|
1858 #ifdef MALLOC_STATS
|
|
1859 base_mapped += csize;
|
|
1860 #endif
|
|
1861
|
|
1862 ret = false;
|
|
1863 RETURN:
|
|
1864 #ifdef MALLOC_PAGEFILE
|
|
1865 if (pfd != -1)
|
|
1866 pagefile_close(pfd);
|
|
1867 #endif
|
|
1868 return (false);
|
|
1869 }
|
|
1870
|
|
1871 static bool
|
|
1872 base_pages_alloc(size_t minsize)
|
|
1873 {
|
|
1874
|
|
1875 if (base_pages_alloc_mmap(minsize) == false)
|
|
1876 return (false);
|
|
1877
|
|
1878 return (true);
|
|
1879 }
|
|
1880
|
|
1881 static void *
|
|
1882 base_alloc(size_t size)
|
|
1883 {
|
|
1884 void *ret;
|
|
1885 size_t csize;
|
|
1886
|
|
1887 /* Round size up to nearest multiple of the cacheline size. */
|
|
1888 csize = CACHELINE_CEILING(size);
|
|
1889
|
|
1890 malloc_mutex_lock(&base_mtx);
|
|
1891 /* Make sure there's enough space for the allocation. */
|
|
1892 if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) {
|
|
1893 if (base_pages_alloc(csize)) {
|
|
1894 malloc_mutex_unlock(&base_mtx);
|
|
1895 return (NULL);
|
|
1896 }
|
|
1897 }
|
|
1898 /* Allocate. */
|
|
1899 ret = base_next_addr;
|
|
1900 base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
|
|
1901 #ifdef MALLOC_DECOMMIT
|
|
1902 /* Make sure enough pages are committed for the new allocation. */
|
|
1903 if ((uintptr_t)base_next_addr > (uintptr_t)base_next_decommitted) {
|
|
1904 void *pbase_next_addr =
|
|
1905 (void *)(PAGE_CEILING((uintptr_t)base_next_addr));
|
|
1906
|
|
1907 pages_commit(base_next_decommitted, (uintptr_t)pbase_next_addr -
|
|
1908 (uintptr_t)base_next_decommitted);
|
|
1909 base_next_decommitted = pbase_next_addr;
|
|
1910 }
|
|
1911 #endif
|
|
1912 malloc_mutex_unlock(&base_mtx);
|
|
1913 VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
|
|
1914
|
|
1915 return (ret);
|
|
1916 }
|
|
1917
|
|
1918 static void *
|
|
1919 base_calloc(size_t number, size_t size)
|
|
1920 {
|
|
1921 void *ret;
|
|
1922
|
|
1923 ret = base_alloc(number * size);
|
|
1924 #ifdef MALLOC_VALGRIND
|
|
1925 if (ret != NULL) {
|
|
1926 VALGRIND_FREELIKE_BLOCK(ret, 0);
|
|
1927 VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, true);
|
|
1928 }
|
|
1929 #endif
|
|
1930 memset(ret, 0, number * size);
|
|
1931
|
|
1932 return (ret);
|
|
1933 }
|
|
1934
|
|
1935 static extent_node_t *
|
|
1936 base_node_alloc(void)
|
|
1937 {
|
|
1938 extent_node_t *ret;
|
|
1939
|
|
1940 malloc_mutex_lock(&base_mtx);
|
|
1941 if (base_nodes != NULL) {
|
|
1942 ret = base_nodes;
|
|
1943 base_nodes = *(extent_node_t **)ret;
|
|
1944 VALGRIND_FREELIKE_BLOCK(ret, 0);
|
|
1945 VALGRIND_MALLOCLIKE_BLOCK(ret, sizeof(extent_node_t), 0, false);
|
|
1946 malloc_mutex_unlock(&base_mtx);
|
|
1947 } else {
|
|
1948 malloc_mutex_unlock(&base_mtx);
|
|
1949 ret = (extent_node_t *)base_alloc(sizeof(extent_node_t));
|
|
1950 }
|
|
1951
|
|
1952 return (ret);
|
|
1953 }
|
|
1954
|
|
1955 static void
|
|
1956 base_node_dealloc(extent_node_t *node)
|
|
1957 {
|
|
1958
|
|
1959 malloc_mutex_lock(&base_mtx);
|
|
1960 VALGRIND_FREELIKE_BLOCK(node, 0);
|
|
1961 VALGRIND_MALLOCLIKE_BLOCK(node, sizeof(extent_node_t *), 0, false);
|
|
1962 *(extent_node_t **)node = base_nodes;
|
|
1963 base_nodes = node;
|
|
1964 malloc_mutex_unlock(&base_mtx);
|
|
1965 }
|
|
1966
|
|
1967 static reserve_reg_t *
|
|
1968 base_reserve_reg_alloc(void)
|
|
1969 {
|
|
1970 reserve_reg_t *ret;
|
|
1971
|
|
1972 malloc_mutex_lock(&base_mtx);
|
|
1973 if (base_reserve_regs != NULL) {
|
|
1974 ret = base_reserve_regs;
|
|
1975 base_reserve_regs = *(reserve_reg_t **)ret;
|
|
1976 VALGRIND_FREELIKE_BLOCK(ret, 0);
|
|
1977 VALGRIND_MALLOCLIKE_BLOCK(ret, sizeof(reserve_reg_t), 0, false);
|
|
1978 malloc_mutex_unlock(&base_mtx);
|
|
1979 } else {
|
|
1980 malloc_mutex_unlock(&base_mtx);
|
|
1981 ret = (reserve_reg_t *)base_alloc(sizeof(reserve_reg_t));
|
|
1982 }
|
|
1983
|
|
1984 return (ret);
|
|
1985 }
|
|
1986
|
|
1987 static void
|
|
1988 base_reserve_reg_dealloc(reserve_reg_t *reg)
|
|
1989 {
|
|
1990
|
|
1991 malloc_mutex_lock(&base_mtx);
|
|
1992 VALGRIND_FREELIKE_BLOCK(reg, 0);
|
|
1993 VALGRIND_MALLOCLIKE_BLOCK(reg, sizeof(reserve_reg_t *), 0, false);
|
|
1994 *(reserve_reg_t **)reg = base_reserve_regs;
|
|
1995 base_reserve_regs = reg;
|
|
1996 malloc_mutex_unlock(&base_mtx);
|
|
1997 }
|
|
1998
|
|
1999 /******************************************************************************/
|
|
2000
|
|
2001 #ifdef MALLOC_STATS
|
|
2002 static void
|
|
2003 stats_print(arena_t *arena)
|
|
2004 {
|
|
2005 unsigned i, gap_start;
|
|
2006
|
|
2007 #ifdef MOZ_MEMORY_WINDOWS
|
|
2008 malloc_printf("dirty: %Iu page%s dirty, %I64u sweep%s,"
|
|
2009 " %I64u madvise%s, %I64u page%s purged\n",
|
|
2010 arena->ndirty, arena->ndirty == 1 ? "" : "s",
|
|
2011 arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
|
|
2012 arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
|
|
2013 arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
|
|
2014 # ifdef MALLOC_DECOMMIT
|
|
2015 malloc_printf("decommit: %I64u decommit%s, %I64u commit%s,"
|
|
2016 " %I64u page%s decommitted\n",
|
|
2017 arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
|
|
2018 arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
|
|
2019 arena->stats.decommitted,
|
|
2020 (arena->stats.decommitted == 1) ? "" : "s");
|
|
2021 # endif
|
|
2022
|
|
2023 malloc_printf(" allocated nmalloc ndalloc\n");
|
|
2024 malloc_printf("small: %12Iu %12I64u %12I64u\n",
|
|
2025 arena->stats.allocated_small, arena->stats.nmalloc_small,
|
|
2026 arena->stats.ndalloc_small);
|
|
2027 malloc_printf("large: %12Iu %12I64u %12I64u\n",
|
|
2028 arena->stats.allocated_large, arena->stats.nmalloc_large,
|
|
2029 arena->stats.ndalloc_large);
|
|
2030 malloc_printf("total: %12Iu %12I64u %12I64u\n",
|
|
2031 arena->stats.allocated_small + arena->stats.allocated_large,
|
|
2032 arena->stats.nmalloc_small + arena->stats.nmalloc_large,
|
|
2033 arena->stats.ndalloc_small + arena->stats.ndalloc_large);
|
|
2034 malloc_printf("mapped: %12Iu\n", arena->stats.mapped);
|
|
2035 #else
|
|
2036 malloc_printf("dirty: %zu page%s dirty, %llu sweep%s,"
|
|
2037 " %llu madvise%s, %llu page%s purged\n",
|
|
2038 arena->ndirty, arena->ndirty == 1 ? "" : "s",
|
|
2039 arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
|
|
2040 arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
|
|
2041 arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
|
|
2042 # ifdef MALLOC_DECOMMIT
|
|
2043 malloc_printf("decommit: %llu decommit%s, %llu commit%s,"
|
|
2044 " %llu page%s decommitted\n",
|
|
2045 arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
|
|
2046 arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
|
|
2047 arena->stats.decommitted,
|
|
2048 (arena->stats.decommitted == 1) ? "" : "s");
|
|
2049 # endif
|
|
2050
|
|
2051 malloc_printf(" allocated nmalloc ndalloc\n");
|
|
2052 malloc_printf("small: %12zu %12llu %12llu\n",
|
|
2053 arena->stats.allocated_small, arena->stats.nmalloc_small,
|
|
2054 arena->stats.ndalloc_small);
|
|
2055 malloc_printf("large: %12zu %12llu %12llu\n",
|
|
2056 arena->stats.allocated_large, arena->stats.nmalloc_large,
|
|
2057 arena->stats.ndalloc_large);
|
|
2058 malloc_printf("total: %12zu %12llu %12llu\n",
|
|
2059 arena->stats.allocated_small + arena->stats.allocated_large,
|
|
2060 arena->stats.nmalloc_small + arena->stats.nmalloc_large,
|
|
2061 arena->stats.ndalloc_small + arena->stats.ndalloc_large);
|
|
2062 malloc_printf("mapped: %12zu\n", arena->stats.mapped);
|
|
2063 #endif
|
|
2064 malloc_printf("bins: bin size regs pgs requests newruns"
|
|
2065 " reruns maxruns curruns\n");
|
|
2066 for (i = 0, gap_start = UINT_MAX; i < ntbins + nqbins + nsbins; i++) {
|
|
2067 if (arena->bins[i].stats.nrequests == 0) {
|
|
2068 if (gap_start == UINT_MAX)
|
|
2069 gap_start = i;
|
|
2070 } else {
|
|
2071 if (gap_start != UINT_MAX) {
|
|
2072 if (i > gap_start + 1) {
|
|
2073 /* Gap of more than one size class. */
|
|
2074 malloc_printf("[%u..%u]\n",
|
|
2075 gap_start, i - 1);
|
|
2076 } else {
|
|
2077 /* Gap of one size class. */
|
|
2078 malloc_printf("[%u]\n", gap_start);
|
|
2079 }
|
|
2080 gap_start = UINT_MAX;
|
|
2081 }
|
|
2082 malloc_printf(
|
|
2083 #if defined(MOZ_MEMORY_WINDOWS)
|
|
2084 "%13u %1s %4u %4u %3u %9I64u %9I64u"
|
|
2085 " %9I64u %7u %7u\n",
|
|
2086 #else
|
|
2087 "%13u %1s %4u %4u %3u %9llu %9llu"
|
|
2088 " %9llu %7lu %7lu\n",
|
|
2089 #endif
|
|
2090 i,
|
|
2091 i < ntbins ? "T" : i < ntbins + nqbins ? "Q" : "S",
|
|
2092 arena->bins[i].reg_size,
|
|
2093 arena->bins[i].nregs,
|
|
2094 arena->bins[i].run_size >> pagesize_2pow,
|
|
2095 arena->bins[i].stats.nrequests,
|
|
2096 arena->bins[i].stats.nruns,
|
|
2097 arena->bins[i].stats.reruns,
|
|
2098 arena->bins[i].stats.highruns,
|
|
2099 arena->bins[i].stats.curruns);
|
|
2100 }
|
|
2101 }
|
|
2102 if (gap_start != UINT_MAX) {
|
|
2103 if (i > gap_start + 1) {
|
|
2104 /* Gap of more than one size class. */
|
|
2105 malloc_printf("[%u..%u]\n", gap_start, i - 1);
|
|
2106 } else {
|
|
2107 /* Gap of one size class. */
|
|
2108 malloc_printf("[%u]\n", gap_start);
|
|
2109 }
|
|
2110 }
|
|
2111 }
|
|
2112 #endif
|
|
2113
|
|
2114 /*
|
|
2115 * End Utility functions/macros.
|
|
2116 */
|
|
2117 /******************************************************************************/
|
|
2118 /*
|
|
2119 * Begin extent tree code.
|
|
2120 */
|
|
2121
|
|
2122 static inline int
|
|
2123 extent_szad_comp(extent_node_t *a, extent_node_t *b)
|
|
2124 {
|
|
2125 int ret;
|
|
2126 size_t a_size = a->size;
|
|
2127 size_t b_size = b->size;
|
|
2128
|
|
2129 ret = (a_size > b_size) - (a_size < b_size);
|
|
2130 if (ret == 0) {
|
|
2131 uintptr_t a_addr = (uintptr_t)a->addr;
|
|
2132 uintptr_t b_addr = (uintptr_t)b->addr;
|
|
2133
|
|
2134 ret = (a_addr > b_addr) - (a_addr < b_addr);
|
|
2135 }
|
|
2136
|
|
2137 return (ret);
|
|
2138 }
|
|
2139
|
|
2140 /* Wrap red-black tree macros in functions. */
|
|
2141 rb_wrap(static, extent_tree_szad_, extent_tree_t, extent_node_t,
|
|
2142 link_szad, extent_szad_comp)
|
|
2143
|
|
2144 static inline int
|
|
2145 extent_ad_comp(extent_node_t *a, extent_node_t *b)
|
|
2146 {
|
|
2147 uintptr_t a_addr = (uintptr_t)a->addr;
|
|
2148 uintptr_t b_addr = (uintptr_t)b->addr;
|
|
2149
|
|
2150 return ((a_addr > b_addr) - (a_addr < b_addr));
|
|
2151 }
|
|
2152
|
|
2153 /* Wrap red-black tree macros in functions. */
|
|
2154 rb_wrap(static, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad,
|
|
2155 extent_ad_comp)
|
|
2156
|
|
2157 /*
|
|
2158 * End extent tree code.
|
|
2159 */
|
|
2160 /******************************************************************************/
|
|
2161 /*
|
|
2162 * Begin chunk management functions.
|
|
2163 */
|
|
2164
|
|
2165 #ifdef MOZ_MEMORY_WINDOWS
|
|
2166 #ifdef MOZ_MEMORY_WINCE
|
|
2167 #define ALIGN_ADDR2OFFSET(al, ad) \
|
|
2168 ((uintptr_t)ad & (al - 1))
|
|
2169 static void *
|
|
2170 pages_map_align(size_t size, int pfd, size_t alignment)
|
|
2171 {
|
|
2172
|
|
2173 void *ret;
|
|
2174 int offset;
|
|
2175 if (size % alignment)
|
|
2176 size += (alignment - (size % alignment));
|
|
2177 assert(size >= alignment);
|
|
2178 ret = pages_map(NULL, size, pfd);
|
|
2179 offset = ALIGN_ADDR2OFFSET(alignment, ret);
|
|
2180 if (offset) {
|
|
2181 /* try to over allocate by the ammount we're offset */
|
|
2182 void *tmp;
|
|
2183 pages_unmap(ret, size);
|
|
2184 tmp = VirtualAlloc(NULL, size + alignment - offset,
|
|
2185 MEM_RESERVE, PAGE_NOACCESS);
|
|
2186 if (offset == ALIGN_ADDR2OFFSET(alignment, tmp))
|
|
2187 ret = VirtualAlloc((void*)((intptr_t)tmp + alignment
|
|
2188 - offset), size, MEM_COMMIT,
|
|
2189 PAGE_READWRITE);
|
|
2190 else
|
|
2191 VirtualFree(tmp, 0, MEM_RELEASE);
|
|
2192 offset = ALIGN_ADDR2OFFSET(alignment, ret);
|
|
2193
|
|
2194
|
|
2195 if (offset) {
|
|
2196 /* over allocate to ensure we have an aligned region */
|
|
2197 ret = VirtualAlloc(NULL, size + alignment, MEM_RESERVE,
|
|
2198 PAGE_NOACCESS);
|
|
2199 offset = ALIGN_ADDR2OFFSET(alignment, ret);
|
|
2200 ret = VirtualAlloc((void*)((intptr_t)ret +
|
|
2201 alignment - offset),
|
|
2202 size, MEM_COMMIT, PAGE_READWRITE);
|
|
2203 }
|
|
2204 }
|
|
2205 return (ret);
|
|
2206 }
|
|
2207 #endif
|
|
2208
|
|
2209 static void *
|
|
2210 pages_map(void *addr, size_t size, int pfd)
|
|
2211 {
|
|
2212 void *ret = NULL;
|
|
2213 #if defined(MOZ_MEMORY_WINCE)
|
|
2214 void *va_ret;
|
|
2215 assert(addr == NULL);
|
|
2216 va_ret = VirtualAlloc(addr, size, MEM_RESERVE, PAGE_NOACCESS);
|
|
2217 if (va_ret)
|
|
2218 ret = VirtualAlloc(va_ret, size, MEM_COMMIT, PAGE_READWRITE);
|
|
2219 assert(va_ret == ret);
|
|
2220 #elif defined(MOZ_MEMORY_WINDOWS)
|
|
2221 ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
|
|
2222 PAGE_READWRITE);
|
|
2223 #endif
|
|
2224 return (ret);
|
|
2225 }
|
|
2226
|
|
2227 static void
|
|
2228 pages_unmap(void *addr, size_t size)
|
|
2229 {
|
|
2230 if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
|
|
2231 #ifdef MOZ_MEMORY_WINCE
|
|
2232 if (GetLastError() == ERROR_INVALID_PARAMETER) {
|
|
2233 MEMORY_BASIC_INFORMATION info;
|
|
2234 VirtualQuery(addr, &info, sizeof(info));
|
|
2235 if (VirtualFree(info.AllocationBase, 0, MEM_RELEASE))
|
|
2236 return;
|
|
2237 }
|
|
2238 #endif
|
|
2239 _malloc_message(_getprogname(),
|
|
2240 ": (malloc) Error in VirtualFree()\n", "", "");
|
|
2241 if (opt_abort)
|
|
2242 abort();
|
|
2243 }
|
|
2244 }
|
|
2245 #elif (defined(MOZ_MEMORY_DARWIN))
|
|
2246 static void *
|
|
2247 pages_map(void *addr, size_t size, int pfd)
|
|
2248 {
|
|
2249 void *ret;
|
|
2250 kern_return_t err;
|
|
2251 int flags;
|
|
2252
|
|
2253 if (addr != NULL) {
|
|
2254 ret = addr;
|
|
2255 flags = 0;
|
|
2256 } else
|
|
2257 flags = VM_FLAGS_ANYWHERE;
|
|
2258
|
|
2259 err = vm_allocate((vm_map_t)mach_task_self(), (vm_address_t *)&ret,
|
|
2260 (vm_size_t)size, flags);
|
|
2261 if (err != KERN_SUCCESS)
|
|
2262 ret = NULL;
|
|
2263
|
|
2264 assert(ret == NULL || (addr == NULL && ret != addr)
|
|
2265 || (addr != NULL && ret == addr));
|
|
2266 return (ret);
|
|
2267 }
|
|
2268
|
|
2269 static void
|
|
2270 pages_unmap(void *addr, size_t size)
|
|
2271 {
|
|
2272 kern_return_t err;
|
|
2273
|
|
2274 err = vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)addr,
|
|
2275 (vm_size_t)size);
|
|
2276 if (err != KERN_SUCCESS) {
|
|
2277 malloc_message(_getprogname(),
|
|
2278 ": (malloc) Error in vm_deallocate(): ",
|
|
2279 mach_error_string(err), "\n");
|
|
2280 if (opt_abort)
|
|
2281 abort();
|
|
2282 }
|
|
2283 }
|
|
2284
|
|
2285 #define VM_COPY_MIN (pagesize << 5)
|
|
2286 static inline void
|
|
2287 pages_copy(void *dest, const void *src, size_t n)
|
|
2288 {
|
|
2289
|
|
2290 assert((void *)((uintptr_t)dest & ~pagesize_mask) == dest);
|
|
2291 assert(n >= VM_COPY_MIN);
|
|
2292 assert((void *)((uintptr_t)src & ~pagesize_mask) == src);
|
|
2293
|
|
2294 vm_copy(mach_task_self(), (vm_address_t)src, (vm_size_t)n,
|
|
2295 (vm_address_t)dest);
|
|
2296 }
|
|
2297 #else /* MOZ_MEMORY_DARWIN */
|
|
2298 #ifdef JEMALLOC_USES_MAP_ALIGN
|
|
2299 static void *
|
|
2300 pages_map_align(size_t size, int pfd, size_t alignment)
|
|
2301 {
|
|
2302 void *ret;
|
|
2303
|
|
2304 /*
|
|
2305 * We don't use MAP_FIXED here, because it can cause the *replacement*
|
|
2306 * of existing mappings, and we only want to create new mappings.
|
|
2307 */
|
|
2308 #ifdef MALLOC_PAGEFILE
|
|
2309 if (pfd != -1) {
|
|
2310 ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
|
|
2311 MAP_NOSYNC | MAP_ALIGN, pfd, 0);
|
|
2312 } else
|
|
2313 #endif
|
|
2314 {
|
|
2315 ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
|
|
2316 MAP_NOSYNC | MAP_ALIGN | MAP_ANON, -1, 0);
|
|
2317 }
|
|
2318 assert(ret != NULL);
|
|
2319
|
|
2320 if (ret == MAP_FAILED)
|
|
2321 ret = NULL;
|
|
2322 return (ret);
|
|
2323 }
|
|
2324 #endif
|
|
2325
|
|
2326 static void *
|
|
2327 pages_map(void *addr, size_t size, int pfd)
|
|
2328 {
|
|
2329 void *ret;
|
|
2330
|
|
2331 /*
|
|
2332 * We don't use MAP_FIXED here, because it can cause the *replacement*
|
|
2333 * of existing mappings, and we only want to create new mappings.
|
|
2334 */
|
|
2335 #ifdef MALLOC_PAGEFILE
|
|
2336 if (pfd != -1) {
|
|
2337 ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
|
|
2338 MAP_NOSYNC, pfd, 0);
|
|
2339 } else
|
|
2340 #endif
|
|
2341 {
|
|
2342 ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
|
|
2343 MAP_ANON, -1, 0);
|
|
2344 }
|
|
2345 assert(ret != NULL);
|
|
2346
|
|
2347 if (ret == MAP_FAILED)
|
|
2348 ret = NULL;
|
|
2349 else if (addr != NULL && ret != addr) {
|
|
2350 /*
|
|
2351 * We succeeded in mapping memory, but not in the right place.
|
|
2352 */
|
|
2353 if (munmap(ret, size) == -1) {
|
|
2354 char buf[STRERROR_BUF];
|
|
2355
|
|
2356 strerror_r(errno, buf, sizeof(buf));
|
|
2357 _malloc_message(_getprogname(),
|
|
2358 ": (malloc) Error in munmap(): ", buf, "\n");
|
|
2359 if (opt_abort)
|
|
2360 abort();
|
|
2361 }
|
|
2362 ret = NULL;
|
|
2363 }
|
|
2364
|
|
2365 assert(ret == NULL || (addr == NULL && ret != addr)
|
|
2366 || (addr != NULL && ret == addr));
|
|
2367 return (ret);
|
|
2368 }
|
|
2369
|
|
2370 static void
|
|
2371 pages_unmap(void *addr, size_t size)
|
|
2372 {
|
|
2373
|
|
2374 if (munmap(addr, size) == -1) {
|
|
2375 char buf[STRERROR_BUF];
|
|
2376
|
|
2377 strerror_r(errno, buf, sizeof(buf));
|
|
2378 _malloc_message(_getprogname(),
|
|
2379 ": (malloc) Error in munmap(): ", buf, "\n");
|
|
2380 if (opt_abort)
|
|
2381 abort();
|
|
2382 }
|
|
2383 }
|
|
2384 #endif
|
|
2385
|
|
2386 #ifdef MALLOC_VALIDATE
|
|
2387 static inline malloc_rtree_t *
|
|
2388 malloc_rtree_new(unsigned bits)
|
|
2389 {
|
|
2390 malloc_rtree_t *ret;
|
|
2391 unsigned bits_per_level, height, i;
|
|
2392
|
|
2393 bits_per_level = ffs(pow2_ceil((MALLOC_RTREE_NODESIZE /
|
|
2394 sizeof(void *)))) - 1;
|
|
2395 height = bits / bits_per_level;
|
|
2396 if (height * bits_per_level != bits)
|
|
2397 height++;
|
|
2398 assert(height * bits_per_level >= bits);
|
|
2399
|
|
2400 ret = (malloc_rtree_t*)base_calloc(1, sizeof(malloc_rtree_t) + (sizeof(unsigned) *
|
|
2401 (height - 1)));
|
|
2402 if (ret == NULL)
|
|
2403 return (NULL);
|
|
2404
|
|
2405 malloc_spin_init(&ret->lock);
|
|
2406 ret->height = height;
|
|
2407 if (bits_per_level * height > bits)
|
|
2408 ret->level2bits[0] = bits % bits_per_level;
|
|
2409 else
|
|
2410 ret->level2bits[0] = bits_per_level;
|
|
2411 for (i = 1; i < height; i++)
|
|
2412 ret->level2bits[i] = bits_per_level;
|
|
2413
|
|
2414 ret->root = (void**)base_calloc(1, sizeof(void *) << ret->level2bits[0]);
|
|
2415 if (ret->root == NULL) {
|
|
2416 /*
|
|
2417 * We leak the rtree here, since there's no generic base
|
|
2418 * deallocation.
|
|
2419 */
|
|
2420 return (NULL);
|
|
2421 }
|
|
2422
|
|
2423 return (ret);
|
|
2424 }
|
|
2425
|
|
2426 /* The least significant bits of the key are ignored. */
|
|
2427 static inline void *
|
|
2428 malloc_rtree_get(malloc_rtree_t *rtree, uintptr_t key)
|
|
2429 {
|
|
2430 void *ret;
|
|
2431 uintptr_t subkey;
|
|
2432 unsigned i, lshift, height, bits;
|
|
2433 void **node, **child;
|
|
2434
|
|
2435 malloc_spin_lock(&rtree->lock);
|
|
2436 for (i = lshift = 0, height = rtree->height, node = rtree->root;
|
|
2437 i < height - 1;
|
|
2438 i++, lshift += bits, node = child) {
|
|
2439 bits = rtree->level2bits[i];
|
|
2440 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
|
|
2441 child = (void**)node[subkey];
|
|
2442 if (child == NULL) {
|
|
2443 malloc_spin_unlock(&rtree->lock);
|
|
2444 return (NULL);
|
|
2445 }
|
|
2446 }
|
|
2447
|
|
2448 /* node is a leaf, so it contains values rather than node pointers. */
|
|
2449 bits = rtree->level2bits[i];
|
|
2450 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
|
|
2451 ret = node[subkey];
|
|
2452 malloc_spin_unlock(&rtree->lock);
|
|
2453
|
|
2454 return (ret);
|
|
2455 }
|
|
2456
|
|
2457 static inline bool
|
|
2458 malloc_rtree_set(malloc_rtree_t *rtree, uintptr_t key, void *val)
|
|
2459 {
|
|
2460 uintptr_t subkey;
|
|
2461 unsigned i, lshift, height, bits;
|
|
2462 void **node, **child;
|
|
2463
|
|
2464 malloc_spin_lock(&rtree->lock);
|
|
2465 for (i = lshift = 0, height = rtree->height, node = rtree->root;
|
|
2466 i < height - 1;
|
|
2467 i++, lshift += bits, node = child) {
|
|
2468 bits = rtree->level2bits[i];
|
|
2469 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
|
|
2470 child = (void**)node[subkey];
|
|
2471 if (child == NULL) {
|
|
2472 child = (void**)base_calloc(1, sizeof(void *) <<
|
|
2473 rtree->level2bits[i+1]);
|
|
2474 if (child == NULL) {
|
|
2475 malloc_spin_unlock(&rtree->lock);
|
|
2476 return (true);
|
|
2477 }
|
|
2478 node[subkey] = child;
|
|
2479 }
|
|
2480 }
|
|
2481
|
|
2482 /* node is a leaf, so it contains values rather than node pointers. */
|
|
2483 bits = rtree->level2bits[i];
|
|
2484 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
|
|
2485 node[subkey] = val;
|
|
2486 malloc_spin_unlock(&rtree->lock);
|
|
2487
|
|
2488 return (false);
|
|
2489 }
|
|
2490 #endif
|
|
2491
|
|
2492 static void *
|
|
2493 chunk_alloc_mmap(size_t size, bool pagefile)
|
|
2494 {
|
|
2495 void *ret;
|
|
2496 #ifndef JEMALLOC_USES_MAP_ALIGN
|
|
2497 size_t offset;
|
|
2498 #endif
|
|
2499 int pfd;
|
|
2500
|
|
2501 #ifdef MALLOC_PAGEFILE
|
|
2502 if (opt_pagefile && pagefile) {
|
|
2503 pfd = pagefile_init(size);
|
|
2504 if (pfd == -1)
|
|
2505 return (NULL);
|
|
2506 } else
|
|
2507 #endif
|
|
2508 pfd = -1;
|
|
2509
|
|
2510 /*
|
|
2511 * Windows requires that there be a 1:1 mapping between VM
|
|
2512 * allocation/deallocation operations. Therefore, take care here to
|
|
2513 * acquire the final result via one mapping operation. This means
|
|
2514 * unmapping any preliminary result that is not correctly aligned.
|
|
2515 *
|
|
2516 * The MALLOC_PAGEFILE code also benefits from this mapping algorithm,
|
|
2517 * since it reduces the number of page files.
|
|
2518 */
|
|
2519
|
|
2520 #ifdef JEMALLOC_USES_MAP_ALIGN
|
|
2521 ret = pages_map_align(size, pfd, chunksize);
|
|
2522 #else
|
|
2523 ret = pages_map(NULL, size, pfd);
|
|
2524 if (ret == NULL)
|
|
2525 goto RETURN;
|
|
2526
|
|
2527 offset = CHUNK_ADDR2OFFSET(ret);
|
|
2528 if (offset != 0) {
|
|
2529 /* Deallocate, then try to allocate at (ret + size - offset). */
|
|
2530 pages_unmap(ret, size);
|
|
2531 ret = pages_map((void *)((uintptr_t)ret + size - offset), size,
|
|
2532 pfd);
|
|
2533 while (ret == NULL) {
|
|
2534 /*
|
|
2535 * Over-allocate in order to map a memory region that
|
|
2536 * is definitely large enough.
|
|
2537 */
|
|
2538 ret = pages_map(NULL, size + chunksize, -1);
|
|
2539 if (ret == NULL)
|
|
2540 goto RETURN;
|
|
2541 /*
|
|
2542 * Deallocate, then allocate the correct size, within
|
|
2543 * the over-sized mapping.
|
|
2544 */
|
|
2545 offset = CHUNK_ADDR2OFFSET(ret);
|
|
2546 pages_unmap(ret, size + chunksize);
|
|
2547 if (offset == 0)
|
|
2548 ret = pages_map(ret, size, pfd);
|
|
2549 else {
|
|
2550 ret = pages_map((void *)((uintptr_t)ret +
|
|
2551 chunksize - offset), size, pfd);
|
|
2552 }
|
|
2553 /*
|
|
2554 * Failure here indicates a race with another thread, so
|
|
2555 * try again.
|
|
2556 */
|
|
2557 }
|
|
2558 }
|
|
2559 RETURN:
|
|
2560 #endif
|
|
2561 #ifdef MALLOC_PAGEFILE
|
|
2562 if (pfd != -1)
|
|
2563 pagefile_close(pfd);
|
|
2564 #endif
|
|
2565 #ifdef MALLOC_STATS
|
|
2566 if (ret != NULL)
|
|
2567 stats_chunks.nchunks += (size / chunksize);
|
|
2568 #endif
|
|
2569 return (ret);
|
|
2570 }
|
|
2571
|
|
2572 #ifdef MALLOC_PAGEFILE
|
|
2573 static int
|
|
2574 pagefile_init(size_t size)
|
|
2575 {
|
|
2576 int ret;
|
|
2577 size_t i;
|
|
2578 char pagefile_path[PATH_MAX];
|
|
2579 char zbuf[MALLOC_PAGEFILE_WRITE_SIZE];
|
|
2580
|
|
2581 /*
|
|
2582 * Create a temporary file, then immediately unlink it so that it will
|
|
2583 * not persist.
|
|
2584 */
|
|
2585 strcpy(pagefile_path, pagefile_templ);
|
|
2586 ret = mkstemp(pagefile_path);
|
|
2587 if (ret == -1)
|
|
2588 return (ret);
|
|
2589 if (unlink(pagefile_path)) {
|
|
2590 char buf[STRERROR_BUF];
|
|
2591
|
|
2592 strerror_r(errno, buf, sizeof(buf));
|
|
2593 _malloc_message(_getprogname(), ": (malloc) Error in unlink(\"",
|
|
2594 pagefile_path, "\"):");
|
|
2595 _malloc_message(buf, "\n", "", "");
|
|
2596 if (opt_abort)
|
|
2597 abort();
|
|
2598 }
|
|
2599
|
|
2600 /*
|
|
2601 * Write sequential zeroes to the file in order to assure that disk
|
|
2602 * space is committed, with minimal fragmentation. It would be
|
|
2603 * sufficient to write one zero per disk block, but that potentially
|
|
2604 * results in more system calls, for no real gain.
|
|
2605 */
|
|
2606 memset(zbuf, 0, sizeof(zbuf));
|
|
2607 for (i = 0; i < size; i += sizeof(zbuf)) {
|
|
2608 if (write(ret, zbuf, sizeof(zbuf)) != sizeof(zbuf)) {
|
|
2609 if (errno != ENOSPC) {
|
|
2610 char buf[STRERROR_BUF];
|
|
2611
|
|
2612 strerror_r(errno, buf, sizeof(buf));
|
|
2613 _malloc_message(_getprogname(),
|
|
2614 ": (malloc) Error in write(): ", buf, "\n");
|
|
2615 if (opt_abort)
|
|
2616 abort();
|
|
2617 }
|
|
2618 pagefile_close(ret);
|
|
2619 return (-1);
|
|
2620 }
|
|
2621 }
|
|
2622
|
|
2623 return (ret);
|
|
2624 }
|
|
2625
|
|
2626 static void
|
|
2627 pagefile_close(int pfd)
|
|
2628 {
|
|
2629
|
|
2630 if (close(pfd)) {
|
|
2631 char buf[STRERROR_BUF];
|
|
2632
|
|
2633 strerror_r(errno, buf, sizeof(buf));
|
|
2634 _malloc_message(_getprogname(),
|
|
2635 ": (malloc) Error in close(): ", buf, "\n");
|
|
2636 if (opt_abort)
|
|
2637 abort();
|
|
2638 }
|
|
2639 }
|
|
2640 #endif
|
|
2641
|
|
2642 static void *
|
|
2643 chunk_recycle_reserve(size_t size, bool zero)
|
|
2644 {
|
|
2645 extent_node_t *node, key;
|
|
2646
|
|
2647 #ifdef MALLOC_DECOMMIT
|
|
2648 if (size != chunksize)
|
|
2649 return (NULL);
|
|
2650 #endif
|
|
2651
|
|
2652 key.addr = NULL;
|
|
2653 key.size = size;
|
|
2654 malloc_mutex_lock(&reserve_mtx);
|
|
2655 node = extent_tree_szad_nsearch(&reserve_chunks_szad, &key);
|
|
2656 if (node != NULL) {
|
|
2657 void *ret = node->addr;
|
|
2658
|
|
2659 /* Remove node from the tree. */
|
|
2660 extent_tree_szad_remove(&reserve_chunks_szad, node);
|
|
2661 #ifndef MALLOC_DECOMMIT
|
|
2662 if (node->size == size) {
|
|
2663 #else
|
|
2664 assert(node->size == size);
|
|
2665 #endif
|
|
2666 extent_tree_ad_remove(&reserve_chunks_ad, node);
|
|
2667 base_node_dealloc(node);
|
|
2668 #ifndef MALLOC_DECOMMIT
|
|
2669 } else {
|
|
2670 /*
|
|
2671 * Insert the remainder of node's address range as a
|
|
2672 * smaller chunk. Its position within reserve_chunks_ad
|
|
2673 * does not change.
|
|
2674 */
|
|
2675 assert(node->size > size);
|
|
2676 node->addr = (void *)((uintptr_t)node->addr + size);
|
|
2677 node->size -= size;
|
|
2678 extent_tree_szad_insert(&reserve_chunks_szad, node);
|
|
2679 }
|
|
2680 #endif
|
|
2681 reserve_cur -= size;
|
|
2682 /*
|
|
2683 * Try to replenish the reserve if this allocation depleted it.
|
|
2684 */
|
|
2685 #ifndef MALLOC_DECOMMIT
|
|
2686 if (reserve_cur < reserve_min) {
|
|
2687 size_t diff = reserve_min - reserve_cur;
|
|
2688 #else
|
|
2689 while (reserve_cur < reserve_min) {
|
|
2690 # define diff chunksize
|
|
2691 #endif
|
|
2692 void *chunk;
|
|
2693
|
|
2694 malloc_mutex_unlock(&reserve_mtx);
|
|
2695 chunk = chunk_alloc_mmap(diff, true);
|
|
2696 malloc_mutex_lock(&reserve_mtx);
|
|
2697 if (chunk == NULL) {
|
|
2698 uint64_t seq = 0;
|
|
2699
|
|
2700 do {
|
|
2701 seq = reserve_notify(RESERVE_CND_LOW,
|
|
2702 size, seq);
|
|
2703 if (seq == 0)
|
|
2704 goto MALLOC_OUT;
|
|
2705 } while (reserve_cur < reserve_min);
|
|
2706 } else {
|
|
2707 extent_node_t *node;
|
|
2708
|
|
2709 node = chunk_dealloc_reserve(chunk, diff);
|
|
2710 if (node == NULL) {
|
|
2711 uint64_t seq = 0;
|
|
2712
|
|
2713 pages_unmap(chunk, diff);
|
|
2714 do {
|
|
2715 seq = reserve_notify(
|
|
2716 RESERVE_CND_LOW, size, seq);
|
|
2717 if (seq == 0)
|
|
2718 goto MALLOC_OUT;
|
|
2719 } while (reserve_cur < reserve_min);
|
|
2720 }
|
|
2721 }
|
|
2722 }
|
|
2723 MALLOC_OUT:
|
|
2724 malloc_mutex_unlock(&reserve_mtx);
|
|
2725
|
|
2726 #ifdef MALLOC_DECOMMIT
|
|
2727 pages_commit(ret, size);
|
|
2728 # undef diff
|
|
2729 #else
|
|
2730 if (zero)
|
|
2731 memset(ret, 0, size);
|
|
2732 #endif
|
|
2733 return (ret);
|
|
2734 }
|
|
2735 malloc_mutex_unlock(&reserve_mtx);
|
|
2736
|
|
2737 return (NULL);
|
|
2738 }
|
|
2739
|
|
2740 static void *
|
|
2741 chunk_alloc(size_t size, bool zero, bool pagefile)
|
|
2742 {
|
|
2743 void *ret;
|
|
2744
|
|
2745 assert(size != 0);
|
|
2746 assert((size & chunksize_mask) == 0);
|
|
2747
|
|
2748 ret = chunk_recycle_reserve(size, zero);
|
|
2749 if (ret != NULL)
|
|
2750 goto RETURN;
|
|
2751
|
|
2752 ret = chunk_alloc_mmap(size, pagefile);
|
|
2753 if (ret != NULL) {
|
|
2754 goto RETURN;
|
|
2755 }
|
|
2756
|
|
2757 /* All strategies for allocation failed. */
|
|
2758 ret = NULL;
|
|
2759 RETURN:
|
|
2760 #ifdef MALLOC_STATS
|
|
2761 if (ret != NULL)
|
|
2762 stats_chunks.curchunks += (size / chunksize);
|
|
2763 if (stats_chunks.curchunks > stats_chunks.highchunks)
|
|
2764 stats_chunks.highchunks = stats_chunks.curchunks;
|
|
2765 #endif
|
|
2766
|
|
2767 #ifdef MALLOC_VALIDATE
|
|
2768 if (ret != NULL) {
|
|
2769 if (malloc_rtree_set(chunk_rtree, (uintptr_t)ret, ret)) {
|
|
2770 chunk_dealloc(ret, size);
|
|
2771 return (NULL);
|
|
2772 }
|
|
2773 }
|
|
2774 #endif
|
|
2775
|
|
2776 assert(CHUNK_ADDR2BASE(ret) == ret);
|
|
2777 return (ret);
|
|
2778 }
|
|
2779
|
|
2780 static extent_node_t *
|
|
2781 chunk_dealloc_reserve(void *chunk, size_t size)
|
|
2782 {
|
|
2783 extent_node_t *node;
|
|
2784
|
|
2785 #ifdef MALLOC_DECOMMIT
|
|
2786 if (size != chunksize)
|
|
2787 return (NULL);
|
|
2788 #else
|
|
2789 extent_node_t *prev, key;
|
|
2790
|
|
2791 key.addr = (void *)((uintptr_t)chunk + size);
|
|
2792 node = extent_tree_ad_nsearch(&reserve_chunks_ad, &key);
|
|
2793 /* Try to coalesce forward. */
|
|
2794 if (node != NULL && node->addr == key.addr) {
|
|
2795 /*
|
|
2796 * Coalesce chunk with the following address range. This does
|
|
2797 * not change the position within reserve_chunks_ad, so only
|
|
2798 * remove/insert from/into reserve_chunks_szad.
|
|
2799 */
|
|
2800 extent_tree_szad_remove(&reserve_chunks_szad, node);
|
|
2801 node->addr = chunk;
|
|
2802 node->size += size;
|
|
2803 extent_tree_szad_insert(&reserve_chunks_szad, node);
|
|
2804 } else {
|
|
2805 #endif
|
|
2806 /* Coalescing forward failed, so insert a new node. */
|
|
2807 node = base_node_alloc();
|
|
2808 if (node == NULL)
|
|
2809 return (NULL);
|
|
2810 node->addr = chunk;
|
|
2811 node->size = size;
|
|
2812 extent_tree_ad_insert(&reserve_chunks_ad, node);
|
|
2813 extent_tree_szad_insert(&reserve_chunks_szad, node);
|
|
2814 #ifndef MALLOC_DECOMMIT
|
|
2815 }
|
|
2816
|
|
2817 /* Try to coalesce backward. */
|
|
2818 prev = extent_tree_ad_prev(&reserve_chunks_ad, node);
|
|
2819 if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) ==
|
|
2820 chunk) {
|
|
2821 /*
|
|
2822 * Coalesce chunk with the previous address range. This does
|
|
2823 * not change the position within reserve_chunks_ad, so only
|
|
2824 * remove/insert node from/into reserve_chunks_szad.
|
|
2825 */
|
|
2826 extent_tree_szad_remove(&reserve_chunks_szad, prev);
|
|
2827 extent_tree_ad_remove(&reserve_chunks_ad, prev);
|
|
2828
|
|
2829 extent_tree_szad_remove(&reserve_chunks_szad, node);
|
|
2830 node->addr = prev->addr;
|
|
2831 node->size += prev->size;
|
|
2832 extent_tree_szad_insert(&reserve_chunks_szad, node);
|
|
2833
|
|
2834 base_node_dealloc(prev);
|
|
2835 }
|
|
2836 #endif
|
|
2837
|
|
2838 #ifdef MALLOC_DECOMMIT
|
|
2839 pages_decommit(chunk, size);
|
|
2840 #else
|
|
2841 madvise(chunk, size, MADV_FREE);
|
|
2842 #endif
|
|
2843
|
|
2844 reserve_cur += size;
|
|
2845 if (reserve_cur > reserve_max)
|
|
2846 reserve_shrink();
|
|
2847
|
|
2848 return (node);
|
|
2849 }
|
|
2850
|
|
2851 static void
|
|
2852 chunk_dealloc_mmap(void *chunk, size_t size)
|
|
2853 {
|
|
2854
|
|
2855 pages_unmap(chunk, size);
|
|
2856 }
|
|
2857
|
|
2858 static void
|
|
2859 chunk_dealloc(void *chunk, size_t size)
|
|
2860 {
|
|
2861 extent_node_t *node;
|
|
2862
|
|
2863 assert(chunk != NULL);
|
|
2864 assert(CHUNK_ADDR2BASE(chunk) == chunk);
|
|
2865 assert(size != 0);
|
|
2866 assert((size & chunksize_mask) == 0);
|
|
2867
|
|
2868 #ifdef MALLOC_STATS
|
|
2869 stats_chunks.curchunks -= (size / chunksize);
|
|
2870 #endif
|
|
2871 #ifdef MALLOC_VALIDATE
|
|
2872 malloc_rtree_set(chunk_rtree, (uintptr_t)chunk, NULL);
|
|
2873 #endif
|
|
2874
|
|
2875 /* Try to merge chunk into the reserve. */
|
|
2876 malloc_mutex_lock(&reserve_mtx);
|
|
2877 node = chunk_dealloc_reserve(chunk, size);
|
|
2878 malloc_mutex_unlock(&reserve_mtx);
|
|
2879 if (node == NULL)
|
|
2880 chunk_dealloc_mmap(chunk, size);
|
|
2881 }
|
|
2882
|
|
2883 /*
|
|
2884 * End chunk management functions.
|
|
2885 */
|
|
2886 /******************************************************************************/
|
|
2887 /*
|
|
2888 * Begin arena.
|
|
2889 */
|
|
2890
|
|
2891 /*
|
|
2892 * Choose an arena based on a per-thread value (fast-path code, calls slow-path
|
|
2893 * code if necessary).
|
|
2894 */
|
|
2895 static inline arena_t *
|
|
2896 choose_arena(void)
|
|
2897 {
|
|
2898 arena_t *ret;
|
|
2899
|
|
2900 /*
|
|
2901 * We can only use TLS if this is a PIC library, since for the static
|
|
2902 * library version, libc's malloc is used by TLS allocation, which
|
|
2903 * introduces a bootstrapping issue.
|
|
2904 */
|
|
2905 #ifndef NO_TLS
|
|
2906 if (__isthreaded == false) {
|
|
2907 /* Avoid the overhead of TLS for single-threaded operation. */
|
|
2908 return (arenas[0]);
|
|
2909 }
|
|
2910
|
|
2911 # ifdef MOZ_MEMORY_WINDOWS
|
|
2912 ret = (arena_t*)TlsGetValue(tlsIndex);
|
|
2913 # else
|
|
2914 ret = arenas_map;
|
|
2915 # endif
|
|
2916
|
|
2917 if (ret == NULL) {
|
|
2918 ret = choose_arena_hard();
|
|
2919 assert(ret != NULL);
|
|
2920 }
|
|
2921 #else
|
|
2922 if (__isthreaded && narenas > 1) {
|
|
2923 unsigned long ind;
|
|
2924
|
|
2925 /*
|
|
2926 * Hash _pthread_self() to one of the arenas. There is a prime
|
|
2927 * number of arenas, so this has a reasonable chance of
|
|
2928 * working. Even so, the hashing can be easily thwarted by
|
|
2929 * inconvenient _pthread_self() values. Without specific
|
|
2930 * knowledge of how _pthread_self() calculates values, we can't
|
|
2931 * easily do much better than this.
|
|
2932 */
|
|
2933 ind = (unsigned long) _pthread_self() % narenas;
|
|
2934
|
|
2935 /*
|
|
2936 * Optimistially assume that arenas[ind] has been initialized.
|
|
2937 * At worst, we find out that some other thread has already
|
|
2938 * done so, after acquiring the lock in preparation. Note that
|
|
2939 * this lazy locking also has the effect of lazily forcing
|
|
2940 * cache coherency; without the lock acquisition, there's no
|
|
2941 * guarantee that modification of arenas[ind] by another thread
|
|
2942 * would be seen on this CPU for an arbitrary amount of time.
|
|
2943 *
|
|
2944 * In general, this approach to modifying a synchronized value
|
|
2945 * isn't a good idea, but in this case we only ever modify the
|
|
2946 * value once, so things work out well.
|
|
2947 */
|
|
2948 ret = arenas[ind];
|
|
2949 if (ret == NULL) {
|
|
2950 /*
|
|
2951 * Avoid races with another thread that may have already
|
|
2952 * initialized arenas[ind].
|
|
2953 */
|
|
2954 malloc_spin_lock(&arenas_lock);
|
|
2955 if (arenas[ind] == NULL)
|
|
2956 ret = arenas_extend((unsigned)ind);
|
|
2957 else
|
|
2958 ret = arenas[ind];
|
|
2959 malloc_spin_unlock(&arenas_lock);
|
|
2960 }
|
|
2961 } else
|
|
2962 ret = arenas[0];
|
|
2963 #endif
|
|
2964
|
|
2965 assert(ret != NULL);
|
|
2966 return (ret);
|
|
2967 }
|
|
2968
|
|
2969 #ifndef NO_TLS
|
|
2970 /*
|
|
2971 * Choose an arena based on a per-thread value (slow-path code only, called
|
|
2972 * only by choose_arena()).
|
|
2973 */
|
|
2974 static arena_t *
|
|
2975 choose_arena_hard(void)
|
|
2976 {
|
|
2977 arena_t *ret;
|
|
2978
|
|
2979 assert(__isthreaded);
|
|
2980
|
|
2981 #ifdef MALLOC_BALANCE
|
|
2982 /* Seed the PRNG used for arena load balancing. */
|
|
2983 SPRN(balance, (uint32_t)(uintptr_t)(_pthread_self()));
|
|
2984 #endif
|
|
2985
|
|
2986 if (narenas > 1) {
|
|
2987 #ifdef MALLOC_BALANCE
|
|
2988 unsigned ind;
|
|
2989
|
|
2990 ind = PRN(balance, narenas_2pow);
|
|
2991 if ((ret = arenas[ind]) == NULL) {
|
|
2992 malloc_spin_lock(&arenas_lock);
|
|
2993 if ((ret = arenas[ind]) == NULL)
|
|
2994 ret = arenas_extend(ind);
|
|
2995 malloc_spin_unlock(&arenas_lock);
|
|
2996 }
|
|
2997 #else
|
|
2998 malloc_spin_lock(&arenas_lock);
|
|
2999 if ((ret = arenas[next_arena]) == NULL)
|
|
3000 ret = arenas_extend(next_arena);
|
|
3001 next_arena = (next_arena + 1) % narenas;
|
|
3002 malloc_spin_unlock(&arenas_lock);
|
|
3003 #endif
|
|
3004 } else
|
|
3005 ret = arenas[0];
|
|
3006
|
|
3007 #ifdef MOZ_MEMORY_WINDOWS
|
|
3008 TlsSetValue(tlsIndex, ret);
|
|
3009 #else
|
|
3010 arenas_map = ret;
|
|
3011 #endif
|
|
3012
|
|
3013 return (ret);
|
|
3014 }
|
|
3015 #endif
|
|
3016
|
|
3017 static inline int
|
|
3018 arena_chunk_comp(arena_chunk_t *a, arena_chunk_t *b)
|
|
3019 {
|
|
3020 uintptr_t a_chunk = (uintptr_t)a;
|
|
3021 uintptr_t b_chunk = (uintptr_t)b;
|
|
3022
|
|
3023 assert(a != NULL);
|
|
3024 assert(b != NULL);
|
|
3025
|
|
3026 return ((a_chunk > b_chunk) - (a_chunk < b_chunk));
|
|
3027 }
|
|
3028
|
|
3029 /* Wrap red-black tree macros in functions. */
|
|
3030 rb_wrap(static, arena_chunk_tree_dirty_, arena_chunk_tree_t,
|
|
3031 arena_chunk_t, link_dirty, arena_chunk_comp)
|
|
3032
|
|
3033 static inline int
|
|
3034 arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
|
|
3035 {
|
|
3036 uintptr_t a_mapelm = (uintptr_t)a;
|
|
3037 uintptr_t b_mapelm = (uintptr_t)b;
|
|
3038
|
|
3039 assert(a != NULL);
|
|
3040 assert(b != NULL);
|
|
3041
|
|
3042 return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm));
|
|
3043 }
|
|
3044
|
|
3045 /* Wrap red-black tree macros in functions. */
|
|
3046 rb_wrap(static, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, link,
|
|
3047 arena_run_comp)
|
|
3048
|
|
3049 static inline int
|
|
3050 arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
|
|
3051 {
|
|
3052 int ret;
|
|
3053 size_t a_size = a->bits & ~pagesize_mask;
|
|
3054 size_t b_size = b->bits & ~pagesize_mask;
|
|
3055
|
|
3056 ret = (a_size > b_size) - (a_size < b_size);
|
|
3057 if (ret == 0) {
|
|
3058 uintptr_t a_mapelm, b_mapelm;
|
|
3059
|
|
3060 if ((a->bits & CHUNK_MAP_KEY) == 0)
|
|
3061 a_mapelm = (uintptr_t)a;
|
|
3062 else {
|
|
3063 /*
|
|
3064 * Treat keys as though they are lower than anything
|
|
3065 * else.
|
|
3066 */
|
|
3067 a_mapelm = 0;
|
|
3068 }
|
|
3069 b_mapelm = (uintptr_t)b;
|
|
3070
|
|
3071 ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm);
|
|
3072 }
|
|
3073
|
|
3074 return (ret);
|
|
3075 }
|
|
3076
|
|
3077 /* Wrap red-black tree macros in functions. */
|
|
3078 rb_wrap(static, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, link,
|
|
3079 arena_avail_comp)
|
|
3080
|
|
3081 static inline void *
|
|
3082 arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin)
|
|
3083 {
|
|
3084 void *ret;
|
|
3085 unsigned i, mask, bit, regind;
|
|
3086
|
|
3087 assert(run->magic == ARENA_RUN_MAGIC);
|
|
3088 assert(run->regs_minelm < bin->regs_mask_nelms);
|
|
3089
|
|
3090 /*
|
|
3091 * Move the first check outside the loop, so that run->regs_minelm can
|
|
3092 * be updated unconditionally, without the possibility of updating it
|
|
3093 * multiple times.
|
|
3094 */
|
|
3095 i = run->regs_minelm;
|
|
3096 mask = run->regs_mask[i];
|
|
3097 if (mask != 0) {
|
|
3098 /* Usable allocation found. */
|
|
3099 bit = ffs((int)mask) - 1;
|
|
3100
|
|
3101 regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
|
|
3102 assert(regind < bin->nregs);
|
|
3103 ret = (void *)(((uintptr_t)run) + bin->reg0_offset
|
|
3104 + (bin->reg_size * regind));
|
|
3105
|
|
3106 /* Clear bit. */
|
|
3107 mask ^= (1U << bit);
|
|
3108 run->regs_mask[i] = mask;
|
|
3109
|
|
3110 return (ret);
|
|
3111 }
|
|
3112
|
|
3113 for (i++; i < bin->regs_mask_nelms; i++) {
|
|
3114 mask = run->regs_mask[i];
|
|
3115 if (mask != 0) {
|
|
3116 /* Usable allocation found. */
|
|
3117 bit = ffs((int)mask) - 1;
|
|
3118
|
|
3119 regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
|
|
3120 assert(regind < bin->nregs);
|
|
3121 ret = (void *)(((uintptr_t)run) + bin->reg0_offset
|
|
3122 + (bin->reg_size * regind));
|
|
3123
|
|
3124 /* Clear bit. */
|
|
3125 mask ^= (1U << bit);
|
|
3126 run->regs_mask[i] = mask;
|
|
3127
|
|
3128 /*
|
|
3129 * Make a note that nothing before this element
|
|
3130 * contains a free region.
|
|
3131 */
|
|
3132 run->regs_minelm = i; /* Low payoff: + (mask == 0); */
|
|
3133
|
|
3134 return (ret);
|
|
3135 }
|
|
3136 }
|
|
3137 /* Not reached. */
|
|
3138 assert(0);
|
|
3139 return (NULL);
|
|
3140 }
|
|
3141
|
|
3142 static inline void
|
|
3143 arena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size)
|
|
3144 {
|
|
3145 /*
|
|
3146 * To divide by a number D that is not a power of two we multiply
|
|
3147 * by (2^21 / D) and then right shift by 21 positions.
|
|
3148 *
|
|
3149 * X / D
|
|
3150 *
|
|
3151 * becomes
|
|
3152 *
|
|
3153 * (X * size_invs[(D >> QUANTUM_2POW_MIN) - 3]) >> SIZE_INV_SHIFT
|
|
3154 */
|
|
3155 #define SIZE_INV_SHIFT 21
|
|
3156 #define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s << QUANTUM_2POW_MIN)) + 1)
|
|
3157 static const unsigned size_invs[] = {
|
|
3158 SIZE_INV(3),
|
|
3159 SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
|
|
3160 SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
|
|
3161 SIZE_INV(12),SIZE_INV(13), SIZE_INV(14), SIZE_INV(15),
|
|
3162 SIZE_INV(16),SIZE_INV(17), SIZE_INV(18), SIZE_INV(19),
|
|
3163 SIZE_INV(20),SIZE_INV(21), SIZE_INV(22), SIZE_INV(23),
|
|
3164 SIZE_INV(24),SIZE_INV(25), SIZE_INV(26), SIZE_INV(27),
|
|
3165 SIZE_INV(28),SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
|
|
3166 #if (QUANTUM_2POW_MIN < 4)
|
|
3167 ,
|
|
3168 SIZE_INV(32), SIZE_INV(33), SIZE_INV(34), SIZE_INV(35),
|
|
3169 SIZE_INV(36), SIZE_INV(37), SIZE_INV(38), SIZE_INV(39),
|
|
3170 SIZE_INV(40), SIZE_INV(41), SIZE_INV(42), SIZE_INV(43),
|
|
3171 SIZE_INV(44), SIZE_INV(45), SIZE_INV(46), SIZE_INV(47),
|
|
3172 SIZE_INV(48), SIZE_INV(49), SIZE_INV(50), SIZE_INV(51),
|
|
3173 SIZE_INV(52), SIZE_INV(53), SIZE_INV(54), SIZE_INV(55),
|
|
3174 SIZE_INV(56), SIZE_INV(57), SIZE_INV(58), SIZE_INV(59),
|
|
3175 SIZE_INV(60), SIZE_INV(61), SIZE_INV(62), SIZE_INV(63)
|
|
3176 #endif
|
|
3177 };
|
|
3178 unsigned diff, regind, elm, bit;
|
|
3179
|
|
3180 assert(run->magic == ARENA_RUN_MAGIC);
|
|
3181 assert(((sizeof(size_invs)) / sizeof(unsigned)) + 3
|
|
3182 >= (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN));
|
|
3183
|
|
3184 /*
|
|
3185 * Avoid doing division with a variable divisor if possible. Using
|
|
3186 * actual division here can reduce allocator throughput by over 20%!
|
|
3187 */
|
|
3188 diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset);
|
|
3189 if ((size & (size - 1)) == 0) {
|
|
3190 /*
|
|
3191 * log2_table allows fast division of a power of two in the
|
|
3192 * [1..128] range.
|
|
3193 *
|
|
3194 * (x / divisor) becomes (x >> log2_table[divisor - 1]).
|
|
3195 */
|
|
3196 static const unsigned char log2_table[] = {
|
|
3197 0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4,
|
|
3198 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,
|
|
3199 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
3200 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6,
|
|
3201 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
3202 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
3203 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
3204 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
|
|
3205 };
|
|
3206
|
|
3207 if (size <= 128)
|
|
3208 regind = (diff >> log2_table[size - 1]);
|
|
3209 else if (size <= 32768)
|
|
3210 regind = diff >> (8 + log2_table[(size >> 8) - 1]);
|
|
3211 else {
|
|
3212 /*
|
|
3213 * The run size is too large for us to use the lookup
|
|
3214 * table. Use real division.
|
|
3215 */
|
|
3216 regind = diff / size;
|
|
3217 }
|
|
3218 } else if (size <= ((sizeof(size_invs) / sizeof(unsigned))
|
|
3219 << QUANTUM_2POW_MIN) + 2) {
|
|
3220 regind = size_invs[(size >> QUANTUM_2POW_MIN) - 3] * diff;
|
|
3221 regind >>= SIZE_INV_SHIFT;
|
|
3222 } else {
|
|
3223 /*
|
|
3224 * size_invs isn't large enough to handle this size class, so
|
|
3225 * calculate regind using actual division. This only happens
|
|
3226 * if the user increases small_max via the 'S' runtime
|
|
3227 * configuration option.
|
|
3228 */
|
|
3229 regind = diff / size;
|
|
3230 };
|
|
3231 assert(diff == regind * size);
|
|
3232 assert(regind < bin->nregs);
|
|
3233
|
|
3234 elm = regind >> (SIZEOF_INT_2POW + 3);
|
|
3235 if (elm < run->regs_minelm)
|
|
3236 run->regs_minelm = elm;
|
|
3237 bit = regind - (elm << (SIZEOF_INT_2POW + 3));
|
|
3238 assert((run->regs_mask[elm] & (1U << bit)) == 0);
|
|
3239 run->regs_mask[elm] |= (1U << bit);
|
|
3240 #undef SIZE_INV
|
|
3241 #undef SIZE_INV_SHIFT
|
|
3242 }
|
|
3243
|
|
3244 static void
|
|
3245 arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
|
|
3246 bool zero)
|
|
3247 {
|
|
3248 arena_chunk_t *chunk;
|
|
3249 size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i;
|
|
3250
|
|
3251 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
|
|
3252 old_ndirty = chunk->ndirty;
|
|
3253 run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk)
|
|
3254 >> pagesize_2pow);
|
|
3255 total_pages = (chunk->map[run_ind].bits & ~pagesize_mask) >>
|
|
3256 pagesize_2pow;
|
|
3257 need_pages = (size >> pagesize_2pow);
|
|
3258 assert(need_pages > 0);
|
|
3259 assert(need_pages <= total_pages);
|
|
3260 rem_pages = total_pages - need_pages;
|
|
3261
|
|
3262 arena_avail_tree_remove(&arena->runs_avail, &chunk->map[run_ind]);
|
|
3263
|
|
3264 /* Keep track of trailing unused pages for later use. */
|
|
3265 if (rem_pages > 0) {
|
|
3266 chunk->map[run_ind+need_pages].bits = (rem_pages <<
|
|
3267 pagesize_2pow) | (chunk->map[run_ind+need_pages].bits &
|
|
3268 pagesize_mask);
|
|
3269 chunk->map[run_ind+total_pages-1].bits = (rem_pages <<
|
|
3270 pagesize_2pow) | (chunk->map[run_ind+total_pages-1].bits &
|
|
3271 pagesize_mask);
|
|
3272 arena_avail_tree_insert(&arena->runs_avail,
|
|
3273 &chunk->map[run_ind+need_pages]);
|
|
3274 }
|
|
3275
|
|
3276 for (i = 0; i < need_pages; i++) {
|
|
3277 #ifdef MALLOC_DECOMMIT
|
|
3278 /*
|
|
3279 * Commit decommitted pages if necessary. If a decommitted
|
|
3280 * page is encountered, commit all needed adjacent decommitted
|
|
3281 * pages in one operation, in order to reduce system call
|
|
3282 * overhead.
|
|
3283 */
|
|
3284 if (chunk->map[run_ind + i].bits & CHUNK_MAP_DECOMMITTED) {
|
|
3285 size_t j;
|
|
3286
|
|
3287 /*
|
|
3288 * Advance i+j to just past the index of the last page
|
|
3289 * to commit. Clear CHUNK_MAP_DECOMMITTED along the
|
|
3290 * way.
|
|
3291 */
|
|
3292 for (j = 0; i + j < need_pages && (chunk->map[run_ind +
|
|
3293 i + j].bits & CHUNK_MAP_DECOMMITTED); j++) {
|
|
3294 chunk->map[run_ind + i + j].bits ^=
|
|
3295 CHUNK_MAP_DECOMMITTED;
|
|
3296 }
|
|
3297
|
|
3298 pages_commit((void *)((uintptr_t)chunk + ((run_ind + i)
|
|
3299 << pagesize_2pow)), (j << pagesize_2pow));
|
|
3300 # ifdef MALLOC_STATS
|
|
3301 arena->stats.ncommit++;
|
|
3302 # endif
|
|
3303 } else /* No need to zero since commit zeros. */
|
|
3304 #endif
|
|
3305
|
|
3306 /* Zero if necessary. */
|
|
3307 if (zero) {
|
|
3308 if ((chunk->map[run_ind + i].bits & CHUNK_MAP_ZEROED)
|
|
3309 == 0) {
|
|
3310 VALGRIND_MALLOCLIKE_BLOCK((void *)((uintptr_t)
|
|
3311 chunk + ((run_ind + i) << pagesize_2pow)),
|
|
3312 pagesize, 0, false);
|
|
3313 memset((void *)((uintptr_t)chunk + ((run_ind
|
|
3314 + i) << pagesize_2pow)), 0, pagesize);
|
|
3315 VALGRIND_FREELIKE_BLOCK((void *)((uintptr_t)
|
|
3316 chunk + ((run_ind + i) << pagesize_2pow)),
|
|
3317 0);
|
|
3318 /* CHUNK_MAP_ZEROED is cleared below. */
|
|
3319 }
|
|
3320 }
|
|
3321
|
|
3322 /* Update dirty page accounting. */
|
|
3323 if (chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) {
|
|
3324 chunk->ndirty--;
|
|
3325 arena->ndirty--;
|
|
3326 /* CHUNK_MAP_DIRTY is cleared below. */
|
|
3327 }
|
|
3328
|
|
3329 /* Initialize the chunk map. */
|
|
3330 if (large) {
|
|
3331 chunk->map[run_ind + i].bits = CHUNK_MAP_LARGE
|
|
3332 | CHUNK_MAP_ALLOCATED;
|
|
3333 } else {
|
|
3334 chunk->map[run_ind + i].bits = (size_t)run
|
|
3335 | CHUNK_MAP_ALLOCATED;
|
|
3336 }
|
|
3337 }
|
|
3338
|
|
3339 /*
|
|
3340 * Set the run size only in the first element for large runs. This is
|
|
3341 * primarily a debugging aid, since the lack of size info for trailing
|
|
3342 * pages only matters if the application tries to operate on an
|
|
3343 * interior pointer.
|
|
3344 */
|
|
3345 if (large)
|
|
3346 chunk->map[run_ind].bits |= size;
|
|
3347
|
|
3348 if (chunk->ndirty == 0 && old_ndirty > 0)
|
|
3349 arena_chunk_tree_dirty_remove(&arena->chunks_dirty, chunk);
|
|
3350 }
|
|
3351
|
|
3352 static void
|
|
3353 arena_chunk_init(arena_t *arena, arena_chunk_t *chunk)
|
|
3354 {
|
|
3355 arena_run_t *run;
|
|
3356 size_t i;
|
|
3357
|
|
3358 VALGRIND_MALLOCLIKE_BLOCK(chunk, (arena_chunk_header_npages <<
|
|
3359 pagesize_2pow), 0, false);
|
|
3360 #ifdef MALLOC_STATS
|
|
3361 arena->stats.mapped += chunksize;
|
|
3362 #endif
|
|
3363
|
|
3364 chunk->arena = arena;
|
|
3365
|
|
3366 /*
|
|
3367 * Claim that no pages are in use, since the header is merely overhead.
|
|
3368 */
|
|
3369 chunk->ndirty = 0;
|
|
3370
|
|
3371 /* Initialize the map to contain one maximal free untouched run. */
|
|
3372 run = (arena_run_t *)((uintptr_t)chunk + (arena_chunk_header_npages <<
|
|
3373 pagesize_2pow));
|
|
3374 for (i = 0; i < arena_chunk_header_npages; i++)
|
|
3375 chunk->map[i].bits = 0;
|
|
3376 chunk->map[i].bits = arena_maxclass
|
|
3377 #ifdef MALLOC_DECOMMIT
|
|
3378 | CHUNK_MAP_DECOMMITTED
|
|
3379 #endif
|
|
3380 | CHUNK_MAP_ZEROED;
|
|
3381 for (i++; i < chunk_npages-1; i++) {
|
|
3382 chunk->map[i].bits =
|
|
3383 #ifdef MALLOC_DECOMMIT
|
|
3384 CHUNK_MAP_DECOMMITTED |
|
|
3385 #endif
|
|
3386 CHUNK_MAP_ZEROED;
|
|
3387 }
|
|
3388 chunk->map[chunk_npages-1].bits = arena_maxclass
|
|
3389 #ifdef MALLOC_DECOMMIT
|
|
3390 | CHUNK_MAP_DECOMMITTED
|
|
3391 #endif
|
|
3392 | CHUNK_MAP_ZEROED;
|
|
3393
|
|
3394 #ifdef MALLOC_DECOMMIT
|
|
3395 /*
|
|
3396 * Start out decommitted, in order to force a closer correspondence
|
|
3397 * between dirty pages and committed untouched pages.
|
|
3398 */
|
|
3399 pages_decommit(run, arena_maxclass);
|
|
3400 # ifdef MALLOC_STATS
|
|
3401 arena->stats.ndecommit++;
|
|
3402 arena->stats.decommitted += (chunk_npages - arena_chunk_header_npages);
|
|
3403 # endif
|
|
3404 #endif
|
|
3405
|
|
3406 /* Insert the run into the runs_avail tree. */
|
|
3407 arena_avail_tree_insert(&arena->runs_avail,
|
|
3408 &chunk->map[arena_chunk_header_npages]);
|
|
3409 }
|
|
3410
|
|
3411 static void
|
|
3412 arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk)
|
|
3413 {
|
|
3414
|
|
3415 if (arena->spare != NULL) {
|
|
3416 if (arena->spare->ndirty > 0) {
|
|
3417 arena_chunk_tree_dirty_remove(
|
|
3418 &chunk->arena->chunks_dirty, arena->spare);
|
|
3419 arena->ndirty -= arena->spare->ndirty;
|
|
3420 }
|
|
3421 VALGRIND_FREELIKE_BLOCK(arena->spare, 0);
|
|
3422 chunk_dealloc((void *)arena->spare, chunksize);
|
|
3423 #ifdef MALLOC_STATS
|
|
3424 arena->stats.mapped -= chunksize;
|
|
3425 #endif
|
|
3426 }
|
|
3427
|
|
3428 /*
|
|
3429 * Remove run from runs_avail, regardless of whether this chunk
|
|
3430 * will be cached, so that the arena does not use it. Dirty page
|
|
3431 * flushing only uses the chunks_dirty tree, so leaving this chunk in
|
|
3432 * the chunks_* trees is sufficient for that purpose.
|
|
3433 */
|
|
3434 arena_avail_tree_remove(&arena->runs_avail,
|
|
3435 &chunk->map[arena_chunk_header_npages]);
|
|
3436
|
|
3437 arena->spare = chunk;
|
|
3438 }
|
|
3439
|
|
3440 static arena_run_t *
|
|
3441 arena_run_alloc(arena_t *arena, arena_bin_t *bin, size_t size, bool large,
|
|
3442 bool zero)
|
|
3443 {
|
|
3444 arena_chunk_t *chunk;
|
|
3445 arena_run_t *run;
|
|
3446 arena_chunk_map_t *mapelm, key;
|
|
3447
|
|
3448 assert(size <= arena_maxclass);
|
|
3449 assert((size & pagesize_mask) == 0);
|
|
3450
|
|
3451 chunk = NULL;
|
|
3452 while (true) {
|
|
3453 /* Search the arena's chunks for the lowest best fit. */
|
|
3454 key.bits = size | CHUNK_MAP_KEY;
|
|
3455 mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key);
|
|
3456 if (mapelm != NULL) {
|
|
3457 arena_chunk_t *run_chunk = (arena_chunk_t*)CHUNK_ADDR2BASE(mapelm);
|
|
3458 size_t pageind = ((uintptr_t)mapelm -
|
|
3459 (uintptr_t)run_chunk->map) /
|
|
3460 sizeof(arena_chunk_map_t);
|
|
3461
|
|
3462 if (chunk != NULL)
|
|
3463 chunk_dealloc(chunk, chunksize);
|
|
3464 run = (arena_run_t *)((uintptr_t)run_chunk + (pageind
|
|
3465 << pagesize_2pow));
|
|
3466 arena_run_split(arena, run, size, large, zero);
|
|
3467 return (run);
|
|
3468 }
|
|
3469
|
|
3470 if (arena->spare != NULL) {
|
|
3471 /* Use the spare. */
|
|
3472 chunk = arena->spare;
|
|
3473 arena->spare = NULL;
|
|
3474 run = (arena_run_t *)((uintptr_t)chunk +
|
|
3475 (arena_chunk_header_npages << pagesize_2pow));
|
|
3476 /* Insert the run into the runs_avail tree. */
|
|
3477 arena_avail_tree_insert(&arena->runs_avail,
|
|
3478 &chunk->map[arena_chunk_header_npages]);
|
|
3479 arena_run_split(arena, run, size, large, zero);
|
|
3480 return (run);
|
|
3481 }
|
|
3482
|
|
3483 /*
|
|
3484 * No usable runs. Create a new chunk from which to allocate
|
|
3485 * the run.
|
|
3486 */
|
|
3487 if (chunk == NULL) {
|
|
3488 uint64_t chunk_seq;
|
|
3489
|
|
3490 /*
|
|
3491 * Record the chunk allocation sequence number in order
|
|
3492 * to detect races.
|
|
3493 */
|
|
3494 arena->chunk_seq++;
|
|
3495 chunk_seq = arena->chunk_seq;
|
|
3496
|
|
3497 /*
|
|
3498 * Drop the arena lock while allocating a chunk, since
|
|
3499 * reserve notifications may cause recursive
|
|
3500 * allocation. Dropping the lock here opens an
|
|
3501 * allocataion race, but we recover.
|
|
3502 */
|
|
3503 malloc_mutex_unlock(&arena->lock);
|
|
3504 chunk = (arena_chunk_t *)chunk_alloc(chunksize, true,
|
|
3505 true);
|
|
3506 malloc_mutex_lock(&arena->lock);
|
|
3507
|
|
3508 /*
|
|
3509 * Check whether a race allowed a usable run to appear.
|
|
3510 */
|
|
3511 if (bin != NULL && (run = bin->runcur) != NULL &&
|
|
3512 run->nfree > 0) {
|
|
3513 if (chunk != NULL)
|
|
3514 chunk_dealloc(chunk, chunksize);
|
|
3515 return (run);
|
|
3516 }
|
|
3517
|
|
3518 /*
|
|
3519 * If this thread raced with another such that multiple
|
|
3520 * chunks were allocated, make sure that there is still
|
|
3521 * inadequate space before using this chunk.
|
|
3522 */
|
|
3523 if (chunk_seq != arena->chunk_seq)
|
|
3524 continue;
|
|
3525
|
|
3526 /*
|
|
3527 * Check for an error *after* checking for a race,
|
|
3528 * since a race could also cause a transient OOM
|
|
3529 * condition.
|
|
3530 */
|
|
3531 if (chunk == NULL)
|
|
3532 return (NULL);
|
|
3533 }
|
|
3534
|
|
3535 arena_chunk_init(arena, chunk);
|
|
3536 run = (arena_run_t *)((uintptr_t)chunk +
|
|
3537 (arena_chunk_header_npages << pagesize_2pow));
|
|
3538 /* Update page map. */
|
|
3539 arena_run_split(arena, run, size, large, zero);
|
|
3540 return (run);
|
|
3541 }
|
|
3542 }
|
|
3543
|
|
3544 static void
|
|
3545 arena_purge(arena_t *arena)
|
|
3546 {
|
|
3547 arena_chunk_t *chunk;
|
|
3548 size_t i, npages;
|
|
3549 #ifdef MALLOC_DEBUG
|
|
3550 size_t ndirty = 0;
|
|
3551 rb_foreach_begin(arena_chunk_t, link_dirty, &arena->chunks_dirty,
|
|
3552 chunk) {
|
|
3553 ndirty += chunk->ndirty;
|
|
3554 } rb_foreach_end(arena_chunk_t, link_dirty, &arena->chunks_dirty, chunk)
|
|
3555 assert(ndirty == arena->ndirty);
|
|
3556 #endif
|
|
3557 assert(arena->ndirty > opt_dirty_max);
|
|
3558
|
|
3559 #ifdef MALLOC_STATS
|
|
3560 arena->stats.npurge++;
|
|
3561 #endif
|
|
3562
|
|
3563 /*
|
|
3564 * Iterate downward through chunks until enough dirty memory has been
|
|
3565 * purged. Terminate as soon as possible in order to minimize the
|
|
3566 * number of system calls, even if a chunk has only been partially
|
|
3567 * purged.
|
|
3568 */
|
|
3569 while (arena->ndirty > (opt_dirty_max >> 1)) {
|
|
3570 chunk = arena_chunk_tree_dirty_last(&arena->chunks_dirty);
|
|
3571 assert(chunk != NULL);
|
|
3572
|
|
3573 for (i = chunk_npages - 1; chunk->ndirty > 0; i--) {
|
|
3574 assert(i >= arena_chunk_header_npages);
|
|
3575
|
|
3576 if (chunk->map[i].bits & CHUNK_MAP_DIRTY) {
|
|
3577 #ifdef MALLOC_DECOMMIT
|
|
3578 assert((chunk->map[i].bits &
|
|
3579 CHUNK_MAP_DECOMMITTED) == 0);
|
|
3580 #endif
|
|
3581 chunk->map[i].bits ^=
|
|
3582 #ifdef MALLOC_DECOMMIT
|
|
3583 CHUNK_MAP_DECOMMITTED |
|
|
3584 #endif
|
|
3585 CHUNK_MAP_DIRTY;
|
|
3586 /* Find adjacent dirty run(s). */
|
|
3587 for (npages = 1; i > arena_chunk_header_npages
|
|
3588 && (chunk->map[i - 1].bits &
|
|
3589 CHUNK_MAP_DIRTY); npages++) {
|
|
3590 i--;
|
|
3591 #ifdef MALLOC_DECOMMIT
|
|
3592 assert((chunk->map[i].bits &
|
|
3593 CHUNK_MAP_DECOMMITTED) == 0);
|
|
3594 #endif
|
|
3595 chunk->map[i].bits ^=
|
|
3596 #ifdef MALLOC_DECOMMIT
|
|
3597 CHUNK_MAP_DECOMMITTED |
|
|
3598 #endif
|
|
3599 CHUNK_MAP_DIRTY;
|
|
3600 }
|
|
3601 chunk->ndirty -= npages;
|
|
3602 arena->ndirty -= npages;
|
|
3603
|
|
3604 #ifdef MALLOC_DECOMMIT
|
|
3605 pages_decommit((void *)((uintptr_t)
|
|
3606 chunk + (i << pagesize_2pow)),
|
|
3607 (npages << pagesize_2pow));
|
|
3608 # ifdef MALLOC_STATS
|
|
3609 arena->stats.ndecommit++;
|
|
3610 arena->stats.decommitted += npages;
|
|
3611 # endif
|
|
3612 #else
|
|
3613 madvise((void *)((uintptr_t)chunk + (i <<
|
|
3614 pagesize_2pow)), (npages << pagesize_2pow),
|
|
3615 MADV_FREE);
|
|
3616 #endif
|
|
3617 #ifdef MALLOC_STATS
|
|
3618 arena->stats.nmadvise++;
|
|
3619 arena->stats.purged += npages;
|
|
3620 #endif
|
|
3621 if (arena->ndirty <= (opt_dirty_max >> 1))
|
|
3622 break;
|
|
3623 }
|
|
3624 }
|
|
3625
|
|
3626 if (chunk->ndirty == 0) {
|
|
3627 arena_chunk_tree_dirty_remove(&arena->chunks_dirty,
|
|
3628 chunk);
|
|
3629 }
|
|
3630 }
|
|
3631 }
|
|
3632
|
|
3633 static void
|
|
3634 arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
|
|
3635 {
|
|
3636 arena_chunk_t *chunk;
|
|
3637 size_t size, run_ind, run_pages;
|
|
3638
|
|
3639 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
|
|
3640 run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk)
|
|
3641 >> pagesize_2pow);
|
|
3642 assert(run_ind >= arena_chunk_header_npages);
|
|
3643 assert(run_ind < chunk_npages);
|
|
3644 if ((chunk->map[run_ind].bits & CHUNK_MAP_LARGE) != 0)
|
|
3645 size = chunk->map[run_ind].bits & ~pagesize_mask;
|
|
3646 else
|
|
3647 size = run->bin->run_size;
|
|
3648 run_pages = (size >> pagesize_2pow);
|
|
3649
|
|
3650 /* Mark pages as unallocated in the chunk map. */
|
|
3651 if (dirty) {
|
|
3652 size_t i;
|
|
3653
|
|
3654 for (i = 0; i < run_pages; i++) {
|
|
3655 assert((chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY)
|
|
3656 == 0);
|
|
3657 chunk->map[run_ind + i].bits = CHUNK_MAP_DIRTY;
|
|
3658 }
|
|
3659
|
|
3660 if (chunk->ndirty == 0) {
|
|
3661 arena_chunk_tree_dirty_insert(&arena->chunks_dirty,
|
|
3662 chunk);
|
|
3663 }
|
|
3664 chunk->ndirty += run_pages;
|
|
3665 arena->ndirty += run_pages;
|
|
3666 } else {
|
|
3667 size_t i;
|
|
3668
|
|
3669 for (i = 0; i < run_pages; i++) {
|
|
3670 chunk->map[run_ind + i].bits &= ~(CHUNK_MAP_LARGE |
|
|
3671 CHUNK_MAP_ALLOCATED);
|
|
3672 }
|
|
3673 }
|
|
3674 chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
|
|
3675 pagesize_mask);
|
|
3676 chunk->map[run_ind+run_pages-1].bits = size |
|
|
3677 (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
|
|
3678
|
|
3679 /* Try to coalesce forward. */
|
|
3680 if (run_ind + run_pages < chunk_npages &&
|
|
3681 (chunk->map[run_ind+run_pages].bits & CHUNK_MAP_ALLOCATED) == 0) {
|
|
3682 size_t nrun_size = chunk->map[run_ind+run_pages].bits &
|
|
3683 ~pagesize_mask;
|
|
3684
|
|
3685 /*
|
|
3686 * Remove successor from runs_avail; the coalesced run is
|
|
3687 * inserted later.
|
|
3688 */
|
|
3689 arena_avail_tree_remove(&arena->runs_avail,
|
|
3690 &chunk->map[run_ind+run_pages]);
|
|
3691
|
|
3692 size += nrun_size;
|
|
3693 run_pages = size >> pagesize_2pow;
|
|
3694
|
|
3695 assert((chunk->map[run_ind+run_pages-1].bits & ~pagesize_mask)
|
|
3696 == nrun_size);
|
|
3697 chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
|
|
3698 pagesize_mask);
|
|
3699 chunk->map[run_ind+run_pages-1].bits = size |
|
|
3700 (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
|
|
3701 }
|
|
3702
|
|
3703 /* Try to coalesce backward. */
|
|
3704 if (run_ind > arena_chunk_header_npages && (chunk->map[run_ind-1].bits &
|
|
3705 CHUNK_MAP_ALLOCATED) == 0) {
|
|
3706 size_t prun_size = chunk->map[run_ind-1].bits & ~pagesize_mask;
|
|
3707
|
|
3708 run_ind -= prun_size >> pagesize_2pow;
|
|
3709
|
|
3710 /*
|
|
3711 * Remove predecessor from runs_avail; the coalesced run is
|
|
3712 * inserted later.
|
|
3713 */
|
|
3714 arena_avail_tree_remove(&arena->runs_avail,
|
|
3715 &chunk->map[run_ind]);
|
|
3716
|
|
3717 size += prun_size;
|
|
3718 run_pages = size >> pagesize_2pow;
|
|
3719
|
|
3720 assert((chunk->map[run_ind].bits & ~pagesize_mask) ==
|
|
3721 prun_size);
|
|
3722 chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
|
|
3723 pagesize_mask);
|
|
3724 chunk->map[run_ind+run_pages-1].bits = size |
|
|
3725 (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
|
|
3726 }
|
|
3727
|
|
3728 /* Insert into runs_avail, now that coalescing is complete. */
|
|
3729 arena_avail_tree_insert(&arena->runs_avail, &chunk->map[run_ind]);
|
|
3730
|
|
3731 /* Deallocate chunk if it is now completely unused. */
|
|
3732 if ((chunk->map[arena_chunk_header_npages].bits & (~pagesize_mask |
|
|
3733 CHUNK_MAP_ALLOCATED)) == arena_maxclass)
|
|
3734 arena_chunk_dealloc(arena, chunk);
|
|
3735
|
|
3736 /* Enforce opt_dirty_max. */
|
|
3737 if (arena->ndirty > opt_dirty_max)
|
|
3738 arena_purge(arena);
|
|
3739 }
|
|
3740
|
|
3741 static void
|
|
3742 arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
|
|
3743 size_t oldsize, size_t newsize)
|
|
3744 {
|
|
3745 size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
|
|
3746 size_t head_npages = (oldsize - newsize) >> pagesize_2pow;
|
|
3747
|
|
3748 assert(oldsize > newsize);
|
|
3749
|
|
3750 /*
|
|
3751 * Update the chunk map so that arena_run_dalloc() can treat the
|
|
3752 * leading run as separately allocated.
|
|
3753 */
|
|
3754 chunk->map[pageind].bits = (oldsize - newsize) | CHUNK_MAP_LARGE |
|
|
3755 CHUNK_MAP_ALLOCATED;
|
|
3756 chunk->map[pageind+head_npages].bits = newsize | CHUNK_MAP_LARGE |
|
|
3757 CHUNK_MAP_ALLOCATED;
|
|
3758
|
|
3759 arena_run_dalloc(arena, run, false);
|
|
3760 }
|
|
3761
|
|
3762 static void
|
|
3763 arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
|
|
3764 size_t oldsize, size_t newsize, bool dirty)
|
|
3765 {
|
|
3766 size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
|
|
3767 size_t npages = newsize >> pagesize_2pow;
|
|
3768
|
|
3769 assert(oldsize > newsize);
|
|
3770
|
|
3771 /*
|
|
3772 * Update the chunk map so that arena_run_dalloc() can treat the
|
|
3773 * trailing run as separately allocated.
|
|
3774 */
|
|
3775 chunk->map[pageind].bits = newsize | CHUNK_MAP_LARGE |
|
|
3776 CHUNK_MAP_ALLOCATED;
|
|
3777 chunk->map[pageind+npages].bits = (oldsize - newsize) | CHUNK_MAP_LARGE
|
|
3778 | CHUNK_MAP_ALLOCATED;
|
|
3779
|
|
3780 arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize),
|
|
3781 dirty);
|
|
3782 }
|
|
3783
|
|
3784 static arena_run_t *
|
|
3785 arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
|
|
3786 {
|
|
3787 arena_chunk_map_t *mapelm;
|
|
3788 arena_run_t *run;
|
|
3789 unsigned i, remainder;
|
|
3790
|
|
3791 /* Look for a usable run. */
|
|
3792 mapelm = arena_run_tree_first(&bin->runs);
|
|
3793 if (mapelm != NULL) {
|
|
3794 /* run is guaranteed to have available space. */
|
|
3795 arena_run_tree_remove(&bin->runs, mapelm);
|
|
3796 run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
|
|
3797 #ifdef MALLOC_STATS
|
|
3798 bin->stats.reruns++;
|
|
3799 #endif
|
|
3800 return (run);
|
|
3801 }
|
|
3802 /* No existing runs have any space available. */
|
|
3803
|
|
3804 /* Allocate a new run. */
|
|
3805 run = arena_run_alloc(arena, bin, bin->run_size, false, false);
|
|
3806 if (run == NULL)
|
|
3807 return (NULL);
|
|
3808 /*
|
|
3809 * Don't initialize if a race in arena_run_alloc() allowed an existing
|
|
3810 * run to become usable.
|
|
3811 */
|
|
3812 if (run == bin->runcur)
|
|
3813 return (run);
|
|
3814
|
|
3815 VALGRIND_MALLOCLIKE_BLOCK(run, sizeof(arena_run_t) + (sizeof(unsigned) *
|
|
3816 (bin->regs_mask_nelms - 1)), 0, false);
|
|
3817
|
|
3818 /* Initialize run internals. */
|
|
3819 run->bin = bin;
|
|
3820
|
|
3821 for (i = 0; i < bin->regs_mask_nelms - 1; i++)
|
|
3822 run->regs_mask[i] = UINT_MAX;
|
|
3823 remainder = bin->nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1);
|
|
3824 if (remainder == 0)
|
|
3825 run->regs_mask[i] = UINT_MAX;
|
|
3826 else {
|
|
3827 /* The last element has spare bits that need to be unset. */
|
|
3828 run->regs_mask[i] = (UINT_MAX >> ((1U << (SIZEOF_INT_2POW + 3))
|
|
3829 - remainder));
|
|
3830 }
|
|
3831
|
|
3832 run->regs_minelm = 0;
|
|
3833
|
|
3834 run->nfree = bin->nregs;
|
|
3835 #ifdef MALLOC_DEBUG
|
|
3836 run->magic = ARENA_RUN_MAGIC;
|
|
3837 #endif
|
|
3838
|
|
3839 #ifdef MALLOC_STATS
|
|
3840 bin->stats.nruns++;
|
|
3841 bin->stats.curruns++;
|
|
3842 if (bin->stats.curruns > bin->stats.highruns)
|
|
3843 bin->stats.highruns = bin->stats.curruns;
|
|
3844 #endif
|
|
3845 return (run);
|
|
3846 }
|
|
3847
|
|
3848 /* bin->runcur must have space available before this function is called. */
|
|
3849 static inline void *
|
|
3850 arena_bin_malloc_easy(arena_t *arena, arena_bin_t *bin, arena_run_t *run)
|
|
3851 {
|
|
3852 void *ret;
|
|
3853
|
|
3854 assert(run->magic == ARENA_RUN_MAGIC);
|
|
3855 assert(run->nfree > 0);
|
|
3856
|
|
3857 ret = arena_run_reg_alloc(run, bin);
|
|
3858 assert(ret != NULL);
|
|
3859 run->nfree--;
|
|
3860
|
|
3861 return (ret);
|
|
3862 }
|
|
3863
|
|
3864 /* Re-fill bin->runcur, then call arena_bin_malloc_easy(). */
|
|
3865 static void *
|
|
3866 arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
|
|
3867 {
|
|
3868
|
|
3869 bin->runcur = arena_bin_nonfull_run_get(arena, bin);
|
|
3870 if (bin->runcur == NULL)
|
|
3871 return (NULL);
|
|
3872 assert(bin->runcur->magic == ARENA_RUN_MAGIC);
|
|
3873 assert(bin->runcur->nfree > 0);
|
|
3874
|
|
3875 return (arena_bin_malloc_easy(arena, bin, bin->runcur));
|
|
3876 }
|
|
3877
|
|
3878 /*
|
|
3879 * Calculate bin->run_size such that it meets the following constraints:
|
|
3880 *
|
|
3881 * *) bin->run_size >= min_run_size
|
|
3882 * *) bin->run_size <= arena_maxclass
|
|
3883 * *) bin->run_size <= RUN_MAX_SMALL
|
|
3884 * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed).
|
|
3885 *
|
|
3886 * bin->nregs, bin->regs_mask_nelms, and bin->reg0_offset are
|
|
3887 * also calculated here, since these settings are all interdependent.
|
|
3888 */
|
|
3889 static size_t
|
|
3890 arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size)
|
|
3891 {
|
|
3892 size_t try_run_size, good_run_size;
|
|
3893 unsigned good_nregs, good_mask_nelms, good_reg0_offset;
|
|
3894 unsigned try_nregs, try_mask_nelms, try_reg0_offset;
|
|
3895
|
|
3896 assert(min_run_size >= pagesize);
|
|
3897 assert(min_run_size <= arena_maxclass);
|
|
3898 assert(min_run_size <= RUN_MAX_SMALL);
|
|
3899
|
|
3900 /*
|
|
3901 * Calculate known-valid settings before entering the run_size
|
|
3902 * expansion loop, so that the first part of the loop always copies
|
|
3903 * valid settings.
|
|
3904 *
|
|
3905 * The do..while loop iteratively reduces the number of regions until
|
|
3906 * the run header and the regions no longer overlap. A closed formula
|
|
3907 * would be quite messy, since there is an interdependency between the
|
|
3908 * header's mask length and the number of regions.
|
|
3909 */
|
|
3910 try_run_size = min_run_size;
|
|
3911 try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size)
|
|
3912 + 1; /* Counter-act try_nregs-- in loop. */
|
|
3913 do {
|
|
3914 try_nregs--;
|
|
3915 try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
|
|
3916 ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ? 1 : 0);
|
|
3917 try_reg0_offset = try_run_size - (try_nregs * bin->reg_size);
|
|
3918 } while (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1))
|
|
3919 > try_reg0_offset);
|
|
3920
|
|
3921 /* run_size expansion loop. */
|
|
3922 do {
|
|
3923 /*
|
|
3924 * Copy valid settings before trying more aggressive settings.
|
|
3925 */
|
|
3926 good_run_size = try_run_size;
|
|
3927 good_nregs = try_nregs;
|
|
3928 good_mask_nelms = try_mask_nelms;
|
|
3929 good_reg0_offset = try_reg0_offset;
|
|
3930
|
|
3931 /* Try more aggressive settings. */
|
|
3932 try_run_size += pagesize;
|
|
3933 try_nregs = ((try_run_size - sizeof(arena_run_t)) /
|
|
3934 bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */
|
|
3935 do {
|
|
3936 try_nregs--;
|
|
3937 try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
|
|
3938 ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ?
|
|
3939 1 : 0);
|
|
3940 try_reg0_offset = try_run_size - (try_nregs *
|
|
3941 bin->reg_size);
|
|
3942 } while (sizeof(arena_run_t) + (sizeof(unsigned) *
|
|
3943 (try_mask_nelms - 1)) > try_reg0_offset);
|
|
3944 } while (try_run_size <= arena_maxclass && try_run_size <= RUN_MAX_SMALL
|
|
3945 && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX
|
|
3946 && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size);
|
|
3947
|
|
3948 assert(sizeof(arena_run_t) + (sizeof(unsigned) * (good_mask_nelms - 1))
|
|
3949 <= good_reg0_offset);
|
|
3950 assert((good_mask_nelms << (SIZEOF_INT_2POW + 3)) >= good_nregs);
|
|
3951
|
|
3952 /* Copy final settings. */
|
|
3953 bin->run_size = good_run_size;
|
|
3954 bin->nregs = good_nregs;
|
|
3955 bin->regs_mask_nelms = good_mask_nelms;
|
|
3956 bin->reg0_offset = good_reg0_offset;
|
|
3957
|
|
3958 return (good_run_size);
|
|
3959 }
|
|
3960
|
|
3961 #ifdef MALLOC_BALANCE
|
|
3962 static inline void
|
|
3963 arena_lock_balance(arena_t *arena)
|
|
3964 {
|
|
3965 unsigned contention;
|
|
3966
|
|
3967 contention = malloc_spin_lock(&arena->lock);
|
|
3968 if (narenas > 1) {
|
|
3969 /*
|
|
3970 * Calculate the exponentially averaged contention for this
|
|
3971 * arena. Due to integer math always rounding down, this value
|
|
3972 * decays somewhat faster then normal.
|
|
3973 */
|
|
3974 arena->contention = (((uint64_t)arena->contention
|
|
3975 * (uint64_t)((1U << BALANCE_ALPHA_INV_2POW)-1))
|
|
3976 + (uint64_t)contention) >> BALANCE_ALPHA_INV_2POW;
|
|
3977 if (arena->contention >= opt_balance_threshold)
|
|
3978 arena_lock_balance_hard(arena);
|
|
3979 }
|
|
3980 }
|
|
3981
|
|
3982 static void
|
|
3983 arena_lock_balance_hard(arena_t *arena)
|
|
3984 {
|
|
3985 uint32_t ind;
|
|
3986
|
|
3987 arena->contention = 0;
|
|
3988 #ifdef MALLOC_STATS
|
|
3989 arena->stats.nbalance++;
|
|
3990 #endif
|
|
3991 ind = PRN(balance, narenas_2pow);
|
|
3992 if (arenas[ind] != NULL) {
|
|
3993 #ifdef MOZ_MEMORY_WINDOWS
|
|
3994 TlsSetValue(tlsIndex, arenas[ind]);
|
|
3995 #else
|
|
3996 arenas_map = arenas[ind];
|
|
3997 #endif
|
|
3998 } else {
|
|
3999 malloc_spin_lock(&arenas_lock);
|
|
4000 if (arenas[ind] != NULL) {
|
|
4001 #ifdef MOZ_MEMORY_WINDOWS
|
|
4002 TlsSetValue(tlsIndex, arenas[ind]);
|
|
4003 #else
|
|
4004 arenas_map = arenas[ind];
|
|
4005 #endif
|
|
4006 } else {
|
|
4007 #ifdef MOZ_MEMORY_WINDOWS
|
|
4008 TlsSetValue(tlsIndex, arenas_extend(ind));
|
|
4009 #else
|
|
4010 arenas_map = arenas_extend(ind);
|
|
4011 #endif
|
|
4012 }
|
|
4013 malloc_spin_unlock(&arenas_lock);
|
|
4014 }
|
|
4015 }
|
|
4016 #endif
|
|
4017
|
|
4018 static inline void *
|
|
4019 arena_malloc_small(arena_t *arena, size_t size, bool zero)
|
|
4020 {
|
|
4021 void *ret;
|
|
4022 arena_bin_t *bin;
|
|
4023 arena_run_t *run;
|
|
4024
|
|
4025 if (size < small_min) {
|
|
4026 /* Tiny. */
|
|
4027 size = pow2_ceil(size);
|
|
4028 bin = &arena->bins[ffs((int)(size >> (TINY_MIN_2POW +
|
|
4029 1)))];
|
|
4030 #if (!defined(NDEBUG) || defined(MALLOC_STATS))
|
|
4031 /*
|
|
4032 * Bin calculation is always correct, but we may need
|
|
4033 * to fix size for the purposes of assertions and/or
|
|
4034 * stats accuracy.
|
|
4035 */
|
|
4036 if (size < (1U << TINY_MIN_2POW))
|
|
4037 size = (1U << TINY_MIN_2POW);
|
|
4038 #endif
|
|
4039 } else if (size <= small_max) {
|
|
4040 /* Quantum-spaced. */
|
|
4041 size = QUANTUM_CEILING(size);
|
|
4042 bin = &arena->bins[ntbins + (size >> opt_quantum_2pow)
|
|
4043 - 1];
|
|
4044 } else {
|
|
4045 /* Sub-page. */
|
|
4046 size = pow2_ceil(size);
|
|
4047 bin = &arena->bins[ntbins + nqbins
|
|
4048 + (ffs((int)(size >> opt_small_max_2pow)) - 2)];
|
|
4049 }
|
|
4050 assert(size == bin->reg_size);
|
|
4051
|
|
4052 #ifdef MALLOC_BALANCE
|
|
4053 arena_lock_balance(arena);
|
|
4054 #else
|
|
4055 malloc_spin_lock(&arena->lock);
|
|
4056 #endif
|
|
4057 if ((run = bin->runcur) != NULL && run->nfree > 0)
|
|
4058 ret = arena_bin_malloc_easy(arena, bin, run);
|
|
4059 else
|
|
4060 ret = arena_bin_malloc_hard(arena, bin);
|
|
4061
|
|
4062 if (ret == NULL) {
|
|
4063 malloc_spin_unlock(&arena->lock);
|
|
4064 return (NULL);
|
|
4065 }
|
|
4066
|
|
4067 #ifdef MALLOC_STATS
|
|
4068 bin->stats.nrequests++;
|
|
4069 arena->stats.nmalloc_small++;
|
|
4070 arena->stats.allocated_small += size;
|
|
4071 #endif
|
|
4072 malloc_spin_unlock(&arena->lock);
|
|
4073
|
|
4074 VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
|
|
4075 if (zero == false) {
|
|
4076 #ifdef MALLOC_FILL
|
|
4077 if (opt_junk)
|
|
4078 memset(ret, 0xa5, size);
|
|
4079 else if (opt_zero)
|
|
4080 memset(ret, 0, size);
|
|
4081 #endif
|
|
4082 } else
|
|
4083 memset(ret, 0, size);
|
|
4084
|
|
4085 return (ret);
|
|
4086 }
|
|
4087
|
|
4088 static void *
|
|
4089 arena_malloc_large(arena_t *arena, size_t size, bool zero)
|
|
4090 {
|
|
4091 void *ret;
|
|
4092
|
|
4093 /* Large allocation. */
|
|
4094 size = PAGE_CEILING(size);
|
|
4095 #ifdef MALLOC_BALANCE
|
|
4096 arena_lock_balance(arena);
|
|
4097 #else
|
|
4098 malloc_spin_lock(&arena->lock);
|
|
4099 #endif
|
|
4100 ret = (void *)arena_run_alloc(arena, NULL, size, true, zero);
|
|
4101 if (ret == NULL) {
|
|
4102 malloc_spin_unlock(&arena->lock);
|
|
4103 return (NULL);
|
|
4104 }
|
|
4105 #ifdef MALLOC_STATS
|
|
4106 arena->stats.nmalloc_large++;
|
|
4107 arena->stats.allocated_large += size;
|
|
4108 #endif
|
|
4109 malloc_spin_unlock(&arena->lock);
|
|
4110
|
|
4111 VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
|
|
4112 if (zero == false) {
|
|
4113 #ifdef MALLOC_FILL
|
|
4114 if (opt_junk)
|
|
4115 memset(ret, 0xa5, size);
|
|
4116 else if (opt_zero)
|
|
4117 memset(ret, 0, size);
|
|
4118 #endif
|
|
4119 }
|
|
4120
|
|
4121 return (ret);
|
|
4122 }
|
|
4123
|
|
4124 static inline void *
|
|
4125 arena_malloc(arena_t *arena, size_t size, bool zero)
|
|
4126 {
|
|
4127
|
|
4128 assert(arena != NULL);
|
|
4129 assert(arena->magic == ARENA_MAGIC);
|
|
4130 assert(size != 0);
|
|
4131 assert(QUANTUM_CEILING(size) <= arena_maxclass);
|
|
4132
|
|
4133 if (size <= bin_maxclass) {
|
|
4134 return (arena_malloc_small(arena, size, zero));
|
|
4135 } else
|
|
4136 return (arena_malloc_large(arena, size, zero));
|
|
4137 }
|
|
4138
|
|
4139 static inline void *
|
|
4140 imalloc(size_t size)
|
|
4141 {
|
|
4142
|
|
4143 assert(size != 0);
|
|
4144
|
|
4145 if (size <= arena_maxclass)
|
|
4146 return (arena_malloc(choose_arena(), size, false));
|
|
4147 else
|
|
4148 return (huge_malloc(size, false));
|
|
4149 }
|
|
4150
|
|
4151 static inline void *
|
|
4152 icalloc(size_t size)
|
|
4153 {
|
|
4154
|
|
4155 if (size <= arena_maxclass)
|
|
4156 return (arena_malloc(choose_arena(), size, true));
|
|
4157 else
|
|
4158 return (huge_malloc(size, true));
|
|
4159 }
|
|
4160
|
|
4161 /* Only handles large allocations that require more than page alignment. */
|
|
4162 static void *
|
|
4163 arena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size)
|
|
4164 {
|
|
4165 void *ret;
|
|
4166 size_t offset;
|
|
4167 arena_chunk_t *chunk;
|
|
4168
|
|
4169 assert((size & pagesize_mask) == 0);
|
|
4170 assert((alignment & pagesize_mask) == 0);
|
|
4171
|
|
4172 #ifdef MALLOC_BALANCE
|
|
4173 arena_lock_balance(arena);
|
|
4174 #else
|
|
4175 malloc_spin_lock(&arena->lock);
|
|
4176 #endif
|
|
4177 ret = (void *)arena_run_alloc(arena, NULL, alloc_size, true, false);
|
|
4178 if (ret == NULL) {
|
|
4179 malloc_spin_unlock(&arena->lock);
|
|
4180 return (NULL);
|
|
4181 }
|
|
4182
|
|
4183 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
|
|
4184
|
|
4185 offset = (uintptr_t)ret & (alignment - 1);
|
|
4186 assert((offset & pagesize_mask) == 0);
|
|
4187 assert(offset < alloc_size);
|
|
4188 if (offset == 0)
|
|
4189 arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, alloc_size, size, false);
|
|
4190 else {
|
|
4191 size_t leadsize, trailsize;
|
|
4192
|
|
4193 leadsize = alignment - offset;
|
|
4194 if (leadsize > 0) {
|
|
4195 arena_run_trim_head(arena, chunk, (arena_run_t*)ret, alloc_size,
|
|
4196 alloc_size - leadsize);
|
|
4197 ret = (void *)((uintptr_t)ret + leadsize);
|
|
4198 }
|
|
4199
|
|
4200 trailsize = alloc_size - leadsize - size;
|
|
4201 if (trailsize != 0) {
|
|
4202 /* Trim trailing space. */
|
|
4203 assert(trailsize < alloc_size);
|
|
4204 arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, size + trailsize,
|
|
4205 size, false);
|
|
4206 }
|
|
4207 }
|
|
4208
|
|
4209 #ifdef MALLOC_STATS
|
|
4210 arena->stats.nmalloc_large++;
|
|
4211 arena->stats.allocated_large += size;
|
|
4212 #endif
|
|
4213 malloc_spin_unlock(&arena->lock);
|
|
4214
|
|
4215 VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
|
|
4216 #ifdef MALLOC_FILL
|
|
4217 if (opt_junk)
|
|
4218 memset(ret, 0xa5, size);
|
|
4219 else if (opt_zero)
|
|
4220 memset(ret, 0, size);
|
|
4221 #endif
|
|
4222 return (ret);
|
|
4223 }
|
|
4224
|
|
4225 static inline void *
|
|
4226 ipalloc(size_t alignment, size_t size)
|
|
4227 {
|
|
4228 void *ret;
|
|
4229 size_t ceil_size;
|
|
4230
|
|
4231 /*
|
|
4232 * Round size up to the nearest multiple of alignment.
|
|
4233 *
|
|
4234 * This done, we can take advantage of the fact that for each small
|
|
4235 * size class, every object is aligned at the smallest power of two
|
|
4236 * that is non-zero in the base two representation of the size. For
|
|
4237 * example:
|
|
4238 *
|
|
4239 * Size | Base 2 | Minimum alignment
|
|
4240 * -----+----------+------------------
|
|
4241 * 96 | 1100000 | 32
|
|
4242 * 144 | 10100000 | 32
|
|
4243 * 192 | 11000000 | 64
|
|
4244 *
|
|
4245 * Depending on runtime settings, it is possible that arena_malloc()
|
|
4246 * will further round up to a power of two, but that never causes
|
|
4247 * correctness issues.
|
|
4248 */
|
|
4249 ceil_size = (size + (alignment - 1)) & (-alignment);
|
|
4250 /*
|
|
4251 * (ceil_size < size) protects against the combination of maximal
|
|
4252 * alignment and size greater than maximal alignment.
|
|
4253 */
|
|
4254 if (ceil_size < size) {
|
|
4255 /* size_t overflow. */
|
|
4256 return (NULL);
|
|
4257 }
|
|
4258
|
|
4259 if (ceil_size <= pagesize || (alignment <= pagesize
|
|
4260 && ceil_size <= arena_maxclass))
|
|
4261 ret = arena_malloc(choose_arena(), ceil_size, false);
|
|
4262 else {
|
|
4263 size_t run_size;
|
|
4264
|
|
4265 /*
|
|
4266 * We can't achieve sub-page alignment, so round up alignment
|
|
4267 * permanently; it makes later calculations simpler.
|
|
4268 */
|
|
4269 alignment = PAGE_CEILING(alignment);
|
|
4270 ceil_size = PAGE_CEILING(size);
|
|
4271 /*
|
|
4272 * (ceil_size < size) protects against very large sizes within
|
|
4273 * pagesize of SIZE_T_MAX.
|
|
4274 *
|
|
4275 * (ceil_size + alignment < ceil_size) protects against the
|
|
4276 * combination of maximal alignment and ceil_size large enough
|
|
4277 * to cause overflow. This is similar to the first overflow
|
|
4278 * check above, but it needs to be repeated due to the new
|
|
4279 * ceil_size value, which may now be *equal* to maximal
|
|
4280 * alignment, whereas before we only detected overflow if the
|
|
4281 * original size was *greater* than maximal alignment.
|
|
4282 */
|
|
4283 if (ceil_size < size || ceil_size + alignment < ceil_size) {
|
|
4284 /* size_t overflow. */
|
|
4285 return (NULL);
|
|
4286 }
|
|
4287
|
|
4288 /*
|
|
4289 * Calculate the size of the over-size run that arena_palloc()
|
|
4290 * would need to allocate in order to guarantee the alignment.
|
|
4291 */
|
|
4292 if (ceil_size >= alignment)
|
|
4293 run_size = ceil_size + alignment - pagesize;
|
|
4294 else {
|
|
4295 /*
|
|
4296 * It is possible that (alignment << 1) will cause
|
|
4297 * overflow, but it doesn't matter because we also
|
|
4298 * subtract pagesize, which in the case of overflow
|
|
4299 * leaves us with a very large run_size. That causes
|
|
4300 * the first conditional below to fail, which means
|
|
4301 * that the bogus run_size value never gets used for
|
|
4302 * anything important.
|
|
4303 */
|
|
4304 run_size = (alignment << 1) - pagesize;
|
|
4305 }
|
|
4306
|
|
4307 if (run_size <= arena_maxclass) {
|
|
4308 ret = arena_palloc(choose_arena(), alignment, ceil_size,
|
|
4309 run_size);
|
|
4310 } else if (alignment <= chunksize)
|
|
4311 ret = huge_malloc(ceil_size, false);
|
|
4312 else
|
|
4313 ret = huge_palloc(alignment, ceil_size);
|
|
4314 }
|
|
4315
|
|
4316 assert(((uintptr_t)ret & (alignment - 1)) == 0);
|
|
4317 return (ret);
|
|
4318 }
|
|
4319
|
|
4320 /* Return the size of the allocation pointed to by ptr. */
|
|
4321 static size_t
|
|
4322 arena_salloc(const void *ptr)
|
|
4323 {
|
|
4324 size_t ret;
|
|
4325 arena_chunk_t *chunk;
|
|
4326 size_t pageind, mapbits;
|
|
4327
|
|
4328 assert(ptr != NULL);
|
|
4329 assert(CHUNK_ADDR2BASE(ptr) != ptr);
|
|
4330
|
|
4331 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
4332 pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow);
|
|
4333 mapbits = chunk->map[pageind].bits;
|
|
4334 assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
|
|
4335 if ((mapbits & CHUNK_MAP_LARGE) == 0) {
|
|
4336 arena_run_t *run = (arena_run_t *)(mapbits & ~pagesize_mask);
|
|
4337 assert(run->magic == ARENA_RUN_MAGIC);
|
|
4338 ret = run->bin->reg_size;
|
|
4339 } else {
|
|
4340 ret = mapbits & ~pagesize_mask;
|
|
4341 assert(ret != 0);
|
|
4342 }
|
|
4343
|
|
4344 return (ret);
|
|
4345 }
|
|
4346
|
|
4347 #if (defined(MALLOC_VALIDATE) || defined(MOZ_MEMORY_DARWIN))
|
|
4348 /*
|
|
4349 * Validate ptr before assuming that it points to an allocation. Currently,
|
|
4350 * the following validation is performed:
|
|
4351 *
|
|
4352 * + Check that ptr is not NULL.
|
|
4353 *
|
|
4354 * + Check that ptr lies within a mapped chunk.
|
|
4355 */
|
|
4356 static inline size_t
|
|
4357 isalloc_validate(const void *ptr)
|
|
4358 {
|
|
4359 arena_chunk_t *chunk;
|
|
4360
|
|
4361 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
4362 if (chunk == NULL)
|
|
4363 return (0);
|
|
4364
|
|
4365 if (malloc_rtree_get(chunk_rtree, (uintptr_t)chunk) == NULL)
|
|
4366 return (0);
|
|
4367
|
|
4368 if (chunk != ptr) {
|
|
4369 assert(chunk->arena->magic == ARENA_MAGIC);
|
|
4370 return (arena_salloc(ptr));
|
|
4371 } else {
|
|
4372 size_t ret;
|
|
4373 extent_node_t *node;
|
|
4374 extent_node_t key;
|
|
4375
|
|
4376 /* Chunk. */
|
|
4377 key.addr = (void *)chunk;
|
|
4378 malloc_mutex_lock(&huge_mtx);
|
|
4379 node = extent_tree_ad_search(&huge, &key);
|
|
4380 if (node != NULL)
|
|
4381 ret = node->size;
|
|
4382 else
|
|
4383 ret = 0;
|
|
4384 malloc_mutex_unlock(&huge_mtx);
|
|
4385 return (ret);
|
|
4386 }
|
|
4387 }
|
|
4388 #endif
|
|
4389
|
|
4390 static inline size_t
|
|
4391 isalloc(const void *ptr)
|
|
4392 {
|
|
4393 size_t ret;
|
|
4394 arena_chunk_t *chunk;
|
|
4395
|
|
4396 assert(ptr != NULL);
|
|
4397
|
|
4398 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
4399 if (chunk != ptr) {
|
|
4400 /* Region. */
|
|
4401 assert(chunk->arena->magic == ARENA_MAGIC);
|
|
4402
|
|
4403 ret = arena_salloc(ptr);
|
|
4404 } else {
|
|
4405 extent_node_t *node, key;
|
|
4406
|
|
4407 /* Chunk (huge allocation). */
|
|
4408
|
|
4409 malloc_mutex_lock(&huge_mtx);
|
|
4410
|
|
4411 /* Extract from tree of huge allocations. */
|
|
4412 key.addr = __DECONST(void *, ptr);
|
|
4413 node = extent_tree_ad_search(&huge, &key);
|
|
4414 assert(node != NULL);
|
|
4415
|
|
4416 ret = node->size;
|
|
4417
|
|
4418 malloc_mutex_unlock(&huge_mtx);
|
|
4419 }
|
|
4420
|
|
4421 return (ret);
|
|
4422 }
|
|
4423
|
|
4424 static inline void
|
|
4425 arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
|
4426 arena_chunk_map_t *mapelm)
|
|
4427 {
|
|
4428 arena_run_t *run;
|
|
4429 arena_bin_t *bin;
|
|
4430 size_t size;
|
|
4431
|
|
4432 run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
|
|
4433 assert(run->magic == ARENA_RUN_MAGIC);
|
|
4434 bin = run->bin;
|
|
4435 size = bin->reg_size;
|
|
4436
|
|
4437 #ifdef MALLOC_FILL
|
|
4438 if (opt_junk)
|
|
4439 memset(ptr, 0x5a, size);
|
|
4440 #endif
|
|
4441
|
|
4442 arena_run_reg_dalloc(run, bin, ptr, size);
|
|
4443 run->nfree++;
|
|
4444
|
|
4445 if (run->nfree == bin->nregs) {
|
|
4446 /* Deallocate run. */
|
|
4447 if (run == bin->runcur)
|
|
4448 bin->runcur = NULL;
|
|
4449 else if (bin->nregs != 1) {
|
|
4450 size_t run_pageind = (((uintptr_t)run -
|
|
4451 (uintptr_t)chunk)) >> pagesize_2pow;
|
|
4452 arena_chunk_map_t *run_mapelm =
|
|
4453 &chunk->map[run_pageind];
|
|
4454 /*
|
|
4455 * This block's conditional is necessary because if the
|
|
4456 * run only contains one region, then it never gets
|
|
4457 * inserted into the non-full runs tree.
|
|
4458 */
|
|
4459 assert(arena_run_tree_search(&bin->runs, run_mapelm) ==
|
|
4460 run_mapelm);
|
|
4461 arena_run_tree_remove(&bin->runs, run_mapelm);
|
|
4462 }
|
|
4463 #ifdef MALLOC_DEBUG
|
|
4464 run->magic = 0;
|
|
4465 #endif
|
|
4466 VALGRIND_FREELIKE_BLOCK(run, 0);
|
|
4467 arena_run_dalloc(arena, run, true);
|
|
4468 #ifdef MALLOC_STATS
|
|
4469 bin->stats.curruns--;
|
|
4470 #endif
|
|
4471 } else if (run->nfree == 1 && run != bin->runcur) {
|
|
4472 /*
|
|
4473 * Make sure that bin->runcur always refers to the lowest
|
|
4474 * non-full run, if one exists.
|
|
4475 */
|
|
4476 if (bin->runcur == NULL)
|
|
4477 bin->runcur = run;
|
|
4478 else if ((uintptr_t)run < (uintptr_t)bin->runcur) {
|
|
4479 /* Switch runcur. */
|
|
4480 if (bin->runcur->nfree > 0) {
|
|
4481 arena_chunk_t *runcur_chunk =
|
|
4482 (arena_chunk_t*)CHUNK_ADDR2BASE(bin->runcur);
|
|
4483 size_t runcur_pageind =
|
|
4484 (((uintptr_t)bin->runcur -
|
|
4485 (uintptr_t)runcur_chunk)) >> pagesize_2pow;
|
|
4486 arena_chunk_map_t *runcur_mapelm =
|
|
4487 &runcur_chunk->map[runcur_pageind];
|
|
4488
|
|
4489 /* Insert runcur. */
|
|
4490 assert(arena_run_tree_search(&bin->runs,
|
|
4491 runcur_mapelm) == NULL);
|
|
4492 arena_run_tree_insert(&bin->runs,
|
|
4493 runcur_mapelm);
|
|
4494 }
|
|
4495 bin->runcur = run;
|
|
4496 } else {
|
|
4497 size_t run_pageind = (((uintptr_t)run -
|
|
4498 (uintptr_t)chunk)) >> pagesize_2pow;
|
|
4499 arena_chunk_map_t *run_mapelm =
|
|
4500 &chunk->map[run_pageind];
|
|
4501
|
|
4502 assert(arena_run_tree_search(&bin->runs, run_mapelm) ==
|
|
4503 NULL);
|
|
4504 arena_run_tree_insert(&bin->runs, run_mapelm);
|
|
4505 }
|
|
4506 }
|
|
4507 #ifdef MALLOC_STATS
|
|
4508 arena->stats.allocated_small -= size;
|
|
4509 arena->stats.ndalloc_small++;
|
|
4510 #endif
|
|
4511 }
|
|
4512
|
|
4513 static void
|
|
4514 arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
|
|
4515 {
|
|
4516 /* Large allocation. */
|
|
4517 malloc_spin_lock(&arena->lock);
|
|
4518
|
|
4519 #ifdef MALLOC_FILL
|
|
4520 #ifndef MALLOC_STATS
|
|
4521 if (opt_junk)
|
|
4522 #endif
|
|
4523 #endif
|
|
4524 {
|
|
4525 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
|
|
4526 pagesize_2pow;
|
|
4527 size_t size = chunk->map[pageind].bits & ~pagesize_mask;
|
|
4528
|
|
4529 #ifdef MALLOC_FILL
|
|
4530 #ifdef MALLOC_STATS
|
|
4531 if (opt_junk)
|
|
4532 #endif
|
|
4533 memset(ptr, 0x5a, size);
|
|
4534 #endif
|
|
4535 #ifdef MALLOC_STATS
|
|
4536 arena->stats.allocated_large -= size;
|
|
4537 #endif
|
|
4538 }
|
|
4539 #ifdef MALLOC_STATS
|
|
4540 arena->stats.ndalloc_large++;
|
|
4541 #endif
|
|
4542
|
|
4543 arena_run_dalloc(arena, (arena_run_t *)ptr, true);
|
|
4544 malloc_spin_unlock(&arena->lock);
|
|
4545 }
|
|
4546
|
|
4547 static inline void
|
|
4548 arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr)
|
|
4549 {
|
|
4550 size_t pageind;
|
|
4551 arena_chunk_map_t *mapelm;
|
|
4552
|
|
4553 assert(arena != NULL);
|
|
4554 assert(arena->magic == ARENA_MAGIC);
|
|
4555 assert(chunk->arena == arena);
|
|
4556 assert(ptr != NULL);
|
|
4557 assert(CHUNK_ADDR2BASE(ptr) != ptr);
|
|
4558
|
|
4559 pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow);
|
|
4560 mapelm = &chunk->map[pageind];
|
|
4561 assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0);
|
|
4562 if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) {
|
|
4563 /* Small allocation. */
|
|
4564 malloc_spin_lock(&arena->lock);
|
|
4565 arena_dalloc_small(arena, chunk, ptr, mapelm);
|
|
4566 malloc_spin_unlock(&arena->lock);
|
|
4567 } else
|
|
4568 arena_dalloc_large(arena, chunk, ptr);
|
|
4569 VALGRIND_FREELIKE_BLOCK(ptr, 0);
|
|
4570 }
|
|
4571
|
|
4572 static inline void
|
|
4573 idalloc(void *ptr)
|
|
4574 {
|
|
4575 arena_chunk_t *chunk;
|
|
4576
|
|
4577 assert(ptr != NULL);
|
|
4578
|
|
4579 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
4580 if (chunk != ptr)
|
|
4581 arena_dalloc(chunk->arena, chunk, ptr);
|
|
4582 else
|
|
4583 huge_dalloc(ptr);
|
|
4584 }
|
|
4585
|
|
4586 static void
|
|
4587 arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
|
4588 size_t size, size_t oldsize)
|
|
4589 {
|
|
4590
|
|
4591 assert(size < oldsize);
|
|
4592
|
|
4593 /*
|
|
4594 * Shrink the run, and make trailing pages available for other
|
|
4595 * allocations.
|
|
4596 */
|
|
4597 #ifdef MALLOC_BALANCE
|
|
4598 arena_lock_balance(arena);
|
|
4599 #else
|
|
4600 malloc_spin_lock(&arena->lock);
|
|
4601 #endif
|
|
4602 arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size,
|
|
4603 true);
|
|
4604 #ifdef MALLOC_STATS
|
|
4605 arena->stats.allocated_large -= oldsize - size;
|
|
4606 #endif
|
|
4607 malloc_spin_unlock(&arena->lock);
|
|
4608 }
|
|
4609
|
|
4610 static bool
|
|
4611 arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
|
4612 size_t size, size_t oldsize)
|
|
4613 {
|
|
4614 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow;
|
|
4615 size_t npages = oldsize >> pagesize_2pow;
|
|
4616
|
|
4617 assert(oldsize == (chunk->map[pageind].bits & ~pagesize_mask));
|
|
4618
|
|
4619 /* Try to extend the run. */
|
|
4620 assert(size > oldsize);
|
|
4621 #ifdef MALLOC_BALANCE
|
|
4622 arena_lock_balance(arena);
|
|
4623 #else
|
|
4624 malloc_spin_lock(&arena->lock);
|
|
4625 #endif
|
|
4626 if (pageind + npages < chunk_npages && (chunk->map[pageind+npages].bits
|
|
4627 & CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[pageind+npages].bits &
|
|
4628 ~pagesize_mask) >= size - oldsize) {
|
|
4629 /*
|
|
4630 * The next run is available and sufficiently large. Split the
|
|
4631 * following run, then merge the first part with the existing
|
|
4632 * allocation.
|
|
4633 */
|
|
4634 arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk +
|
|
4635 ((pageind+npages) << pagesize_2pow)), size - oldsize, true,
|
|
4636 false);
|
|
4637
|
|
4638 chunk->map[pageind].bits = size | CHUNK_MAP_LARGE |
|
|
4639 CHUNK_MAP_ALLOCATED;
|
|
4640 chunk->map[pageind+npages].bits = CHUNK_MAP_LARGE |
|
|
4641 CHUNK_MAP_ALLOCATED;
|
|
4642
|
|
4643 #ifdef MALLOC_STATS
|
|
4644 arena->stats.allocated_large += size - oldsize;
|
|
4645 #endif
|
|
4646 malloc_spin_unlock(&arena->lock);
|
|
4647 return (false);
|
|
4648 }
|
|
4649 malloc_spin_unlock(&arena->lock);
|
|
4650
|
|
4651 return (true);
|
|
4652 }
|
|
4653
|
|
4654 /*
|
|
4655 * Try to resize a large allocation, in order to avoid copying. This will
|
|
4656 * always fail if growing an object, and the following run is already in use.
|
|
4657 */
|
|
4658 static bool
|
|
4659 arena_ralloc_large(void *ptr, size_t size, size_t oldsize)
|
|
4660 {
|
|
4661 size_t psize;
|
|
4662
|
|
4663 psize = PAGE_CEILING(size);
|
|
4664 if (psize == oldsize) {
|
|
4665 /* Same size class. */
|
|
4666 #ifdef MALLOC_FILL
|
|
4667 if (opt_junk && size < oldsize) {
|
|
4668 memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize -
|
|
4669 size);
|
|
4670 }
|
|
4671 #endif
|
|
4672 return (false);
|
|
4673 } else {
|
|
4674 arena_chunk_t *chunk;
|
|
4675 arena_t *arena;
|
|
4676
|
|
4677 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
|
4678 arena = chunk->arena;
|
|
4679 assert(arena->magic == ARENA_MAGIC);
|
|
4680
|
|
4681 if (psize < oldsize) {
|
|
4682 #ifdef MALLOC_FILL
|
|
4683 /* Fill before shrinking in order avoid a race. */
|
|
4684 if (opt_junk) {
|
|
4685 memset((void *)((uintptr_t)ptr + size), 0x5a,
|
|
4686 oldsize - size);
|
|
4687 }
|
|
4688 #endif
|
|
4689 arena_ralloc_large_shrink(arena, chunk, ptr, psize,
|
|
4690 oldsize);
|
|
4691 return (false);
|
|
4692 } else {
|
|
4693 bool ret = arena_ralloc_large_grow(arena, chunk, ptr,
|
|
4694 psize, oldsize);
|
|
4695 #ifdef MALLOC_FILL
|
|
4696 if (ret == false && opt_zero) {
|
|
4697 memset((void *)((uintptr_t)ptr + oldsize), 0,
|
|
4698 size - oldsize);
|
|
4699 }
|
|
4700 #endif
|
|
4701 return (ret);
|
|
4702 }
|
|
4703 }
|
|
4704 }
|
|
4705
|
|
4706 static void *
|
|
4707 arena_ralloc(void *ptr, size_t size, size_t oldsize)
|
|
4708 {
|
|
4709 void *ret;
|
|
4710 size_t copysize;
|
|
4711
|
|
4712 /* Try to avoid moving the allocation. */
|
|
4713 if (size < small_min) {
|
|
4714 if (oldsize < small_min &&
|
|
4715 ffs((int)(pow2_ceil(size) >> (TINY_MIN_2POW + 1)))
|
|
4716 == ffs((int)(pow2_ceil(oldsize) >> (TINY_MIN_2POW + 1))))
|
|
4717 goto IN_PLACE; /* Same size class. */
|
|
4718 } else if (size <= small_max) {
|
|
4719 if (oldsize >= small_min && oldsize <= small_max &&
|
|
4720 (QUANTUM_CEILING(size) >> opt_quantum_2pow)
|
|
4721 == (QUANTUM_CEILING(oldsize) >> opt_quantum_2pow))
|
|
4722 goto IN_PLACE; /* Same size class. */
|
|
4723 } else if (size <= bin_maxclass) {
|
|
4724 if (oldsize > small_max && oldsize <= bin_maxclass &&
|
|
4725 pow2_ceil(size) == pow2_ceil(oldsize))
|
|
4726 goto IN_PLACE; /* Same size class. */
|
|
4727 } else if (oldsize > bin_maxclass && oldsize <= arena_maxclass) {
|
|
4728 assert(size > bin_maxclass);
|
|
4729 if (arena_ralloc_large(ptr, size, oldsize) == false)
|
|
4730 return (ptr);
|
|
4731 }
|
|
4732
|
|
4733 /*
|
|
4734 * If we get here, then size and oldsize are different enough that we
|
|
4735 * need to move the object. In that case, fall back to allocating new
|
|
4736 * space and copying.
|
|
4737 */
|
|
4738 ret = arena_malloc(choose_arena(), size, false);
|
|
4739 if (ret == NULL)
|
|
4740 return (NULL);
|
|
4741
|
|
4742 /* Junk/zero-filling were already done by arena_malloc(). */
|
|
4743 copysize = (size < oldsize) ? size : oldsize;
|
|
4744 #ifdef VM_COPY_MIN
|
|
4745 if (copysize >= VM_COPY_MIN)
|
|
4746 pages_copy(ret, ptr, copysize);
|
|
4747 else
|
|
4748 #endif
|
|
4749 memcpy(ret, ptr, copysize);
|
|
4750 idalloc(ptr);
|
|
4751 return (ret);
|
|
4752 IN_PLACE:
|
|
4753 #ifdef MALLOC_FILL
|
|
4754 if (opt_junk && size < oldsize)
|
|
4755 memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size);
|
|
4756 else if (opt_zero && size > oldsize)
|
|
4757 memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize);
|
|
4758 #endif
|
|
4759 return (ptr);
|
|
4760 }
|
|
4761
|
|
4762 static inline void *
|
|
4763 iralloc(void *ptr, size_t size)
|
|
4764 {
|
|
4765 size_t oldsize;
|
|
4766
|
|
4767 assert(ptr != NULL);
|
|
4768 assert(size != 0);
|
|
4769
|
|
4770 oldsize = isalloc(ptr);
|
|
4771
|
|
4772 #ifndef MALLOC_VALGRIND
|
|
4773 if (size <= arena_maxclass)
|
|
4774 return (arena_ralloc(ptr, size, oldsize));
|
|
4775 else
|
|
4776 return (huge_ralloc(ptr, size, oldsize));
|
|
4777 #else
|
|
4778 /*
|
|
4779 * Valgrind does not provide a public interface for modifying an
|
|
4780 * existing allocation, so use malloc/memcpy/free instead.
|
|
4781 */
|
|
4782 {
|
|
4783 void *ret = imalloc(size);
|
|
4784 if (ret != NULL) {
|
|
4785 if (oldsize < size)
|
|
4786 memcpy(ret, ptr, oldsize);
|
|
4787 else
|
|
4788 memcpy(ret, ptr, size);
|
|
4789 idalloc(ptr);
|
|
4790 }
|
|
4791 return (ret);
|
|
4792 }
|
|
4793 #endif
|
|
4794 }
|
|
4795
|
|
4796 static bool
|
|
4797 arena_new(arena_t *arena)
|
|
4798 {
|
|
4799 unsigned i;
|
|
4800 arena_bin_t *bin;
|
|
4801 size_t pow2_size, prev_run_size;
|
|
4802
|
|
4803 if (malloc_spin_init(&arena->lock))
|
|
4804 return (true);
|
|
4805
|
|
4806 #ifdef MALLOC_STATS
|
|
4807 memset(&arena->stats, 0, sizeof(arena_stats_t));
|
|
4808 #endif
|
|
4809
|
|
4810 arena->chunk_seq = 0;
|
|
4811
|
|
4812 /* Initialize chunks. */
|
|
4813 arena_chunk_tree_dirty_new(&arena->chunks_dirty);
|
|
4814 arena->spare = NULL;
|
|
4815
|
|
4816 arena->ndirty = 0;
|
|
4817
|
|
4818 arena_avail_tree_new(&arena->runs_avail);
|
|
4819
|
|
4820 #ifdef MALLOC_BALANCE
|
|
4821 arena->contention = 0;
|
|
4822 #endif
|
|
4823
|
|
4824 /* Initialize bins. */
|
|
4825 prev_run_size = pagesize;
|
|
4826
|
|
4827 /* (2^n)-spaced tiny bins. */
|
|
4828 for (i = 0; i < ntbins; i++) {
|
|
4829 bin = &arena->bins[i];
|
|
4830 bin->runcur = NULL;
|
|
4831 arena_run_tree_new(&bin->runs);
|
|
4832
|
|
4833 bin->reg_size = (1U << (TINY_MIN_2POW + i));
|
|
4834
|
|
4835 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
|
|
4836
|
|
4837 #ifdef MALLOC_STATS
|
|
4838 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
|
|
4839 #endif
|
|
4840 }
|
|
4841
|
|
4842 /* Quantum-spaced bins. */
|
|
4843 for (; i < ntbins + nqbins; i++) {
|
|
4844 bin = &arena->bins[i];
|
|
4845 bin->runcur = NULL;
|
|
4846 arena_run_tree_new(&bin->runs);
|
|
4847
|
|
4848 bin->reg_size = quantum * (i - ntbins + 1);
|
|
4849
|
|
4850 pow2_size = pow2_ceil(quantum * (i - ntbins + 1));
|
|
4851 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
|
|
4852
|
|
4853 #ifdef MALLOC_STATS
|
|
4854 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
|
|
4855 #endif
|
|
4856 }
|
|
4857
|
|
4858 /* (2^n)-spaced sub-page bins. */
|
|
4859 for (; i < ntbins + nqbins + nsbins; i++) {
|
|
4860 bin = &arena->bins[i];
|
|
4861 bin->runcur = NULL;
|
|
4862 arena_run_tree_new(&bin->runs);
|
|
4863
|
|
4864 bin->reg_size = (small_max << (i - (ntbins + nqbins) + 1));
|
|
4865
|
|
4866 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
|
|
4867
|
|
4868 #ifdef MALLOC_STATS
|
|
4869 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
|
|
4870 #endif
|
|
4871 }
|
|
4872
|
|
4873 #ifdef MALLOC_DEBUG
|
|
4874 arena->magic = ARENA_MAGIC;
|
|
4875 #endif
|
|
4876
|
|
4877 return (false);
|
|
4878 }
|
|
4879
|
|
4880 /* Create a new arena and insert it into the arenas array at index ind. */
|
|
4881 static arena_t *
|
|
4882 arenas_extend(unsigned ind)
|
|
4883 {
|
|
4884 arena_t *ret;
|
|
4885
|
|
4886 /* Allocate enough space for trailing bins. */
|
|
4887 ret = (arena_t *)base_alloc(sizeof(arena_t)
|
|
4888 + (sizeof(arena_bin_t) * (ntbins + nqbins + nsbins - 1)));
|
|
4889 if (ret != NULL && arena_new(ret) == false) {
|
|
4890 arenas[ind] = ret;
|
|
4891 return (ret);
|
|
4892 }
|
|
4893 /* Only reached if there is an OOM error. */
|
|
4894
|
|
4895 /*
|
|
4896 * OOM here is quite inconvenient to propagate, since dealing with it
|
|
4897 * would require a check for failure in the fast path. Instead, punt
|
|
4898 * by using arenas[0]. In practice, this is an extremely unlikely
|
|
4899 * failure.
|
|
4900 */
|
|
4901 _malloc_message(_getprogname(),
|
|
4902 ": (malloc) Error initializing arena\n", "", "");
|
|
4903 if (opt_abort)
|
|
4904 abort();
|
|
4905
|
|
4906 return (arenas[0]);
|
|
4907 }
|
|
4908
|
|
4909 /*
|
|
4910 * End arena.
|
|
4911 */
|
|
4912 /******************************************************************************/
|
|
4913 /*
|
|
4914 * Begin general internal functions.
|
|
4915 */
|
|
4916
|
|
4917 static void *
|
|
4918 huge_malloc(size_t size, bool zero)
|
|
4919 {
|
|
4920 void *ret;
|
|
4921 size_t csize;
|
|
4922 #ifdef MALLOC_DECOMMIT
|
|
4923 size_t psize;
|
|
4924 #endif
|
|
4925 extent_node_t *node;
|
|
4926
|
|
4927 /* Allocate one or more contiguous chunks for this request. */
|
|
4928
|
|
4929 csize = CHUNK_CEILING(size);
|
|
4930 if (csize == 0) {
|
|
4931 /* size is large enough to cause size_t wrap-around. */
|
|
4932 return (NULL);
|
|
4933 }
|
|
4934
|
|
4935 /* Allocate an extent node with which to track the chunk. */
|
|
4936 node = base_node_alloc();
|
|
4937 if (node == NULL)
|
|
4938 return (NULL);
|
|
4939
|
|
4940 ret = chunk_alloc(csize, zero, true);
|
|
4941 if (ret == NULL) {
|
|
4942 base_node_dealloc(node);
|
|
4943 return (NULL);
|
|
4944 }
|
|
4945
|
|
4946 /* Insert node into huge. */
|
|
4947 node->addr = ret;
|
|
4948 #ifdef MALLOC_DECOMMIT
|
|
4949 psize = PAGE_CEILING(size);
|
|
4950 node->size = psize;
|
|
4951 #else
|
|
4952 node->size = csize;
|
|
4953 #endif
|
|
4954
|
|
4955 malloc_mutex_lock(&huge_mtx);
|
|
4956 extent_tree_ad_insert(&huge, node);
|
|
4957 #ifdef MALLOC_STATS
|
|
4958 huge_nmalloc++;
|
|
4959 # ifdef MALLOC_DECOMMIT
|
|
4960 huge_allocated += psize;
|
|
4961 # else
|
|
4962 huge_allocated += csize;
|
|
4963 # endif
|
|
4964 #endif
|
|
4965 malloc_mutex_unlock(&huge_mtx);
|
|
4966
|
|
4967 #ifdef MALLOC_DECOMMIT
|
|
4968 if (csize - psize > 0)
|
|
4969 pages_decommit((void *)((uintptr_t)ret + psize), csize - psize);
|
|
4970 #endif
|
|
4971
|
|
4972 #ifdef MALLOC_DECOMMIT
|
|
4973 VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, zero);
|
|
4974 #else
|
|
4975 VALGRIND_MALLOCLIKE_BLOCK(ret, csize, 0, zero);
|
|
4976 #endif
|
|
4977
|
|
4978 #ifdef MALLOC_FILL
|
|
4979 if (zero == false) {
|
|
4980 if (opt_junk)
|
|
4981 # ifdef MALLOC_DECOMMIT
|
|
4982 memset(ret, 0xa5, psize);
|
|
4983 # else
|
|
4984 memset(ret, 0xa5, csize);
|
|
4985 # endif
|
|
4986 else if (opt_zero)
|
|
4987 # ifdef MALLOC_DECOMMIT
|
|
4988 memset(ret, 0, psize);
|
|
4989 # else
|
|
4990 memset(ret, 0, csize);
|
|
4991 # endif
|
|
4992 }
|
|
4993 #endif
|
|
4994
|
|
4995 return (ret);
|
|
4996 }
|
|
4997
|
|
4998 /* Only handles large allocations that require more than chunk alignment. */
|
|
4999 static void *
|
|
5000 huge_palloc(size_t alignment, size_t size)
|
|
5001 {
|
|
5002 void *ret;
|
|
5003 size_t alloc_size, chunk_size, offset;
|
|
5004 #ifdef MALLOC_DECOMMIT
|
|
5005 size_t psize;
|
|
5006 #endif
|
|
5007 extent_node_t *node;
|
|
5008 int pfd;
|
|
5009
|
|
5010 /*
|
|
5011 * This allocation requires alignment that is even larger than chunk
|
|
5012 * alignment. This means that huge_malloc() isn't good enough.
|
|
5013 *
|
|
5014 * Allocate almost twice as many chunks as are demanded by the size or
|
|
5015 * alignment, in order to assure the alignment can be achieved, then
|
|
5016 * unmap leading and trailing chunks.
|
|
5017 */
|
|
5018 assert(alignment >= chunksize);
|
|
5019
|
|
5020 chunk_size = CHUNK_CEILING(size);
|
|
5021
|
|
5022 if (size >= alignment)
|
|
5023 alloc_size = chunk_size + alignment - chunksize;
|
|
5024 else
|
|
5025 alloc_size = (alignment << 1) - chunksize;
|
|
5026
|
|
5027 /* Allocate an extent node with which to track the chunk. */
|
|
5028 node = base_node_alloc();
|
|
5029 if (node == NULL)
|
|
5030 return (NULL);
|
|
5031
|
|
5032 /*
|
|
5033 * Windows requires that there be a 1:1 mapping between VM
|
|
5034 * allocation/deallocation operations. Therefore, take care here to
|
|
5035 * acquire the final result via one mapping operation.
|
|
5036 *
|
|
5037 * The MALLOC_PAGEFILE code also benefits from this mapping algorithm,
|
|
5038 * since it reduces the number of page files.
|
|
5039 */
|
|
5040 #ifdef MALLOC_PAGEFILE
|
|
5041 if (opt_pagefile) {
|
|
5042 pfd = pagefile_init(size);
|
|
5043 if (pfd == -1)
|
|
5044 return (NULL);
|
|
5045 } else
|
|
5046 #endif
|
|
5047 pfd = -1;
|
|
5048 #ifdef JEMALLOC_USES_MAP_ALIGN
|
|
5049 ret = pages_map_align(chunk_size, pfd, alignment);
|
|
5050 #else
|
|
5051 do {
|
|
5052 void *over;
|
|
5053
|
|
5054 over = chunk_alloc(alloc_size, false, false);
|
|
5055 if (over == NULL) {
|
|
5056 base_node_dealloc(node);
|
|
5057 ret = NULL;
|
|
5058 goto RETURN;
|
|
5059 }
|
|
5060
|
|
5061 offset = (uintptr_t)over & (alignment - 1);
|
|
5062 assert((offset & chunksize_mask) == 0);
|
|
5063 assert(offset < alloc_size);
|
|
5064 ret = (void *)((uintptr_t)over + offset);
|
|
5065 chunk_dealloc(over, alloc_size);
|
|
5066 ret = pages_map(ret, chunk_size, pfd);
|
|
5067 /*
|
|
5068 * Failure here indicates a race with another thread, so try
|
|
5069 * again.
|
|
5070 */
|
|
5071 } while (ret == NULL);
|
|
5072 #endif
|
|
5073 /* Insert node into huge. */
|
|
5074 node->addr = ret;
|
|
5075 #ifdef MALLOC_DECOMMIT
|
|
5076 psize = PAGE_CEILING(size);
|
|
5077 node->size = psize;
|
|
5078 #else
|
|
5079 node->size = chunk_size;
|
|
5080 #endif
|
|
5081
|
|
5082 malloc_mutex_lock(&huge_mtx);
|
|
5083 extent_tree_ad_insert(&huge, node);
|
|
5084 #ifdef MALLOC_STATS
|
|
5085 huge_nmalloc++;
|
|
5086 # ifdef MALLOC_DECOMMIT
|
|
5087 huge_allocated += psize;
|
|
5088 # else
|
|
5089 huge_allocated += chunk_size;
|
|
5090 # endif
|
|
5091 #endif
|
|
5092 malloc_mutex_unlock(&huge_mtx);
|
|
5093
|
|
5094 #ifdef MALLOC_DECOMMIT
|
|
5095 if (chunk_size - psize > 0) {
|
|
5096 pages_decommit((void *)((uintptr_t)ret + psize),
|
|
5097 chunk_size - psize);
|
|
5098 }
|
|
5099 #endif
|
|
5100
|
|
5101 #ifdef MALLOC_DECOMMIT
|
|
5102 VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, false);
|
|
5103 #else
|
|
5104 VALGRIND_MALLOCLIKE_BLOCK(ret, chunk_size, 0, false);
|
|
5105 #endif
|
|
5106
|
|
5107 #ifdef MALLOC_FILL
|
|
5108 if (opt_junk)
|
|
5109 # ifdef MALLOC_DECOMMIT
|
|
5110 memset(ret, 0xa5, psize);
|
|
5111 # else
|
|
5112 memset(ret, 0xa5, chunk_size);
|
|
5113 # endif
|
|
5114 else if (opt_zero)
|
|
5115 # ifdef MALLOC_DECOMMIT
|
|
5116 memset(ret, 0, psize);
|
|
5117 # else
|
|
5118 memset(ret, 0, chunk_size);
|
|
5119 # endif
|
|
5120 #endif
|
|
5121
|
|
5122 RETURN:
|
|
5123 #ifdef MALLOC_PAGEFILE
|
|
5124 if (pfd != -1)
|
|
5125 pagefile_close(pfd);
|
|
5126 #endif
|
|
5127 return (ret);
|
|
5128 }
|
|
5129
|
|
5130 static void *
|
|
5131 huge_ralloc(void *ptr, size_t size, size_t oldsize)
|
|
5132 {
|
|
5133 void *ret;
|
|
5134 size_t copysize;
|
|
5135
|
|
5136 /* Avoid moving the allocation if the size class would not change. */
|
|
5137
|
|
5138 if (oldsize > arena_maxclass &&
|
|
5139 CHUNK_CEILING(size) == CHUNK_CEILING(oldsize)) {
|
|
5140 #ifdef MALLOC_DECOMMIT
|
|
5141 size_t psize = PAGE_CEILING(size);
|
|
5142 #endif
|
|
5143 #ifdef MALLOC_FILL
|
|
5144 if (opt_junk && size < oldsize) {
|
|
5145 memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize
|
|
5146 - size);
|
|
5147 }
|
|
5148 #endif
|
|
5149 #ifdef MALLOC_DECOMMIT
|
|
5150 if (psize < oldsize) {
|
|
5151 extent_node_t *node, key;
|
|
5152
|
|
5153 pages_decommit((void *)((uintptr_t)ptr + psize),
|
|
5154 oldsize - psize);
|
|
5155
|
|
5156 /* Update recorded size. */
|
|
5157 malloc_mutex_lock(&huge_mtx);
|
|
5158 key.addr = __DECONST(void *, ptr);
|
|
5159 node = extent_tree_ad_search(&huge, &key);
|
|
5160 assert(node != NULL);
|
|
5161 assert(node->size == oldsize);
|
|
5162 # ifdef MALLOC_STATS
|
|
5163 huge_allocated -= oldsize - psize;
|
|
5164 # endif
|
|
5165 node->size = psize;
|
|
5166 malloc_mutex_unlock(&huge_mtx);
|
|
5167 } else if (psize > oldsize) {
|
|
5168 extent_node_t *node, key;
|
|
5169
|
|
5170 pages_commit((void *)((uintptr_t)ptr + oldsize),
|
|
5171 psize - oldsize);
|
|
5172
|
|
5173 /* Update recorded size. */
|
|
5174 malloc_mutex_lock(&huge_mtx);
|
|
5175 key.addr = __DECONST(void *, ptr);
|
|
5176 node = extent_tree_ad_search(&huge, &key);
|
|
5177 assert(node != NULL);
|
|
5178 assert(node->size == oldsize);
|
|
5179 # ifdef MALLOC_STATS
|
|
5180 huge_allocated += psize - oldsize;
|
|
5181 # endif
|
|
5182 node->size = psize;
|
|
5183 malloc_mutex_unlock(&huge_mtx);
|
|
5184 }
|
|
5185 #endif
|
|
5186 #ifdef MALLOC_FILL
|
|
5187 if (opt_zero && size > oldsize) {
|
|
5188 memset((void *)((uintptr_t)ptr + oldsize), 0, size
|
|
5189 - oldsize);
|
|
5190 }
|
|
5191 #endif
|
|
5192 return (ptr);
|
|
5193 }
|
|
5194
|
|
5195 /*
|
|
5196 * If we get here, then size and oldsize are different enough that we
|
|
5197 * need to use a different size class. In that case, fall back to
|
|
5198 * allocating new space and copying.
|
|
5199 */
|
|
5200 ret = huge_malloc(size, false);
|
|
5201 if (ret == NULL)
|
|
5202 return (NULL);
|
|
5203
|
|
5204 copysize = (size < oldsize) ? size : oldsize;
|
|
5205 #ifdef VM_COPY_MIN
|
|
5206 if (copysize >= VM_COPY_MIN)
|
|
5207 pages_copy(ret, ptr, copysize);
|
|
5208 else
|
|
5209 #endif
|
|
5210 memcpy(ret, ptr, copysize);
|
|
5211 idalloc(ptr);
|
|
5212 return (ret);
|
|
5213 }
|
|
5214
|
|
5215 static void
|
|
5216 huge_dalloc(void *ptr)
|
|
5217 {
|
|
5218 extent_node_t *node, key;
|
|
5219
|
|
5220 malloc_mutex_lock(&huge_mtx);
|
|
5221
|
|
5222 /* Extract from tree of huge allocations. */
|
|
5223 key.addr = ptr;
|
|
5224 node = extent_tree_ad_search(&huge, &key);
|
|
5225 assert(node != NULL);
|
|
5226 assert(node->addr == ptr);
|
|
5227 extent_tree_ad_remove(&huge, node);
|
|
5228
|
|
5229 #ifdef MALLOC_STATS
|
|
5230 huge_ndalloc++;
|
|
5231 huge_allocated -= node->size;
|
|
5232 #endif
|
|
5233
|
|
5234 malloc_mutex_unlock(&huge_mtx);
|
|
5235
|
|
5236 /* Unmap chunk. */
|
|
5237 #ifdef MALLOC_FILL
|
|
5238 if (opt_junk)
|
|
5239 memset(node->addr, 0x5a, node->size);
|
|
5240 #endif
|
|
5241 #ifdef MALLOC_DECOMMIT
|
|
5242 chunk_dealloc(node->addr, CHUNK_CEILING(node->size));
|
|
5243 #else
|
|
5244 chunk_dealloc(node->addr, node->size);
|
|
5245 #endif
|
|
5246 VALGRIND_FREELIKE_BLOCK(node->addr, 0);
|
|
5247
|
|
5248 base_node_dealloc(node);
|
|
5249 }
|
|
5250
|
|
5251 #ifdef MOZ_MEMORY_BSD
|
|
5252 static inline unsigned
|
|
5253 malloc_ncpus(void)
|
|
5254 {
|
|
5255 unsigned ret;
|
|
5256 int mib[2];
|
|
5257 size_t len;
|
|
5258
|
|
5259 mib[0] = CTL_HW;
|
|
5260 mib[1] = HW_NCPU;
|
|
5261 len = sizeof(ret);
|
|
5262 if (sysctl(mib, 2, &ret, &len, (void *) 0, 0) == -1) {
|
|
5263 /* Error. */
|
|
5264 return (1);
|
|
5265 }
|
|
5266
|
|
5267 return (ret);
|
|
5268 }
|
|
5269 #elif (defined(MOZ_MEMORY_LINUX))
|
|
5270 #include <fcntl.h>
|
|
5271
|
|
5272 static inline unsigned
|
|
5273 malloc_ncpus(void)
|
|
5274 {
|
|
5275 unsigned ret;
|
|
5276 int fd, nread, column;
|
|
5277 char buf[1024];
|
|
5278 static const char matchstr[] = "processor\t:";
|
|
5279 int i;
|
|
5280
|
|
5281 /*
|
|
5282 * sysconf(3) would be the preferred method for determining the number
|
|
5283 * of CPUs, but it uses malloc internally, which causes untennable
|
|
5284 * recursion during malloc initialization.
|
|
5285 */
|
|
5286 fd = open("/proc/cpuinfo", O_RDONLY);
|
|
5287 if (fd == -1)
|
|
5288 return (1); /* Error. */
|
|
5289 /*
|
|
5290 * Count the number of occurrences of matchstr at the beginnings of
|
|
5291 * lines. This treats hyperthreaded CPUs as multiple processors.
|
|
5292 */
|
|
5293 column = 0;
|
|
5294 ret = 0;
|
|
5295 while (true) {
|
|
5296 nread = read(fd, &buf, sizeof(buf));
|
|
5297 if (nread <= 0)
|
|
5298 break; /* EOF or error. */
|
|
5299 for (i = 0;i < nread;i++) {
|
|
5300 char c = buf[i];
|
|
5301 if (c == '\n')
|
|
5302 column = 0;
|
|
5303 else if (column != -1) {
|
|
5304 if (c == matchstr[column]) {
|
|
5305 column++;
|
|
5306 if (column == sizeof(matchstr) - 1) {
|
|
5307 column = -1;
|
|
5308 ret++;
|
|
5309 }
|
|
5310 } else
|
|
5311 column = -1;
|
|
5312 }
|
|
5313 }
|
|
5314 }
|
|
5315
|
|
5316 if (ret == 0)
|
|
5317 ret = 1; /* Something went wrong in the parser. */
|
|
5318 close(fd);
|
|
5319
|
|
5320 return (ret);
|
|
5321 }
|
|
5322 #elif (defined(MOZ_MEMORY_DARWIN))
|
|
5323 #include <mach/mach_init.h>
|
|
5324 #include <mach/mach_host.h>
|
|
5325
|
|
5326 static inline unsigned
|
|
5327 malloc_ncpus(void)
|
|
5328 {
|
|
5329 kern_return_t error;
|
|
5330 natural_t n;
|
|
5331 processor_info_array_t pinfo;
|
|
5332 mach_msg_type_number_t pinfocnt;
|
|
5333
|
|
5334 error = host_processor_info(mach_host_self(), PROCESSOR_BASIC_INFO,
|
|
5335 &n, &pinfo, &pinfocnt);
|
|
5336 if (error != KERN_SUCCESS)
|
|
5337 return (1); /* Error. */
|
|
5338 else
|
|
5339 return (n);
|
|
5340 }
|
|
5341 #elif (defined(MOZ_MEMORY_SOLARIS))
|
|
5342
|
|
5343 static inline unsigned
|
|
5344 malloc_ncpus(void)
|
|
5345 {
|
|
5346 return sysconf(_SC_NPROCESSORS_ONLN);
|
|
5347 }
|
|
5348 #else
|
|
5349 static inline unsigned
|
|
5350 malloc_ncpus(void)
|
|
5351 {
|
|
5352
|
|
5353 /*
|
|
5354 * We lack a way to determine the number of CPUs on this platform, so
|
|
5355 * assume 1 CPU.
|
|
5356 */
|
|
5357 return (1);
|
|
5358 }
|
|
5359 #endif
|
|
5360
|
|
5361 static void
|
|
5362 malloc_print_stats(void)
|
|
5363 {
|
|
5364
|
|
5365 if (opt_print_stats) {
|
|
5366 char s[UMAX2S_BUFSIZE];
|
|
5367 _malloc_message("___ Begin malloc statistics ___\n", "", "",
|
|
5368 "");
|
|
5369 _malloc_message("Assertions ",
|
|
5370 #ifdef NDEBUG
|
|
5371 "disabled",
|
|
5372 #else
|
|
5373 "enabled",
|
|
5374 #endif
|
|
5375 "\n", "");
|
|
5376 _malloc_message("Boolean MALLOC_OPTIONS: ",
|
|
5377 opt_abort ? "A" : "a", "", "");
|
|
5378 #ifdef MALLOC_FILL
|
|
5379 _malloc_message(opt_junk ? "J" : "j", "", "", "");
|
|
5380 #endif
|
|
5381 #ifdef MALLOC_PAGEFILE
|
|
5382 _malloc_message(opt_pagefile ? "o" : "O", "", "", "");
|
|
5383 #endif
|
|
5384 _malloc_message("P", "", "", "");
|
|
5385 #ifdef MALLOC_UTRACE
|
|
5386 _malloc_message(opt_utrace ? "U" : "u", "", "", "");
|
|
5387 #endif
|
|
5388 #ifdef MALLOC_SYSV
|
|
5389 _malloc_message(opt_sysv ? "V" : "v", "", "", "");
|
|
5390 #endif
|
|
5391 #ifdef MALLOC_XMALLOC
|
|
5392 _malloc_message(opt_xmalloc ? "X" : "x", "", "", "");
|
|
5393 #endif
|
|
5394 #ifdef MALLOC_FILL
|
|
5395 _malloc_message(opt_zero ? "Z" : "z", "", "", "");
|
|
5396 #endif
|
|
5397 _malloc_message("\n", "", "", "");
|
|
5398
|
|
5399 _malloc_message("CPUs: ", umax2s(ncpus, s), "\n", "");
|
|
5400 _malloc_message("Max arenas: ", umax2s(narenas, s), "\n", "");
|
|
5401 #ifdef MALLOC_BALANCE
|
|
5402 _malloc_message("Arena balance threshold: ",
|
|
5403 umax2s(opt_balance_threshold, s), "\n", "");
|
|
5404 #endif
|
|
5405 _malloc_message("Pointer size: ", umax2s(sizeof(void *), s),
|
|
5406 "\n", "");
|
|
5407 _malloc_message("Quantum size: ", umax2s(quantum, s), "\n", "");
|
|
5408 _malloc_message("Max small size: ", umax2s(small_max, s), "\n",
|
|
5409 "");
|
|
5410 _malloc_message("Max dirty pages per arena: ",
|
|
5411 umax2s(opt_dirty_max, s), "\n", "");
|
|
5412
|
|
5413 _malloc_message("Chunk size: ", umax2s(chunksize, s), "", "");
|
|
5414 _malloc_message(" (2^", umax2s(opt_chunk_2pow, s), ")\n", "");
|
|
5415
|
|
5416 #ifdef MALLOC_STATS
|
|
5417 {
|
|
5418 size_t allocated, mapped;
|
|
5419 #ifdef MALLOC_BALANCE
|
|
5420 uint64_t nbalance = 0;
|
|
5421 #endif
|
|
5422 unsigned i;
|
|
5423 arena_t *arena;
|
|
5424
|
|
5425 /* Calculate and print allocated/mapped stats. */
|
|
5426
|
|
5427 /* arenas. */
|
|
5428 for (i = 0, allocated = 0; i < narenas; i++) {
|
|
5429 if (arenas[i] != NULL) {
|
|
5430 malloc_spin_lock(&arenas[i]->lock);
|
|
5431 allocated +=
|
|
5432 arenas[i]->stats.allocated_small;
|
|
5433 allocated +=
|
|
5434 arenas[i]->stats.allocated_large;
|
|
5435 #ifdef MALLOC_BALANCE
|
|
5436 nbalance += arenas[i]->stats.nbalance;
|
|
5437 #endif
|
|
5438 malloc_spin_unlock(&arenas[i]->lock);
|
|
5439 }
|
|
5440 }
|
|
5441
|
|
5442 /* huge/base. */
|
|
5443 malloc_mutex_lock(&huge_mtx);
|
|
5444 allocated += huge_allocated;
|
|
5445 mapped = stats_chunks.curchunks * chunksize;
|
|
5446 malloc_mutex_unlock(&huge_mtx);
|
|
5447
|
|
5448 malloc_mutex_lock(&base_mtx);
|
|
5449 mapped += base_mapped;
|
|
5450 malloc_mutex_unlock(&base_mtx);
|
|
5451
|
|
5452 #ifdef MOZ_MEMORY_WINDOWS
|
|
5453 malloc_printf("Allocated: %lu, mapped: %lu\n",
|
|
5454 allocated, mapped);
|
|
5455 #else
|
|
5456 malloc_printf("Allocated: %zu, mapped: %zu\n",
|
|
5457 allocated, mapped);
|
|
5458 #endif
|
|
5459
|
|
5460 malloc_mutex_lock(&reserve_mtx);
|
|
5461 malloc_printf("Reserve: min "
|
|
5462 "cur max\n");
|
|
5463 #ifdef MOZ_MEMORY_WINDOWS
|
|
5464 malloc_printf(" %12lu %12lu %12lu\n",
|
|
5465 CHUNK_CEILING(reserve_min) >> opt_chunk_2pow,
|
|
5466 reserve_cur >> opt_chunk_2pow,
|
|
5467 reserve_max >> opt_chunk_2pow);
|
|
5468 #else
|
|
5469 malloc_printf(" %12zu %12zu %12zu\n",
|
|
5470 CHUNK_CEILING(reserve_min) >> opt_chunk_2pow,
|
|
5471 reserve_cur >> opt_chunk_2pow,
|
|
5472 reserve_max >> opt_chunk_2pow);
|
|
5473 #endif
|
|
5474 malloc_mutex_unlock(&reserve_mtx);
|
|
5475
|
|
5476 #ifdef MALLOC_BALANCE
|
|
5477 malloc_printf("Arena balance reassignments: %llu\n",
|
|
5478 nbalance);
|
|
5479 #endif
|
|
5480
|
|
5481 /* Print chunk stats. */
|
|
5482 {
|
|
5483 chunk_stats_t chunks_stats;
|
|
5484
|
|
5485 malloc_mutex_lock(&huge_mtx);
|
|
5486 chunks_stats = stats_chunks;
|
|
5487 malloc_mutex_unlock(&huge_mtx);
|
|
5488
|
|
5489 malloc_printf("chunks: nchunks "
|
|
5490 "highchunks curchunks\n");
|
|
5491 malloc_printf(" %13llu%13lu%13lu\n",
|
|
5492 chunks_stats.nchunks,
|
|
5493 chunks_stats.highchunks,
|
|
5494 chunks_stats.curchunks);
|
|
5495 }
|
|
5496
|
|
5497 /* Print chunk stats. */
|
|
5498 malloc_printf(
|
|
5499 "huge: nmalloc ndalloc allocated\n");
|
|
5500 #ifdef MOZ_MEMORY_WINDOWS
|
|
5501 malloc_printf(" %12llu %12llu %12lu\n",
|
|
5502 huge_nmalloc, huge_ndalloc, huge_allocated);
|
|
5503 #else
|
|
5504 malloc_printf(" %12llu %12llu %12zu\n",
|
|
5505 huge_nmalloc, huge_ndalloc, huge_allocated);
|
|
5506 #endif
|
|
5507 /* Print stats for each arena. */
|
|
5508 for (i = 0; i < narenas; i++) {
|
|
5509 arena = arenas[i];
|
|
5510 if (arena != NULL) {
|
|
5511 malloc_printf(
|
|
5512 "\narenas[%u]:\n", i);
|
|
5513 malloc_spin_lock(&arena->lock);
|
|
5514 stats_print(arena);
|
|
5515 malloc_spin_unlock(&arena->lock);
|
|
5516 }
|
|
5517 }
|
|
5518 }
|
|
5519 #endif /* #ifdef MALLOC_STATS */
|
|
5520 _malloc_message("--- End malloc statistics ---\n", "", "", "");
|
|
5521 }
|
|
5522 }
|
|
5523
|
|
5524 /*
|
|
5525 * FreeBSD's pthreads implementation calls malloc(3), so the malloc
|
|
5526 * implementation has to take pains to avoid infinite recursion during
|
|
5527 * initialization.
|
|
5528 */
|
|
5529 #if (defined(MOZ_MEMORY_WINDOWS) || defined(MOZ_MEMORY_DARWIN)) && !defined(MOZ_MEMORY_WINCE)
|
|
5530 #define malloc_init() false
|
|
5531 #else
|
|
5532 static inline bool
|
|
5533 malloc_init(void)
|
|
5534 {
|
|
5535
|
|
5536 if (malloc_initialized == false)
|
|
5537 return (malloc_init_hard());
|
|
5538
|
|
5539 return (false);
|
|
5540 }
|
|
5541 #endif
|
|
5542
|
|
5543 #if !defined(MOZ_MEMORY_WINDOWS) || defined(MOZ_MEMORY_WINCE)
|
|
5544 static
|
|
5545 #endif
|
|
5546 bool
|
|
5547 malloc_init_hard(void)
|
|
5548 {
|
|
5549 unsigned i;
|
|
5550 char buf[PATH_MAX + 1];
|
|
5551 const char *opts;
|
|
5552 long result;
|
|
5553 #ifndef MOZ_MEMORY_WINDOWS
|
|
5554 int linklen;
|
|
5555 #endif
|
|
5556
|
|
5557 #ifndef MOZ_MEMORY_WINDOWS
|
|
5558 malloc_mutex_lock(&init_lock);
|
|
5559 #endif
|
|
5560
|
|
5561 if (malloc_initialized) {
|
|
5562 /*
|
|
5563 * Another thread initialized the allocator before this one
|
|
5564 * acquired init_lock.
|
|
5565 */
|
|
5566 #ifndef MOZ_MEMORY_WINDOWS
|
|
5567 malloc_mutex_unlock(&init_lock);
|
|
5568 #endif
|
|
5569 return (false);
|
|
5570 }
|
|
5571
|
|
5572 #ifdef MOZ_MEMORY_WINDOWS
|
|
5573 /* get a thread local storage index */
|
|
5574 tlsIndex = TlsAlloc();
|
|
5575 #endif
|
|
5576
|
|
5577 /* Get page size and number of CPUs */
|
|
5578 #ifdef MOZ_MEMORY_WINDOWS
|
|
5579 {
|
|
5580 SYSTEM_INFO info;
|
|
5581
|
|
5582 GetSystemInfo(&info);
|
|
5583 result = info.dwPageSize;
|
|
5584
|
|
5585 pagesize = (unsigned) result;
|
|
5586
|
|
5587 ncpus = info.dwNumberOfProcessors;
|
|
5588 }
|
|
5589 #else
|
|
5590 ncpus = malloc_ncpus();
|
|
5591
|
|
5592 result = sysconf(_SC_PAGESIZE);
|
|
5593 assert(result != -1);
|
|
5594
|
|
5595 pagesize = (unsigned) result;
|
|
5596 #endif
|
|
5597
|
|
5598 /*
|
|
5599 * We assume that pagesize is a power of 2 when calculating
|
|
5600 * pagesize_mask and pagesize_2pow.
|
|
5601 */
|
|
5602 assert(((result - 1) & result) == 0);
|
|
5603 pagesize_mask = result - 1;
|
|
5604 pagesize_2pow = ffs((int)result) - 1;
|
|
5605
|
|
5606 #ifdef MALLOC_PAGEFILE
|
|
5607 /*
|
|
5608 * Determine where to create page files. It is insufficient to
|
|
5609 * unconditionally use P_tmpdir (typically "/tmp"), since for some
|
|
5610 * operating systems /tmp is a separate filesystem that is rather small.
|
|
5611 * Therefore prefer, in order, the following locations:
|
|
5612 *
|
|
5613 * 1) MALLOC_TMPDIR
|
|
5614 * 2) TMPDIR
|
|
5615 * 3) P_tmpdir
|
|
5616 */
|
|
5617 {
|
|
5618 char *s;
|
|
5619 size_t slen;
|
|
5620 static const char suffix[] = "/jemalloc.XXXXXX";
|
|
5621
|
|
5622 if ((s = getenv("MALLOC_TMPDIR")) == NULL && (s =
|
|
5623 getenv("TMPDIR")) == NULL)
|
|
5624 s = P_tmpdir;
|
|
5625 slen = strlen(s);
|
|
5626 if (slen + sizeof(suffix) > sizeof(pagefile_templ)) {
|
|
5627 _malloc_message(_getprogname(),
|
|
5628 ": (malloc) Page file path too long\n",
|
|
5629 "", "");
|
|
5630 abort();
|
|
5631 }
|
|
5632 memcpy(pagefile_templ, s, slen);
|
|
5633 memcpy(&pagefile_templ[slen], suffix, sizeof(suffix));
|
|
5634 }
|
|
5635 #endif
|
|
5636
|
|
5637 for (i = 0; i < 3; i++) {
|
|
5638 unsigned j;
|
|
5639
|
|
5640 /* Get runtime configuration. */
|
|
5641 switch (i) {
|
|
5642 case 0:
|
|
5643 #ifndef MOZ_MEMORY_WINDOWS
|
|
5644 if ((linklen = readlink("/etc/malloc.conf", buf,
|
|
5645 sizeof(buf) - 1)) != -1) {
|
|
5646 /*
|
|
5647 * Use the contents of the "/etc/malloc.conf"
|
|
5648 * symbolic link's name.
|
|
5649 */
|
|
5650 buf[linklen] = '\0';
|
|
5651 opts = buf;
|
|
5652 } else
|
|
5653 #endif
|
|
5654 {
|
|
5655 /* No configuration specified. */
|
|
5656 buf[0] = '\0';
|
|
5657 opts = buf;
|
|
5658 }
|
|
5659 break;
|
|
5660 case 1:
|
|
5661 if (issetugid() == 0 && (opts =
|
|
5662 getenv("MALLOC_OPTIONS")) != NULL) {
|
|
5663 /*
|
|
5664 * Do nothing; opts is already initialized to
|
|
5665 * the value of the MALLOC_OPTIONS environment
|
|
5666 * variable.
|
|
5667 */
|
|
5668 } else {
|
|
5669 /* No configuration specified. */
|
|
5670 buf[0] = '\0';
|
|
5671 opts = buf;
|
|
5672 }
|
|
5673 break;
|
|
5674 case 2:
|
|
5675 if (_malloc_options != NULL) {
|
|
5676 /*
|
|
5677 * Use options that were compiled into the
|
|
5678 * program.
|
|
5679 */
|
|
5680 opts = _malloc_options;
|
|
5681 } else {
|
|
5682 /* No configuration specified. */
|
|
5683 buf[0] = '\0';
|
|
5684 opts = buf;
|
|
5685 }
|
|
5686 break;
|
|
5687 default:
|
|
5688 /* NOTREACHED */
|
|
5689 buf[0] = '\0';
|
|
5690 opts = buf;
|
|
5691 assert(false);
|
|
5692 }
|
|
5693
|
|
5694 for (j = 0; opts[j] != '\0'; j++) {
|
|
5695 unsigned k, nreps;
|
|
5696 bool nseen;
|
|
5697
|
|
5698 /* Parse repetition count, if any. */
|
|
5699 for (nreps = 0, nseen = false;; j++, nseen = true) {
|
|
5700 switch (opts[j]) {
|
|
5701 case '0': case '1': case '2': case '3':
|
|
5702 case '4': case '5': case '6': case '7':
|
|
5703 case '8': case '9':
|
|
5704 nreps *= 10;
|
|
5705 nreps += opts[j] - '0';
|
|
5706 break;
|
|
5707 default:
|
|
5708 goto MALLOC_OUT;
|
|
5709 }
|
|
5710 }
|
|
5711 MALLOC_OUT:
|
|
5712 if (nseen == false)
|
|
5713 nreps = 1;
|
|
5714
|
|
5715 for (k = 0; k < nreps; k++) {
|
|
5716 switch (opts[j]) {
|
|
5717 case 'a':
|
|
5718 opt_abort = false;
|
|
5719 break;
|
|
5720 case 'A':
|
|
5721 opt_abort = true;
|
|
5722 break;
|
|
5723 case 'b':
|
|
5724 #ifdef MALLOC_BALANCE
|
|
5725 opt_balance_threshold >>= 1;
|
|
5726 #endif
|
|
5727 break;
|
|
5728 case 'B':
|
|
5729 #ifdef MALLOC_BALANCE
|
|
5730 if (opt_balance_threshold == 0)
|
|
5731 opt_balance_threshold = 1;
|
|
5732 else if ((opt_balance_threshold << 1)
|
|
5733 > opt_balance_threshold)
|
|
5734 opt_balance_threshold <<= 1;
|
|
5735 #endif
|
|
5736 break;
|
|
5737 case 'f':
|
|
5738 opt_dirty_max >>= 1;
|
|
5739 break;
|
|
5740 case 'F':
|
|
5741 if (opt_dirty_max == 0)
|
|
5742 opt_dirty_max = 1;
|
|
5743 else if ((opt_dirty_max << 1) != 0)
|
|
5744 opt_dirty_max <<= 1;
|
|
5745 break;
|
|
5746 case 'g':
|
|
5747 opt_reserve_range_lshift--;
|
|
5748 break;
|
|
5749 case 'G':
|
|
5750 opt_reserve_range_lshift++;
|
|
5751 break;
|
|
5752 #ifdef MALLOC_FILL
|
|
5753 case 'j':
|
|
5754 opt_junk = false;
|
|
5755 break;
|
|
5756 case 'J':
|
|
5757 opt_junk = true;
|
|
5758 break;
|
|
5759 #endif
|
|
5760 case 'k':
|
|
5761 /*
|
|
5762 * Chunks always require at least one
|
|
5763 * header page, so chunks can never be
|
|
5764 * smaller than two pages.
|
|
5765 */
|
|
5766 if (opt_chunk_2pow > pagesize_2pow + 1)
|
|
5767 opt_chunk_2pow--;
|
|
5768 break;
|
|
5769 case 'K':
|
|
5770 if (opt_chunk_2pow + 1 <
|
|
5771 (sizeof(size_t) << 3))
|
|
5772 opt_chunk_2pow++;
|
|
5773 break;
|
|
5774 case 'n':
|
|
5775 opt_narenas_lshift--;
|
|
5776 break;
|
|
5777 case 'N':
|
|
5778 opt_narenas_lshift++;
|
|
5779 break;
|
|
5780 #ifdef MALLOC_PAGEFILE
|
|
5781 case 'o':
|
|
5782 /* Do not over-commit. */
|
|
5783 opt_pagefile = true;
|
|
5784 break;
|
|
5785 case 'O':
|
|
5786 /* Allow over-commit. */
|
|
5787 opt_pagefile = false;
|
|
5788 break;
|
|
5789 #endif
|
|
5790 case 'p':
|
|
5791 opt_print_stats = false;
|
|
5792 break;
|
|
5793 case 'P':
|
|
5794 opt_print_stats = true;
|
|
5795 break;
|
|
5796 case 'q':
|
|
5797 if (opt_quantum_2pow > QUANTUM_2POW_MIN)
|
|
5798 opt_quantum_2pow--;
|
|
5799 break;
|
|
5800 case 'Q':
|
|
5801 if (opt_quantum_2pow < pagesize_2pow -
|
|
5802 1)
|
|
5803 opt_quantum_2pow++;
|
|
5804 break;
|
|
5805 case 'r':
|
|
5806 opt_reserve_min_lshift--;
|
|
5807 break;
|
|
5808 case 'R':
|
|
5809 opt_reserve_min_lshift++;
|
|
5810 break;
|
|
5811 case 's':
|
|
5812 if (opt_small_max_2pow >
|
|
5813 QUANTUM_2POW_MIN)
|
|
5814 opt_small_max_2pow--;
|
|
5815 break;
|
|
5816 case 'S':
|
|
5817 if (opt_small_max_2pow < pagesize_2pow
|
|
5818 - 1)
|
|
5819 opt_small_max_2pow++;
|
|
5820 break;
|
|
5821 #ifdef MALLOC_UTRACE
|
|
5822 case 'u':
|
|
5823 opt_utrace = false;
|
|
5824 break;
|
|
5825 case 'U':
|
|
5826 opt_utrace = true;
|
|
5827 break;
|
|
5828 #endif
|
|
5829 #ifdef MALLOC_SYSV
|
|
5830 case 'v':
|
|
5831 opt_sysv = false;
|
|
5832 break;
|
|
5833 case 'V':
|
|
5834 opt_sysv = true;
|
|
5835 break;
|
|
5836 #endif
|
|
5837 #ifdef MALLOC_XMALLOC
|
|
5838 case 'x':
|
|
5839 opt_xmalloc = false;
|
|
5840 break;
|
|
5841 case 'X':
|
|
5842 opt_xmalloc = true;
|
|
5843 break;
|
|
5844 #endif
|
|
5845 #ifdef MALLOC_FILL
|
|
5846 case 'z':
|
|
5847 opt_zero = false;
|
|
5848 break;
|
|
5849 case 'Z':
|
|
5850 opt_zero = true;
|
|
5851 break;
|
|
5852 #endif
|
|
5853 default: {
|
|
5854 char cbuf[2];
|
|
5855
|
|
5856 cbuf[0] = opts[j];
|
|
5857 cbuf[1] = '\0';
|
|
5858 _malloc_message(_getprogname(),
|
|
5859 ": (malloc) Unsupported character "
|
|
5860 "in malloc options: '", cbuf,
|
|
5861 "'\n");
|
|
5862 }
|
|
5863 }
|
|
5864 }
|
|
5865 }
|
|
5866 }
|
|
5867
|
|
5868 /* Take care to call atexit() only once. */
|
|
5869 if (opt_print_stats) {
|
|
5870 #ifndef MOZ_MEMORY_WINDOWS
|
|
5871 /* Print statistics at exit. */
|
|
5872 atexit(malloc_print_stats);
|
|
5873 #endif
|
|
5874 }
|
|
5875
|
|
5876 #if (!defined(MOZ_MEMORY_WINDOWS) && !defined(MOZ_MEMORY_DARWIN))
|
|
5877 /* Prevent potential deadlock on malloc locks after fork. */
|
|
5878 pthread_atfork(_malloc_prefork, _malloc_postfork, _malloc_postfork);
|
|
5879 #endif
|
|
5880
|
|
5881 /* Set variables according to the value of opt_small_max_2pow. */
|
|
5882 if (opt_small_max_2pow < opt_quantum_2pow)
|
|
5883 opt_small_max_2pow = opt_quantum_2pow;
|
|
5884 small_max = (1U << opt_small_max_2pow);
|
|
5885
|
|
5886 /* Set bin-related variables. */
|
|
5887 bin_maxclass = (pagesize >> 1);
|
|
5888 assert(opt_quantum_2pow >= TINY_MIN_2POW);
|
|
5889 ntbins = opt_quantum_2pow - TINY_MIN_2POW;
|
|
5890 assert(ntbins <= opt_quantum_2pow);
|
|
5891 nqbins = (small_max >> opt_quantum_2pow);
|
|
5892 nsbins = pagesize_2pow - opt_small_max_2pow - 1;
|
|
5893
|
|
5894 /* Set variables according to the value of opt_quantum_2pow. */
|
|
5895 quantum = (1U << opt_quantum_2pow);
|
|
5896 quantum_mask = quantum - 1;
|
|
5897 if (ntbins > 0)
|
|
5898 small_min = (quantum >> 1) + 1;
|
|
5899 else
|
|
5900 small_min = 1;
|
|
5901 assert(small_min <= quantum);
|
|
5902
|
|
5903 /* Set variables according to the value of opt_chunk_2pow. */
|
|
5904 chunksize = (1LU << opt_chunk_2pow);
|
|
5905 chunksize_mask = chunksize - 1;
|
|
5906 chunk_npages = (chunksize >> pagesize_2pow);
|
|
5907 {
|
|
5908 size_t header_size;
|
|
5909
|
|
5910 /*
|
|
5911 * Compute the header size such that it is large
|
|
5912 * enough to contain the page map and enough nodes for the
|
|
5913 * worst case: one node per non-header page plus one extra for
|
|
5914 * situations where we briefly have one more node allocated
|
|
5915 * than we will need.
|
|
5916 */
|
|
5917 header_size = sizeof(arena_chunk_t) +
|
|
5918 (sizeof(arena_chunk_map_t) * (chunk_npages - 1));
|
|
5919 arena_chunk_header_npages = (header_size >> pagesize_2pow) +
|
|
5920 ((header_size & pagesize_mask) != 0);
|
|
5921 }
|
|
5922 arena_maxclass = chunksize - (arena_chunk_header_npages <<
|
|
5923 pagesize_2pow);
|
|
5924
|
|
5925 #ifdef JEMALLOC_USES_MAP_ALIGN
|
|
5926 /*
|
|
5927 * When using MAP_ALIGN, the alignment parameter must be a power of two
|
|
5928 * multiple of the system pagesize, or mmap will fail.
|
|
5929 */
|
|
5930 assert((chunksize % pagesize) == 0);
|
|
5931 assert((1 << (ffs(chunksize / pagesize) - 1)) == (chunksize/pagesize));
|
|
5932 #endif
|
|
5933
|
|
5934 UTRACE(0, 0, 0);
|
|
5935
|
|
5936 #ifdef MALLOC_STATS
|
|
5937 memset(&stats_chunks, 0, sizeof(chunk_stats_t));
|
|
5938 #endif
|
|
5939
|
|
5940 /* Various sanity checks that regard configuration. */
|
|
5941 assert(quantum >= sizeof(void *));
|
|
5942 assert(quantum <= pagesize);
|
|
5943 assert(chunksize >= pagesize);
|
|
5944 assert(quantum * 4 <= chunksize);
|
|
5945
|
|
5946 /* Initialize chunks data. */
|
|
5947 malloc_mutex_init(&huge_mtx);
|
|
5948 extent_tree_ad_new(&huge);
|
|
5949 #ifdef MALLOC_STATS
|
|
5950 huge_nmalloc = 0;
|
|
5951 huge_ndalloc = 0;
|
|
5952 huge_allocated = 0;
|
|
5953 #endif
|
|
5954
|
|
5955 /* Initialize base allocation data structures. */
|
|
5956 #ifdef MALLOC_STATS
|
|
5957 base_mapped = 0;
|
|
5958 #endif
|
|
5959 base_nodes = NULL;
|
|
5960 base_reserve_regs = NULL;
|
|
5961 malloc_mutex_init(&base_mtx);
|
|
5962
|
|
5963 #ifdef MOZ_MEMORY_NARENAS_DEFAULT_ONE
|
|
5964 narenas = 1;
|
|
5965 #else
|
|
5966 if (ncpus > 1) {
|
|
5967 /*
|
|
5968 * For SMP systems, create four times as many arenas as there
|
|
5969 * are CPUs by default.
|
|
5970 */
|
|
5971 opt_narenas_lshift += 2;
|
|
5972 }
|
|
5973
|
|
5974 /* Determine how many arenas to use. */
|
|
5975 narenas = ncpus;
|
|
5976 #endif
|
|
5977 if (opt_narenas_lshift > 0) {
|
|
5978 if ((narenas << opt_narenas_lshift) > narenas)
|
|
5979 narenas <<= opt_narenas_lshift;
|
|
5980 /*
|
|
5981 * Make sure not to exceed the limits of what base_alloc() can
|
|
5982 * handle.
|
|
5983 */
|
|
5984 if (narenas * sizeof(arena_t *) > chunksize)
|
|
5985 narenas = chunksize / sizeof(arena_t *);
|
|
5986 } else if (opt_narenas_lshift < 0) {
|
|
5987 if ((narenas >> -opt_narenas_lshift) < narenas)
|
|
5988 narenas >>= -opt_narenas_lshift;
|
|
5989 /* Make sure there is at least one arena. */
|
|
5990 if (narenas == 0)
|
|
5991 narenas = 1;
|
|
5992 }
|
|
5993 #ifdef MALLOC_BALANCE
|
|
5994 assert(narenas != 0);
|
|
5995 for (narenas_2pow = 0;
|
|
5996 (narenas >> (narenas_2pow + 1)) != 0;
|
|
5997 narenas_2pow++);
|
|
5998 #endif
|
|
5999
|
|
6000 #ifdef NO_TLS
|
|
6001 if (narenas > 1) {
|
|
6002 static const unsigned primes[] = {1, 3, 5, 7, 11, 13, 17, 19,
|
|
6003 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83,
|
|
6004 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149,
|
|
6005 151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211,
|
|
6006 223, 227, 229, 233, 239, 241, 251, 257, 263};
|
|
6007 unsigned nprimes, parenas;
|
|
6008
|
|
6009 /*
|
|
6010 * Pick a prime number of hash arenas that is more than narenas
|
|
6011 * so that direct hashing of pthread_self() pointers tends to
|
|
6012 * spread allocations evenly among the arenas.
|
|
6013 */
|
|
6014 assert((narenas & 1) == 0); /* narenas must be even. */
|
|
6015 nprimes = (sizeof(primes) >> SIZEOF_INT_2POW);
|
|
6016 parenas = primes[nprimes - 1]; /* In case not enough primes. */
|
|
6017 for (i = 1; i < nprimes; i++) {
|
|
6018 if (primes[i] > narenas) {
|
|
6019 parenas = primes[i];
|
|
6020 break;
|
|
6021 }
|
|
6022 }
|
|
6023 narenas = parenas;
|
|
6024 }
|
|
6025 #endif
|
|
6026
|
|
6027 #ifndef NO_TLS
|
|
6028 # ifndef MALLOC_BALANCE
|
|
6029 next_arena = 0;
|
|
6030 # endif
|
|
6031 #endif
|
|
6032
|
|
6033 /* Allocate and initialize arenas. */
|
|
6034 arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas);
|
|
6035 if (arenas == NULL) {
|
|
6036 #ifndef MOZ_MEMORY_WINDOWS
|
|
6037 malloc_mutex_unlock(&init_lock);
|
|
6038 #endif
|
|
6039 return (true);
|
|
6040 }
|
|
6041 /*
|
|
6042 * Zero the array. In practice, this should always be pre-zeroed,
|
|
6043 * since it was just mmap()ed, but let's be sure.
|
|
6044 */
|
|
6045 memset(arenas, 0, sizeof(arena_t *) * narenas);
|
|
6046
|
|
6047 /*
|
|
6048 * Initialize one arena here. The rest are lazily created in
|
|
6049 * choose_arena_hard().
|
|
6050 */
|
|
6051 arenas_extend(0);
|
|
6052 if (arenas[0] == NULL) {
|
|
6053 #ifndef MOZ_MEMORY_WINDOWS
|
|
6054 malloc_mutex_unlock(&init_lock);
|
|
6055 #endif
|
|
6056 return (true);
|
|
6057 }
|
|
6058 #ifndef NO_TLS
|
|
6059 /*
|
|
6060 * Assign the initial arena to the initial thread, in order to avoid
|
|
6061 * spurious creation of an extra arena if the application switches to
|
|
6062 * threaded mode.
|
|
6063 */
|
|
6064 #ifdef MOZ_MEMORY_WINDOWS
|
|
6065 TlsSetValue(tlsIndex, arenas[0]);
|
|
6066 #else
|
|
6067 arenas_map = arenas[0];
|
|
6068 #endif
|
|
6069 #endif
|
|
6070
|
|
6071 /*
|
|
6072 * Seed here for the initial thread, since choose_arena_hard() is only
|
|
6073 * called for other threads. The seed value doesn't really matter.
|
|
6074 */
|
|
6075 #ifdef MALLOC_BALANCE
|
|
6076 SPRN(balance, 42);
|
|
6077 #endif
|
|
6078
|
|
6079 malloc_spin_init(&arenas_lock);
|
|
6080
|
|
6081 #ifdef MALLOC_VALIDATE
|
|
6082 chunk_rtree = malloc_rtree_new((SIZEOF_PTR << 3) - opt_chunk_2pow);
|
|
6083 if (chunk_rtree == NULL)
|
|
6084 return (true);
|
|
6085 #endif
|
|
6086
|
|
6087 /*
|
|
6088 * Configure and initialize the memory reserve. This needs to happen
|
|
6089 * late during initialization, since chunks are allocated.
|
|
6090 */
|
|
6091 malloc_mutex_init(&reserve_mtx);
|
|
6092 reserve_min = 0;
|
|
6093 reserve_cur = 0;
|
|
6094 reserve_max = 0;
|
|
6095 if (RESERVE_RANGE_2POW_DEFAULT + opt_reserve_range_lshift >= 0) {
|
|
6096 reserve_max += chunksize << (RESERVE_RANGE_2POW_DEFAULT +
|
|
6097 opt_reserve_range_lshift);
|
|
6098 }
|
|
6099 ql_new(&reserve_regs);
|
|
6100 reserve_seq = 0;
|
|
6101 extent_tree_szad_new(&reserve_chunks_szad);
|
|
6102 extent_tree_ad_new(&reserve_chunks_ad);
|
|
6103 if (RESERVE_MIN_2POW_DEFAULT + opt_reserve_min_lshift >= 0) {
|
|
6104 reserve_min_set(chunksize << (RESERVE_MIN_2POW_DEFAULT +
|
|
6105 opt_reserve_min_lshift));
|
|
6106 }
|
|
6107
|
|
6108 malloc_initialized = true;
|
|
6109 #ifndef MOZ_MEMORY_WINDOWS
|
|
6110 malloc_mutex_unlock(&init_lock);
|
|
6111 #endif
|
|
6112 return (false);
|
|
6113 }
|
|
6114
|
|
6115 /* XXX Why not just expose malloc_print_stats()? */
|
|
6116 #ifdef MOZ_MEMORY_WINDOWS
|
|
6117 void
|
|
6118 malloc_shutdown()
|
|
6119 {
|
|
6120
|
|
6121 malloc_print_stats();
|
|
6122 }
|
|
6123 #endif
|
|
6124
|
|
6125 /*
|
|
6126 * End general internal functions.
|
|
6127 */
|
|
6128 /******************************************************************************/
|
|
6129 /*
|
|
6130 * Begin malloc(3)-compatible functions.
|
|
6131 */
|
|
6132
|
|
6133 /*
|
|
6134 * Inline the standard malloc functions if they are being subsumed by Darwin's
|
|
6135 * zone infrastructure.
|
|
6136 */
|
|
6137 #ifdef MOZ_MEMORY_DARWIN
|
|
6138 # define ZONE_INLINE inline
|
|
6139 #else
|
|
6140 # define ZONE_INLINE
|
|
6141 #endif
|
|
6142
|
|
6143 /* Mangle standard interfaces on Darwin and Windows CE,
|
|
6144 in order to avoid linking problems. */
|
|
6145 #if defined(MOZ_MEMORY_DARWIN) || defined(MOZ_MEMORY_WINCE)
|
|
6146 #define malloc(a) moz_malloc(a)
|
|
6147 #define valloc(a) moz_valloc(a)
|
|
6148 #define calloc(a, b) moz_calloc(a, b)
|
|
6149 #define realloc(a, b) moz_realloc(a, b)
|
|
6150 #define free(a) moz_free(a)
|
|
6151 #endif
|
|
6152
|
|
6153 ZONE_INLINE
|
|
6154 void *
|
|
6155 malloc(size_t size)
|
|
6156 {
|
|
6157 void *ret;
|
|
6158
|
|
6159 if (malloc_init()) {
|
|
6160 ret = NULL;
|
|
6161 goto RETURN;
|
|
6162 }
|
|
6163
|
|
6164 if (size == 0) {
|
|
6165 #ifdef MALLOC_SYSV
|
|
6166 if (opt_sysv == false)
|
|
6167 #endif
|
|
6168 size = 1;
|
|
6169 #ifdef MALLOC_SYSV
|
|
6170 else {
|
|
6171 ret = NULL;
|
|
6172 goto RETURN;
|
|
6173 }
|
|
6174 #endif
|
|
6175 }
|
|
6176
|
|
6177 ret = imalloc(size);
|
|
6178
|
|
6179 RETURN:
|
|
6180 if (ret == NULL) {
|
|
6181 #ifdef MALLOC_XMALLOC
|
|
6182 if (opt_xmalloc) {
|
|
6183 _malloc_message(_getprogname(),
|
|
6184 ": (malloc) Error in malloc(): out of memory\n", "",
|
|
6185 "");
|
|
6186 abort();
|
|
6187 }
|
|
6188 #endif
|
|
6189 errno = ENOMEM;
|
|
6190 }
|
|
6191
|
|
6192 UTRACE(0, size, ret);
|
|
6193 return (ret);
|
|
6194 }
|
|
6195
|
|
6196 #ifdef MOZ_MEMORY_SOLARIS
|
|
6197 # ifdef __SUNPRO_C
|
|
6198 void *
|
|
6199 memalign(size_t alignment, size_t size);
|
|
6200 #pragma no_inline(memalign)
|
|
6201 # elif (defined(__GNU_C__))
|
|
6202 __attribute__((noinline))
|
|
6203 # endif
|
|
6204 #else
|
|
6205 inline
|
|
6206 #endif
|
|
6207 void *
|
|
6208 memalign(size_t alignment, size_t size)
|
|
6209 {
|
|
6210 void *ret;
|
|
6211
|
|
6212 assert(((alignment - 1) & alignment) == 0 && alignment >=
|
|
6213 sizeof(void *));
|
|
6214
|
|
6215 if (malloc_init()) {
|
|
6216 ret = NULL;
|
|
6217 goto RETURN;
|
|
6218 }
|
|
6219
|
|
6220 ret = ipalloc(alignment, size);
|
|
6221
|
|
6222 RETURN:
|
|
6223 #ifdef MALLOC_XMALLOC
|
|
6224 if (opt_xmalloc && ret == NULL) {
|
|
6225 _malloc_message(_getprogname(),
|
|
6226 ": (malloc) Error in memalign(): out of memory\n", "", "");
|
|
6227 abort();
|
|
6228 }
|
|
6229 #endif
|
|
6230 UTRACE(0, size, ret);
|
|
6231 return (ret);
|
|
6232 }
|
|
6233
|
|
6234 ZONE_INLINE
|
|
6235 int
|
|
6236 posix_memalign(void **memptr, size_t alignment, size_t size)
|
|
6237 {
|
|
6238 void *result;
|
|
6239
|
|
6240 /* Make sure that alignment is a large enough power of 2. */
|
|
6241 if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *)) {
|
|
6242 #ifdef MALLOC_XMALLOC
|
|
6243 if (opt_xmalloc) {
|
|
6244 _malloc_message(_getprogname(),
|
|
6245 ": (malloc) Error in posix_memalign(): "
|
|
6246 "invalid alignment\n", "", "");
|
|
6247 abort();
|
|
6248 }
|
|
6249 #endif
|
|
6250 return (EINVAL);
|
|
6251 }
|
|
6252
|
|
6253 #ifdef MOZ_MEMORY_DARWIN
|
|
6254 result = moz_memalign(alignment, size);
|
|
6255 #else
|
|
6256 result = memalign(alignment, size);
|
|
6257 #endif
|
|
6258 if (result == NULL)
|
|
6259 return (ENOMEM);
|
|
6260
|
|
6261 *memptr = result;
|
|
6262 return (0);
|
|
6263 }
|
|
6264
|
|
6265 ZONE_INLINE
|
|
6266 void *
|
|
6267 valloc(size_t size)
|
|
6268 {
|
|
6269 #ifdef MOZ_MEMORY_DARWIN
|
|
6270 return (moz_memalign(pagesize, size));
|
|
6271 #else
|
|
6272 return (memalign(pagesize, size));
|
|
6273 #endif
|
|
6274 }
|
|
6275
|
|
6276 ZONE_INLINE
|
|
6277 void *
|
|
6278 calloc(size_t num, size_t size)
|
|
6279 {
|
|
6280 void *ret;
|
|
6281 size_t num_size;
|
|
6282
|
|
6283 if (malloc_init()) {
|
|
6284 num_size = 0;
|
|
6285 ret = NULL;
|
|
6286 goto RETURN;
|
|
6287 }
|
|
6288
|
|
6289 num_size = num * size;
|
|
6290 if (num_size == 0) {
|
|
6291 #ifdef MALLOC_SYSV
|
|
6292 if ((opt_sysv == false) && ((num == 0) || (size == 0)))
|
|
6293 #endif
|
|
6294 num_size = 1;
|
|
6295 #ifdef MALLOC_SYSV
|
|
6296 else {
|
|
6297 ret = NULL;
|
|
6298 goto RETURN;
|
|
6299 }
|
|
6300 #endif
|
|
6301 /*
|
|
6302 * Try to avoid division here. We know that it isn't possible to
|
|
6303 * overflow during multiplication if neither operand uses any of the
|
|
6304 * most significant half of the bits in a size_t.
|
|
6305 */
|
|
6306 } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
|
|
6307 && (num_size / size != num)) {
|
|
6308 /* size_t overflow. */
|
|
6309 ret = NULL;
|
|
6310 goto RETURN;
|
|
6311 }
|
|
6312
|
|
6313 ret = icalloc(num_size);
|
|
6314
|
|
6315 RETURN:
|
|
6316 if (ret == NULL) {
|
|
6317 #ifdef MALLOC_XMALLOC
|
|
6318 if (opt_xmalloc) {
|
|
6319 _malloc_message(_getprogname(),
|
|
6320 ": (malloc) Error in calloc(): out of memory\n", "",
|
|
6321 "");
|
|
6322 abort();
|
|
6323 }
|
|
6324 #endif
|
|
6325 errno = ENOMEM;
|
|
6326 }
|
|
6327
|
|
6328 UTRACE(0, num_size, ret);
|
|
6329 return (ret);
|
|
6330 }
|
|
6331
|
|
6332 ZONE_INLINE
|
|
6333 void *
|
|
6334 realloc(void *ptr, size_t size)
|
|
6335 {
|
|
6336 void *ret;
|
|
6337
|
|
6338 if (size == 0) {
|
|
6339 #ifdef MALLOC_SYSV
|
|
6340 if (opt_sysv == false)
|
|
6341 #endif
|
|
6342 size = 1;
|
|
6343 #ifdef MALLOC_SYSV
|
|
6344 else {
|
|
6345 if (ptr != NULL)
|
|
6346 idalloc(ptr);
|
|
6347 ret = NULL;
|
|
6348 goto RETURN;
|
|
6349 }
|
|
6350 #endif
|
|
6351 }
|
|
6352
|
|
6353 if (ptr != NULL) {
|
|
6354 assert(malloc_initialized);
|
|
6355
|
|
6356 ret = iralloc(ptr, size);
|
|
6357
|
|
6358 if (ret == NULL) {
|
|
6359 #ifdef MALLOC_XMALLOC
|
|
6360 if (opt_xmalloc) {
|
|
6361 _malloc_message(_getprogname(),
|
|
6362 ": (malloc) Error in realloc(): out of "
|
|
6363 "memory\n", "", "");
|
|
6364 abort();
|
|
6365 }
|
|
6366 #endif
|
|
6367 errno = ENOMEM;
|
|
6368 }
|
|
6369 } else {
|
|
6370 if (malloc_init())
|
|
6371 ret = NULL;
|
|
6372 else
|
|
6373 ret = imalloc(size);
|
|
6374
|
|
6375 if (ret == NULL) {
|
|
6376 #ifdef MALLOC_XMALLOC
|
|
6377 if (opt_xmalloc) {
|
|
6378 _malloc_message(_getprogname(),
|
|
6379 ": (malloc) Error in realloc(): out of "
|
|
6380 "memory\n", "", "");
|
|
6381 abort();
|
|
6382 }
|
|
6383 #endif
|
|
6384 errno = ENOMEM;
|
|
6385 }
|
|
6386 }
|
|
6387
|
|
6388 #ifdef MALLOC_SYSV
|
|
6389 RETURN:
|
|
6390 #endif
|
|
6391 UTRACE(ptr, size, ret);
|
|
6392 return (ret);
|
|
6393 }
|
|
6394
|
|
6395 ZONE_INLINE
|
|
6396 void
|
|
6397 free(void *ptr)
|
|
6398 {
|
|
6399
|
|
6400 UTRACE(ptr, 0, 0);
|
|
6401 if (ptr != NULL) {
|
|
6402 assert(malloc_initialized);
|
|
6403
|
|
6404 idalloc(ptr);
|
|
6405 }
|
|
6406 }
|
|
6407
|
|
6408 /*
|
|
6409 * End malloc(3)-compatible functions.
|
|
6410 */
|
|
6411 /******************************************************************************/
|
|
6412 /*
|
|
6413 * Begin non-standard functions.
|
|
6414 */
|
|
6415
|
|
6416 size_t
|
|
6417 malloc_usable_size(const void *ptr)
|
|
6418 {
|
|
6419
|
|
6420 #ifdef MALLOC_VALIDATE
|
|
6421 return (isalloc_validate(ptr));
|
|
6422 #else
|
|
6423 assert(ptr != NULL);
|
|
6424
|
|
6425 return (isalloc(ptr));
|
|
6426 #endif
|
|
6427 }
|
|
6428
|
|
6429 void
|
|
6430 jemalloc_stats(jemalloc_stats_t *stats)
|
|
6431 {
|
|
6432 size_t i;
|
|
6433
|
|
6434 assert(stats != NULL);
|
|
6435
|
|
6436 /*
|
|
6437 * Gather runtime settings.
|
|
6438 */
|
|
6439 stats->opt_abort = opt_abort;
|
|
6440 stats->opt_junk =
|
|
6441 #ifdef MALLOC_FILL
|
|
6442 opt_junk ? true :
|
|
6443 #endif
|
|
6444 false;
|
|
6445 stats->opt_utrace =
|
|
6446 #ifdef MALLOC_UTRACE
|
|
6447 opt_utrace ? true :
|
|
6448 #endif
|
|
6449 false;
|
|
6450 stats->opt_sysv =
|
|
6451 #ifdef MALLOC_SYSV
|
|
6452 opt_sysv ? true :
|
|
6453 #endif
|
|
6454 false;
|
|
6455 stats->opt_xmalloc =
|
|
6456 #ifdef MALLOC_XMALLOC
|
|
6457 opt_xmalloc ? true :
|
|
6458 #endif
|
|
6459 false;
|
|
6460 stats->opt_zero =
|
|
6461 #ifdef MALLOC_FILL
|
|
6462 opt_zero ? true :
|
|
6463 #endif
|
|
6464 false;
|
|
6465 stats->narenas = narenas;
|
|
6466 stats->balance_threshold =
|
|
6467 #ifdef MALLOC_BALANCE
|
|
6468 opt_balance_threshold
|
|
6469 #else
|
|
6470 SIZE_T_MAX
|
|
6471 #endif
|
|
6472 ;
|
|
6473 stats->quantum = quantum;
|
|
6474 stats->small_max = small_max;
|
|
6475 stats->large_max = arena_maxclass;
|
|
6476 stats->chunksize = chunksize;
|
|
6477 stats->dirty_max = opt_dirty_max;
|
|
6478
|
|
6479 malloc_mutex_lock(&reserve_mtx);
|
|
6480 stats->reserve_min = reserve_min;
|
|
6481 stats->reserve_max = reserve_max;
|
|
6482 stats->reserve_cur = reserve_cur;
|
|
6483 malloc_mutex_unlock(&reserve_mtx);
|
|
6484
|
|
6485 /*
|
|
6486 * Gather current memory usage statistics.
|
|
6487 */
|
|
6488 stats->mapped = 0;
|
|
6489 stats->committed = 0;
|
|
6490 stats->allocated = 0;
|
|
6491 stats->dirty = 0;
|
|
6492
|
|
6493 /* Get huge mapped/allocated. */
|
|
6494 malloc_mutex_lock(&huge_mtx);
|
|
6495 stats->mapped += stats_chunks.curchunks * chunksize;
|
|
6496 #ifdef MALLOC_DECOMMIT
|
|
6497 stats->committed += huge_allocated;
|
|
6498 #endif
|
|
6499 stats->allocated += huge_allocated;
|
|
6500 malloc_mutex_unlock(&huge_mtx);
|
|
6501
|
|
6502 /* Get base mapped. */
|
|
6503 malloc_mutex_lock(&base_mtx);
|
|
6504 stats->mapped += base_mapped;
|
|
6505 #ifdef MALLOC_DECOMMIT
|
|
6506 stats->committed += base_mapped;
|
|
6507 #endif
|
|
6508 malloc_mutex_unlock(&base_mtx);
|
|
6509
|
|
6510 /* Iterate over arenas and their chunks. */
|
|
6511 for (i = 0; i < narenas; i++) {
|
|
6512 arena_t *arena = arenas[i];
|
|
6513 if (arena != NULL) {
|
|
6514 arena_chunk_t *chunk;
|
|
6515
|
|
6516 malloc_spin_lock(&arena->lock);
|
|
6517 stats->allocated += arena->stats.allocated_small;
|
|
6518 stats->allocated += arena->stats.allocated_large;
|
|
6519 #ifdef MALLOC_DECOMMIT
|
|
6520 rb_foreach_begin(arena_chunk_t, link_dirty,
|
|
6521 &arena->chunks_dirty, chunk) {
|
|
6522 size_t j;
|
|
6523
|
|
6524 for (j = 0; j < chunk_npages; j++) {
|
|
6525 if ((chunk->map[j].bits &
|
|
6526 CHUNK_MAP_DECOMMITTED) == 0)
|
|
6527 stats->committed += pagesize;
|
|
6528 }
|
|
6529 } rb_foreach_end(arena_chunk_t, link_dirty,
|
|
6530 &arena->chunks_dirty, chunk)
|
|
6531 #endif
|
|
6532 stats->dirty += (arena->ndirty << pagesize_2pow);
|
|
6533 malloc_spin_unlock(&arena->lock);
|
|
6534 }
|
|
6535 }
|
|
6536
|
|
6537 #ifndef MALLOC_DECOMMIT
|
|
6538 stats->committed = stats->mapped;
|
|
6539 #endif
|
|
6540 }
|
|
6541
|
|
6542 void *
|
|
6543 xmalloc(size_t size)
|
|
6544 {
|
|
6545 void *ret;
|
|
6546
|
|
6547 if (malloc_init())
|
|
6548 reserve_fail(size, "xmalloc");
|
|
6549
|
|
6550 if (size == 0) {
|
|
6551 #ifdef MALLOC_SYSV
|
|
6552 if (opt_sysv == false)
|
|
6553 #endif
|
|
6554 size = 1;
|
|
6555 #ifdef MALLOC_SYSV
|
|
6556 else {
|
|
6557 _malloc_message(_getprogname(),
|
|
6558 ": (malloc) Error in xmalloc(): ",
|
|
6559 "invalid size 0", "\n");
|
|
6560 abort();
|
|
6561 }
|
|
6562 #endif
|
|
6563 }
|
|
6564
|
|
6565 ret = imalloc(size);
|
|
6566 if (ret == NULL) {
|
|
6567 uint64_t seq = 0;
|
|
6568
|
|
6569 do {
|
|
6570 seq = reserve_crit(size, "xmalloc", seq);
|
|
6571 ret = imalloc(size);
|
|
6572 } while (ret == NULL);
|
|
6573 }
|
|
6574
|
|
6575 UTRACE(0, size, ret);
|
|
6576 return (ret);
|
|
6577 }
|
|
6578
|
|
6579 void *
|
|
6580 xcalloc(size_t num, size_t size)
|
|
6581 {
|
|
6582 void *ret;
|
|
6583 size_t num_size;
|
|
6584
|
|
6585 num_size = num * size;
|
|
6586 if (malloc_init())
|
|
6587 reserve_fail(num_size, "xcalloc");
|
|
6588
|
|
6589 if (num_size == 0) {
|
|
6590 #ifdef MALLOC_SYSV
|
|
6591 if ((opt_sysv == false) && ((num == 0) || (size == 0)))
|
|
6592 #endif
|
|
6593 num_size = 1;
|
|
6594 #ifdef MALLOC_SYSV
|
|
6595 else {
|
|
6596 _malloc_message(_getprogname(),
|
|
6597 ": (malloc) Error in xcalloc(): ",
|
|
6598 "invalid size 0", "\n");
|
|
6599 abort();
|
|
6600 }
|
|
6601 #endif
|
|
6602 /*
|
|
6603 * Try to avoid division here. We know that it isn't possible to
|
|
6604 * overflow during multiplication if neither operand uses any of the
|
|
6605 * most significant half of the bits in a size_t.
|
|
6606 */
|
|
6607 } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
|
|
6608 && (num_size / size != num)) {
|
|
6609 /* size_t overflow. */
|
|
6610 _malloc_message(_getprogname(),
|
|
6611 ": (malloc) Error in xcalloc(): ",
|
|
6612 "size overflow", "\n");
|
|
6613 abort();
|
|
6614 }
|
|
6615
|
|
6616 ret = icalloc(num_size);
|
|
6617 if (ret == NULL) {
|
|
6618 uint64_t seq = 0;
|
|
6619
|
|
6620 do {
|
|
6621 seq = reserve_crit(num_size, "xcalloc", seq);
|
|
6622 ret = icalloc(num_size);
|
|
6623 } while (ret == NULL);
|
|
6624 }
|
|
6625
|
|
6626 UTRACE(0, num_size, ret);
|
|
6627 return (ret);
|
|
6628 }
|
|
6629
|
|
6630 void *
|
|
6631 xrealloc(void *ptr, size_t size)
|
|
6632 {
|
|
6633 void *ret;
|
|
6634
|
|
6635 if (size == 0) {
|
|
6636 #ifdef MALLOC_SYSV
|
|
6637 if (opt_sysv == false)
|
|
6638 #endif
|
|
6639 size = 1;
|
|
6640 #ifdef MALLOC_SYSV
|
|
6641 else {
|
|
6642 if (ptr != NULL)
|
|
6643 idalloc(ptr);
|
|
6644 _malloc_message(_getprogname(),
|
|
6645 ": (malloc) Error in xrealloc(): ",
|
|
6646 "invalid size 0", "\n");
|
|
6647 abort();
|
|
6648 }
|
|
6649 #endif
|
|
6650 }
|
|
6651
|
|
6652 if (ptr != NULL) {
|
|
6653 assert(malloc_initialized);
|
|
6654
|
|
6655 ret = iralloc(ptr, size);
|
|
6656 if (ret == NULL) {
|
|
6657 uint64_t seq = 0;
|
|
6658
|
|
6659 do {
|
|
6660 seq = reserve_crit(size, "xrealloc", seq);
|
|
6661 ret = iralloc(ptr, size);
|
|
6662 } while (ret == NULL);
|
|
6663 }
|
|
6664 } else {
|
|
6665 if (malloc_init())
|
|
6666 reserve_fail(size, "xrealloc");
|
|
6667
|
|
6668 ret = imalloc(size);
|
|
6669 if (ret == NULL) {
|
|
6670 uint64_t seq = 0;
|
|
6671
|
|
6672 do {
|
|
6673 seq = reserve_crit(size, "xrealloc", seq);
|
|
6674 ret = imalloc(size);
|
|
6675 } while (ret == NULL);
|
|
6676 }
|
|
6677 }
|
|
6678
|
|
6679 UTRACE(ptr, size, ret);
|
|
6680 return (ret);
|
|
6681 }
|
|
6682
|
|
6683 void *
|
|
6684 xmemalign(size_t alignment, size_t size)
|
|
6685 {
|
|
6686 void *ret;
|
|
6687
|
|
6688 assert(((alignment - 1) & alignment) == 0 && alignment >=
|
|
6689 sizeof(void *));
|
|
6690
|
|
6691 if (malloc_init())
|
|
6692 reserve_fail(size, "xmemalign");
|
|
6693
|
|
6694 ret = ipalloc(alignment, size);
|
|
6695 if (ret == NULL) {
|
|
6696 uint64_t seq = 0;
|
|
6697
|
|
6698 do {
|
|
6699 seq = reserve_crit(size, "xmemalign", seq);
|
|
6700 ret = ipalloc(alignment, size);
|
|
6701 } while (ret == NULL);
|
|
6702 }
|
|
6703
|
|
6704 UTRACE(0, size, ret);
|
|
6705 return (ret);
|
|
6706 }
|
|
6707
|
|
6708 static void
|
|
6709 reserve_shrink(void)
|
|
6710 {
|
|
6711 extent_node_t *node;
|
|
6712
|
|
6713 assert(reserve_cur > reserve_max);
|
|
6714 #ifdef MALLOC_DEBUG
|
|
6715 {
|
|
6716 extent_node_t *node;
|
|
6717 size_t reserve_size;
|
|
6718
|
|
6719 reserve_size = 0;
|
|
6720 rb_foreach_begin(extent_node_t, link_szad, &reserve_chunks_szad,
|
|
6721 node) {
|
|
6722 reserve_size += node->size;
|
|
6723 } rb_foreach_end(extent_node_t, link_szad, &reserve_chunks_szad,
|
|
6724 node)
|
|
6725 assert(reserve_size == reserve_cur);
|
|
6726
|
|
6727 reserve_size = 0;
|
|
6728 rb_foreach_begin(extent_node_t, link_ad, &reserve_chunks_ad,
|
|
6729 node) {
|
|
6730 reserve_size += node->size;
|
|
6731 } rb_foreach_end(extent_node_t, link_ad, &reserve_chunks_ad,
|
|
6732 node)
|
|
6733 assert(reserve_size == reserve_cur);
|
|
6734 }
|
|
6735 #endif
|
|
6736
|
|
6737 /* Discard chunks until the the reserve is below the size limit. */
|
|
6738 rb_foreach_reverse_begin(extent_node_t, link_ad, &reserve_chunks_ad,
|
|
6739 node) {
|
|
6740 #ifndef MALLOC_DECOMMIT
|
|
6741 if (node->size <= reserve_cur - reserve_max) {
|
|
6742 #endif
|
|
6743 extent_node_t *tnode = extent_tree_ad_prev(
|
|
6744 &reserve_chunks_ad, node);
|
|
6745
|
|
6746 #ifdef MALLOC_DECOMMIT
|
|
6747 assert(node->size <= reserve_cur - reserve_max);
|
|
6748 #endif
|
|
6749
|
|
6750 /* Discard the entire [multi-]chunk. */
|
|
6751 extent_tree_szad_remove(&reserve_chunks_szad, node);
|
|
6752 extent_tree_ad_remove(&reserve_chunks_ad, node);
|
|
6753 reserve_cur -= node->size;
|
|
6754 pages_unmap(node->addr, node->size);
|
|
6755 #ifdef MALLOC_STATS
|
|
6756 stats_chunks.curchunks -= (node->size / chunksize);
|
|
6757 #endif
|
|
6758 base_node_dealloc(node);
|
|
6759 if (reserve_cur == reserve_max)
|
|
6760 break;
|
|
6761
|
|
6762 rb_foreach_reverse_prev(extent_node_t, link_ad,
|
|
6763 extent_ad_comp, &reserve_chunks_ad, tnode);
|
|
6764 #ifndef MALLOC_DECOMMIT
|
|
6765 } else {
|
|
6766 /* Discard the end of the multi-chunk. */
|
|
6767 extent_tree_szad_remove(&reserve_chunks_szad, node);
|
|
6768 node->size -= reserve_cur - reserve_max;
|
|
6769 extent_tree_szad_insert(&reserve_chunks_szad, node);
|
|
6770 pages_unmap((void *)((uintptr_t)node->addr +
|
|
6771 node->size), reserve_cur - reserve_max);
|
|
6772 #ifdef MALLOC_STATS
|
|
6773 stats_chunks.curchunks -= ((reserve_cur - reserve_max) /
|
|
6774 chunksize);
|
|
6775 #endif
|
|
6776 reserve_cur = reserve_max;
|
|
6777 break;
|
|
6778 }
|
|
6779 #endif
|
|
6780 assert(reserve_cur > reserve_max);
|
|
6781 } rb_foreach_reverse_end(extent_node_t, link_ad, &reserve_chunks_ad,
|
|
6782 node)
|
|
6783 }
|
|
6784
|
|
6785 /* Send a condition notification. */
|
|
6786 static uint64_t
|
|
6787 reserve_notify(reserve_cnd_t cnd, size_t size, uint64_t seq)
|
|
6788 {
|
|
6789 reserve_reg_t *reg;
|
|
6790
|
|
6791 /* seq is used to keep track of distinct condition-causing events. */
|
|
6792 if (seq == 0) {
|
|
6793 /* Allocate new sequence number. */
|
|
6794 reserve_seq++;
|
|
6795 seq = reserve_seq;
|
|
6796 }
|
|
6797
|
|
6798 /*
|
|
6799 * Advance to the next callback registration and send a notification,
|
|
6800 * unless one has already been sent for this condition-causing event.
|
|
6801 */
|
|
6802 reg = ql_first(&reserve_regs);
|
|
6803 if (reg == NULL)
|
|
6804 return (0);
|
|
6805 ql_first(&reserve_regs) = ql_next(&reserve_regs, reg, link);
|
|
6806 if (reg->seq == seq)
|
|
6807 return (0);
|
|
6808 reg->seq = seq;
|
|
6809 malloc_mutex_unlock(&reserve_mtx);
|
|
6810 reg->cb(reg->ctx, cnd, size);
|
|
6811 malloc_mutex_lock(&reserve_mtx);
|
|
6812
|
|
6813 return (seq);
|
|
6814 }
|
|
6815
|
|
6816 /* Allocation failure due to OOM. Try to free some memory via callbacks. */
|
|
6817 static uint64_t
|
|
6818 reserve_crit(size_t size, const char *fname, uint64_t seq)
|
|
6819 {
|
|
6820
|
|
6821 /*
|
|
6822 * Send one condition notification. Iteration is handled by the
|
|
6823 * caller of this function.
|
|
6824 */
|
|
6825 malloc_mutex_lock(&reserve_mtx);
|
|
6826 seq = reserve_notify(RESERVE_CND_CRIT, size, seq);
|
|
6827 malloc_mutex_unlock(&reserve_mtx);
|
|
6828
|
|
6829 /* If no notification could be sent, then no further recourse exists. */
|
|
6830 if (seq == 0)
|
|
6831 reserve_fail(size, fname);
|
|
6832
|
|
6833 return (seq);
|
|
6834 }
|
|
6835
|
|
6836 /* Permanent allocation failure due to OOM. */
|
|
6837 static void
|
|
6838 reserve_fail(size_t size, const char *fname)
|
|
6839 {
|
|
6840 uint64_t seq = 0;
|
|
6841
|
|
6842 /* Send fail notifications. */
|
|
6843 malloc_mutex_lock(&reserve_mtx);
|
|
6844 do {
|
|
6845 seq = reserve_notify(RESERVE_CND_FAIL, size, seq);
|
|
6846 } while (seq != 0);
|
|
6847 malloc_mutex_unlock(&reserve_mtx);
|
|
6848
|
|
6849 /* Terminate the application. */
|
|
6850 _malloc_message(_getprogname(),
|
|
6851 ": (malloc) Error in ", fname, "(): out of memory\n");
|
|
6852 abort();
|
|
6853 }
|
|
6854
|
|
6855 bool
|
|
6856 reserve_cb_register(reserve_cb_t *cb, void *ctx)
|
|
6857 {
|
|
6858 reserve_reg_t *reg = base_reserve_reg_alloc();
|
|
6859 if (reg == NULL)
|
|
6860 return (true);
|
|
6861
|
|
6862 ql_elm_new(reg, link);
|
|
6863 reg->cb = cb;
|
|
6864 reg->ctx = ctx;
|
|
6865 reg->seq = 0;
|
|
6866
|
|
6867 malloc_mutex_lock(&reserve_mtx);
|
|
6868 ql_head_insert(&reserve_regs, reg, link);
|
|
6869 malloc_mutex_unlock(&reserve_mtx);
|
|
6870
|
|
6871 return (false);
|
|
6872 }
|
|
6873
|
|
6874 bool
|
|
6875 reserve_cb_unregister(reserve_cb_t *cb, void *ctx)
|
|
6876 {
|
|
6877 reserve_reg_t *reg = NULL;
|
|
6878
|
|
6879 malloc_mutex_lock(&reserve_mtx);
|
|
6880 ql_foreach(reg, &reserve_regs, link) {
|
|
6881 if (reg->cb == cb && reg->ctx == ctx) {
|
|
6882 ql_remove(&reserve_regs, reg, link);
|
|
6883 break;
|
|
6884 }
|
|
6885 }
|
|
6886 malloc_mutex_unlock(&reserve_mtx);
|
|
6887
|
|
6888 if (reg != NULL)
|
|
6889 base_reserve_reg_dealloc(reg);
|
|
6890 return (false);
|
|
6891 return (true);
|
|
6892 }
|
|
6893
|
|
6894 size_t
|
|
6895 reserve_cur_get(void)
|
|
6896 {
|
|
6897 size_t ret;
|
|
6898
|
|
6899 malloc_mutex_lock(&reserve_mtx);
|
|
6900 ret = reserve_cur;
|
|
6901 malloc_mutex_unlock(&reserve_mtx);
|
|
6902
|
|
6903 return (ret);
|
|
6904 }
|
|
6905
|
|
6906 size_t
|
|
6907 reserve_min_get(void)
|
|
6908 {
|
|
6909 size_t ret;
|
|
6910
|
|
6911 malloc_mutex_lock(&reserve_mtx);
|
|
6912 ret = reserve_min;
|
|
6913 malloc_mutex_unlock(&reserve_mtx);
|
|
6914
|
|
6915 return (ret);
|
|
6916 }
|
|
6917
|
|
6918 bool
|
|
6919 reserve_min_set(size_t min)
|
|
6920 {
|
|
6921
|
|
6922 min = CHUNK_CEILING(min);
|
|
6923
|
|
6924 malloc_mutex_lock(&reserve_mtx);
|
|
6925 /* Keep |reserve_max - reserve_min| the same. */
|
|
6926 if (min < reserve_min) {
|
|
6927 reserve_max -= reserve_min - min;
|
|
6928 reserve_min = min;
|
|
6929 } else {
|
|
6930 /* Protect against wrap-around. */
|
|
6931 if (reserve_max + min - reserve_min < reserve_max) {
|
|
6932 reserve_min = SIZE_T_MAX - (reserve_max - reserve_min)
|
|
6933 - chunksize + 1;
|
|
6934 reserve_max = SIZE_T_MAX - chunksize + 1;
|
|
6935 } else {
|
|
6936 reserve_max += min - reserve_min;
|
|
6937 reserve_min = min;
|
|
6938 }
|
|
6939 }
|
|
6940
|
|
6941 /* Resize the reserve if necessary. */
|
|
6942 if (reserve_cur < reserve_min) {
|
|
6943 size_t size = reserve_min - reserve_cur;
|
|
6944
|
|
6945 /* Force the reserve to grow by allocating/deallocating. */
|
|
6946 malloc_mutex_unlock(&reserve_mtx);
|
|
6947 #ifdef MALLOC_DECOMMIT
|
|
6948 {
|
|
6949 void **chunks;
|
|
6950 size_t i, n;
|
|
6951
|
|
6952 n = size >> opt_chunk_2pow;
|
|
6953 chunks = (void**)imalloc(n * sizeof(void *));
|
|
6954 if (chunks == NULL)
|
|
6955 return (true);
|
|
6956 for (i = 0; i < n; i++) {
|
|
6957 chunks[i] = huge_malloc(chunksize, false);
|
|
6958 if (chunks[i] == NULL) {
|
|
6959 size_t j;
|
|
6960
|
|
6961 for (j = 0; j < i; j++) {
|
|
6962 huge_dalloc(chunks[j]);
|
|
6963 }
|
|
6964 idalloc(chunks);
|
|
6965 return (true);
|
|
6966 }
|
|
6967 }
|
|
6968 for (i = 0; i < n; i++)
|
|
6969 huge_dalloc(chunks[i]);
|
|
6970 idalloc(chunks);
|
|
6971 }
|
|
6972 #else
|
|
6973 {
|
|
6974 void *x = huge_malloc(size, false);
|
|
6975 if (x == NULL) {
|
|
6976 return (true);
|
|
6977 }
|
|
6978 huge_dalloc(x);
|
|
6979 }
|
|
6980 #endif
|
|
6981 } else if (reserve_cur > reserve_max) {
|
|
6982 reserve_shrink();
|
|
6983 malloc_mutex_unlock(&reserve_mtx);
|
|
6984 } else
|
|
6985 malloc_mutex_unlock(&reserve_mtx);
|
|
6986
|
|
6987 return (false);
|
|
6988 }
|
|
6989
|
|
6990 #ifdef MOZ_MEMORY_WINDOWS
|
|
6991 void*
|
|
6992 _recalloc(void *ptr, size_t count, size_t size)
|
|
6993 {
|
|
6994 size_t oldsize = (ptr != NULL) ? isalloc(ptr) : 0;
|
|
6995 size_t newsize = count * size;
|
|
6996
|
|
6997 /*
|
|
6998 * In order for all trailing bytes to be zeroed, the caller needs to
|
|
6999 * use calloc(), followed by recalloc(). However, the current calloc()
|
|
7000 * implementation only zeros the bytes requested, so if recalloc() is
|
|
7001 * to work 100% correctly, calloc() will need to change to zero
|
|
7002 * trailing bytes.
|
|
7003 */
|
|
7004
|
|
7005 ptr = realloc(ptr, newsize);
|
|
7006 if (ptr != NULL && oldsize < newsize) {
|
|
7007 memset((void *)((uintptr_t)ptr + oldsize), 0, newsize -
|
|
7008 oldsize);
|
|
7009 }
|
|
7010
|
|
7011 return ptr;
|
|
7012 }
|
|
7013
|
|
7014 /*
|
|
7015 * This impl of _expand doesn't ever actually expand or shrink blocks: it
|
|
7016 * simply replies that you may continue using a shrunk block.
|
|
7017 */
|
|
7018 void*
|
|
7019 _expand(void *ptr, size_t newsize)
|
|
7020 {
|
|
7021 if (isalloc(ptr) >= newsize)
|
|
7022 return ptr;
|
|
7023
|
|
7024 return NULL;
|
|
7025 }
|
|
7026
|
|
7027 size_t
|
|
7028 _msize(const void *ptr)
|
|
7029 {
|
|
7030
|
|
7031 return malloc_usable_size(ptr);
|
|
7032 }
|
|
7033 #endif
|
|
7034
|
|
7035 /*
|
|
7036 * End non-standard functions.
|
|
7037 */
|
|
7038 /******************************************************************************/
|
|
7039 /*
|
|
7040 * Begin library-private functions, used by threading libraries for protection
|
|
7041 * of malloc during fork(). These functions are only called if the program is
|
|
7042 * running in threaded mode, so there is no need to check whether the program
|
|
7043 * is threaded here.
|
|
7044 */
|
|
7045
|
|
7046 void
|
|
7047 _malloc_prefork(void)
|
|
7048 {
|
|
7049 unsigned i;
|
|
7050
|
|
7051 /* Acquire all mutexes in a safe order. */
|
|
7052
|
|
7053 malloc_spin_lock(&arenas_lock);
|
|
7054 for (i = 0; i < narenas; i++) {
|
|
7055 if (arenas[i] != NULL)
|
|
7056 malloc_spin_lock(&arenas[i]->lock);
|
|
7057 }
|
|
7058 malloc_spin_unlock(&arenas_lock);
|
|
7059
|
|
7060 malloc_mutex_lock(&base_mtx);
|
|
7061
|
|
7062 malloc_mutex_lock(&huge_mtx);
|
|
7063 }
|
|
7064
|
|
7065 void
|
|
7066 _malloc_postfork(void)
|
|
7067 {
|
|
7068 unsigned i;
|
|
7069
|
|
7070 /* Release all mutexes, now that fork() has completed. */
|
|
7071
|
|
7072 malloc_mutex_unlock(&huge_mtx);
|
|
7073
|
|
7074 malloc_mutex_unlock(&base_mtx);
|
|
7075
|
|
7076 malloc_spin_lock(&arenas_lock);
|
|
7077 for (i = 0; i < narenas; i++) {
|
|
7078 if (arenas[i] != NULL)
|
|
7079 malloc_spin_unlock(&arenas[i]->lock);
|
|
7080 }
|
|
7081 malloc_spin_unlock(&arenas_lock);
|
|
7082 }
|
|
7083
|
|
7084 /*
|
|
7085 * End library-private functions.
|
|
7086 */
|
|
7087 /******************************************************************************/
|
|
7088
|
|
7089 #ifdef HAVE_LIBDL
|
|
7090 # include <dlfcn.h>
|
|
7091 #endif
|
|
7092
|
|
7093 #ifdef MOZ_MEMORY_DARWIN
|
|
7094 static malloc_zone_t zone;
|
|
7095 static struct malloc_introspection_t zone_introspect;
|
|
7096
|
|
7097 static size_t
|
|
7098 zone_size(malloc_zone_t *zone, void *ptr)
|
|
7099 {
|
|
7100
|
|
7101 /*
|
|
7102 * There appear to be places within Darwin (such as setenv(3)) that
|
|
7103 * cause calls to this function with pointers that *no* zone owns. If
|
|
7104 * we knew that all pointers were owned by *some* zone, we could split
|
|
7105 * our zone into two parts, and use one as the default allocator and
|
|
7106 * the other as the default deallocator/reallocator. Since that will
|
|
7107 * not work in practice, we must check all pointers to assure that they
|
|
7108 * reside within a mapped chunk before determining size.
|
|
7109 */
|
|
7110 return (isalloc_validate(ptr));
|
|
7111 }
|
|
7112
|
|
7113 static void *
|
|
7114 zone_malloc(malloc_zone_t *zone, size_t size)
|
|
7115 {
|
|
7116
|
|
7117 return (malloc(size));
|
|
7118 }
|
|
7119
|
|
7120 static void *
|
|
7121 zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
|
|
7122 {
|
|
7123
|
|
7124 return (calloc(num, size));
|
|
7125 }
|
|
7126
|
|
7127 static void *
|
|
7128 zone_valloc(malloc_zone_t *zone, size_t size)
|
|
7129 {
|
|
7130 void *ret = NULL; /* Assignment avoids useless compiler warning. */
|
|
7131
|
|
7132 posix_memalign(&ret, pagesize, size);
|
|
7133
|
|
7134 return (ret);
|
|
7135 }
|
|
7136
|
|
7137 static void
|
|
7138 zone_free(malloc_zone_t *zone, void *ptr)
|
|
7139 {
|
|
7140
|
|
7141 free(ptr);
|
|
7142 }
|
|
7143
|
|
7144 static void *
|
|
7145 zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
|
|
7146 {
|
|
7147
|
|
7148 return (realloc(ptr, size));
|
|
7149 }
|
|
7150
|
|
7151 static void *
|
|
7152 zone_destroy(malloc_zone_t *zone)
|
|
7153 {
|
|
7154
|
|
7155 /* This function should never be called. */
|
|
7156 assert(false);
|
|
7157 return (NULL);
|
|
7158 }
|
|
7159
|
|
7160 static size_t
|
|
7161 zone_good_size(malloc_zone_t *zone, size_t size)
|
|
7162 {
|
|
7163 size_t ret;
|
|
7164 void *p;
|
|
7165
|
|
7166 /*
|
|
7167 * Actually create an object of the appropriate size, then find out
|
|
7168 * how large it could have been without moving up to the next size
|
|
7169 * class.
|
|
7170 */
|
|
7171 p = malloc(size);
|
|
7172 if (p != NULL) {
|
|
7173 ret = isalloc(p);
|
|
7174 free(p);
|
|
7175 } else
|
|
7176 ret = size;
|
|
7177
|
|
7178 return (ret);
|
|
7179 }
|
|
7180
|
|
7181 static void
|
|
7182 zone_force_lock(malloc_zone_t *zone)
|
|
7183 {
|
|
7184
|
|
7185 _malloc_prefork();
|
|
7186 }
|
|
7187
|
|
7188 static void
|
|
7189 zone_force_unlock(malloc_zone_t *zone)
|
|
7190 {
|
|
7191
|
|
7192 _malloc_postfork();
|
|
7193 }
|
|
7194
|
|
7195 static malloc_zone_t *
|
|
7196 create_zone(void)
|
|
7197 {
|
|
7198
|
|
7199 assert(malloc_initialized);
|
|
7200
|
|
7201 zone.size = (void *)zone_size;
|
|
7202 zone.malloc = (void *)zone_malloc;
|
|
7203 zone.calloc = (void *)zone_calloc;
|
|
7204 zone.valloc = (void *)zone_valloc;
|
|
7205 zone.free = (void *)zone_free;
|
|
7206 zone.realloc = (void *)zone_realloc;
|
|
7207 zone.destroy = (void *)zone_destroy;
|
|
7208 zone.zone_name = "jemalloc_zone";
|
|
7209 zone.batch_malloc = NULL;
|
|
7210 zone.batch_free = NULL;
|
|
7211 zone.introspect = &zone_introspect;
|
|
7212
|
|
7213 zone_introspect.enumerator = NULL;
|
|
7214 zone_introspect.good_size = (void *)zone_good_size;
|
|
7215 zone_introspect.check = NULL;
|
|
7216 zone_introspect.print = NULL;
|
|
7217 zone_introspect.log = NULL;
|
|
7218 zone_introspect.force_lock = (void *)zone_force_lock;
|
|
7219 zone_introspect.force_unlock = (void *)zone_force_unlock;
|
|
7220 zone_introspect.statistics = NULL;
|
|
7221
|
|
7222 return (&zone);
|
|
7223 }
|
|
7224
|
|
7225 __attribute__((constructor))
|
|
7226 void
|
|
7227 jemalloc_darwin_init(void)
|
|
7228 {
|
|
7229 extern unsigned malloc_num_zones;
|
|
7230 extern malloc_zone_t **malloc_zones;
|
|
7231
|
|
7232 if (malloc_init_hard())
|
|
7233 abort();
|
|
7234
|
|
7235 /*
|
|
7236 * The following code is *not* thread-safe, so it's critical that
|
|
7237 * initialization be manually triggered.
|
|
7238 */
|
|
7239
|
|
7240 /* Register the custom zones. */
|
|
7241 malloc_zone_register(create_zone());
|
|
7242 assert(malloc_zones[malloc_num_zones - 1] == &zone);
|
|
7243
|
|
7244 /*
|
|
7245 * Shift malloc_zones around so that zone is first, which makes it the
|
|
7246 * default zone.
|
|
7247 */
|
|
7248 assert(malloc_num_zones > 1);
|
|
7249 memmove(&malloc_zones[1], &malloc_zones[0],
|
|
7250 sizeof(malloc_zone_t *) * (malloc_num_zones - 1));
|
|
7251 malloc_zones[0] = &zone;
|
|
7252 }
|
|
7253
|
|
7254 #elif defined(__GLIBC__) && !defined(__UCLIBC__)
|
|
7255 /*
|
|
7256 * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
|
|
7257 * to inconsistently reference libc's malloc(3)-compatible functions
|
|
7258 * (bug 493541).
|
|
7259 *
|
|
7260 * These definitions interpose hooks in glibc. The functions are actually
|
|
7261 * passed an extra argument for the caller return address, which will be
|
|
7262 * ignored.
|
|
7263 */
|
|
7264 void (*__free_hook)(void *ptr) = free;
|
|
7265 void *(*__malloc_hook)(size_t size) = malloc;
|
|
7266 void *(*__realloc_hook)(void *ptr, size_t size) = realloc;
|
|
7267 void *(*__memalign_hook)(size_t alignment, size_t size) = memalign;
|
|
7268
|
|
7269 #elif defined(RTLD_DEEPBIND)
|
|
7270 /*
|
|
7271 * XXX On systems that support RTLD_GROUP or DF_1_GROUP, do their
|
|
7272 * implementations permit similar inconsistencies? Should STV_SINGLETON
|
|
7273 * visibility be used for interposition where available?
|
|
7274 */
|
|
7275 # error "Interposing malloc is unsafe on this system without libc malloc hooks."
|
|
7276 #endif
|