From e72b84cec44852dd76365cb7e1bf691b56a8adfc Mon Sep 17 00:00:00 2001 From: Alexey Makhalov <amakhalov@vmware.com> Date: Tue, 29 Aug 2017 21:10:08 +0000 Subject: [PATCH 2/2] malloc arena fix --- elf/dl-tunables.list | 5 ++++ malloc/arena.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++ malloc/malloc.c | 31 +++++++++++++++++++++++++ malloc/malloc.h | 1 + 4 files changed, 101 insertions(+) diff --git a/elf/dl-tunables.list b/elf/dl-tunables.list index c188c6a..15a1a14 100644 --- a/elf/dl-tunables.list +++ b/elf/dl-tunables.list @@ -76,6 +76,11 @@ glibc { minval: 1 security_level: SXID_IGNORE } + arena_stickiness { + type: SIZE_T + env_alias: MALLOC_ARENA_STICKINESS + security_level: SXID_IGNORE + } tcache_max { type: SIZE_T } diff --git a/malloc/arena.c b/malloc/arena.c index dc14fae..f0edf2b 100644 --- a/malloc/arena.c +++ b/malloc/arena.c @@ -63,6 +63,12 @@ typedef struct _heap_info char pad[-6 * SIZE_SZ & MALLOC_ALIGN_MASK]; } heap_info; +typedef struct _arena_tracker +{ + mstate arena; /* Arena most recently tracked for growth. */ + size_t growth; /* Current size in bytes. */ +} arena_tracker; + /* Get a compile-time error if the heap_info padding is not correct to make alignment work as expected in sYSMALLOc. */ extern int sanity_check_heap_info_alignment[(sizeof (heap_info) @@ -73,6 +79,8 @@ extern int sanity_check_heap_info_alignment[(sizeof (heap_info) static __thread mstate thread_arena attribute_tls_model_ie; +static __thread arena_tracker thread_arena_tracker attribute_tls_model_ie; + /* Arena free list. free_list_lock synchronizes access to the free_list variable below, and the next_free and attached_threads members of struct malloc_state objects. No other locks must be @@ -236,6 +244,7 @@ TUNABLE_CALLBACK_FNDECL (set_perturb_byte, int32_t) TUNABLE_CALLBACK_FNDECL (set_trim_threshold, size_t) TUNABLE_CALLBACK_FNDECL (set_arena_max, size_t) TUNABLE_CALLBACK_FNDECL (set_arena_test, size_t) +TUNABLE_CALLBACK_FNDECL (set_arena_stickiness, size_t) #if USE_TCACHE TUNABLE_CALLBACK_FNDECL (set_tcache_max, size_t) TUNABLE_CALLBACK_FNDECL (set_tcache_count, size_t) @@ -327,6 +336,7 @@ ptmalloc_init (void) TUNABLE_GET (mmap_max, int32_t, TUNABLE_CALLBACK (set_mmaps_max)); TUNABLE_GET (arena_max, size_t, TUNABLE_CALLBACK (set_arena_max)); TUNABLE_GET (arena_test, size_t, TUNABLE_CALLBACK (set_arena_test)); + TUNABLE_GET (arena_stickiness, size_t, TUNABLE_CALLBACK (set_arena_stickiness)); # if USE_TCACHE TUNABLE_GET (tcache_max, size_t, TUNABLE_CALLBACK (set_tcache_max)); TUNABLE_GET (tcache_count, size_t, TUNABLE_CALLBACK (set_tcache_count)); @@ -392,6 +402,13 @@ ptmalloc_init (void) __libc_mallopt (M_MMAP_THRESHOLD, atoi (&envline[16])); } break; + case 16: + if (!__builtin_expect (__libc_enable_secure, 0)) + { + if (memcmp (envline, "ARENA_STICKINESS", 16) == 0) + __libc_mallopt (M_ARENA_STICKINESS, atoi (&envline[17])); + } + break; default: break; } @@ -974,6 +991,51 @@ arena_get_retry (mstate ar_ptr, size_t bytes) return ar_ptr; } +static void +arena_stickiness_track_alloc (void *victim) +{ + if (!victim || chunk_is_mmapped (mem2chunk (victim))) + return; + + if (thread_arena_tracker.arena != arena_for_chunk (mem2chunk (victim))) { + thread_arena_tracker.growth = 0; + thread_arena_tracker.arena = arena_for_chunk (mem2chunk (victim)); + } else { + thread_arena_tracker.growth += chunksize (mem2chunk (victim)); + if (thread_arena_tracker.growth >= mp_.arena_stickiness) { + /* Swtich thread to the next arena */ + mstate replaced_arena = thread_arena; + mstate next_to_use = replaced_arena->next; + + __libc_lock_lock (free_list_lock); + detach_arena (replaced_arena); +#if 0 + /* If this was the last attached thread for this arena, put the + arena on the free list. */ + if (replaced_arena->attached_threads == 0) + { + replaced_arena->next_free = free_list; + free_list = replaced_arena; + } +#endif + if (next_to_use->attached_threads == 0) + remove_from_free_list (next_to_use); + ++next_to_use->attached_threads; + + __libc_lock_unlock (free_list_lock); + thread_arena = next_to_use; + } + } +} + +/* chunk must be valid and not mmaped. */ +static void +arena_stickiness_track_free (mchunkptr chunk) +{ + if (thread_arena_tracker.arena == arena_for_chunk (chunk)) + thread_arena_tracker.growth -= chunksize (chunk); +} + void __malloc_arena_thread_freeres (void) { diff --git a/malloc/malloc.c b/malloc/malloc.c index 54e406b..29787a5 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -1723,6 +1723,7 @@ struct malloc_par INTERNAL_SIZE_T mmap_threshold; INTERNAL_SIZE_T arena_test; INTERNAL_SIZE_T arena_max; + INTERNAL_SIZE_T arena_stickiness; /* Memory map support */ int n_mmaps; @@ -1787,6 +1788,7 @@ static struct malloc_par mp_ = .mmap_threshold = DEFAULT_MMAP_THRESHOLD, .trim_threshold = DEFAULT_TRIM_THRESHOLD, #define NARENAS_FROM_NCORES(n) ((n) * (sizeof (long) == 4 ? 2 : 8)) + .arena_stickiness = 0, .arena_test = NARENAS_FROM_NCORES (1) #if USE_TCACHE , @@ -3083,6 +3085,10 @@ __libc_malloc (size_t bytes) assert (!victim || chunk_is_mmapped (mem2chunk (victim)) || ar_ptr == arena_for_chunk (mem2chunk (victim))); + + if (mp_.arena_stickiness > 0) + arena_stickiness_track_alloc (victim); + return victim; } libc_hidden_def (__libc_malloc) @@ -3126,6 +3132,9 @@ __libc_free (void *mem) MAYBE_INIT_TCACHE (); + if (mp_.arena_stickiness > 0) + arena_stickiness_track_free (p); + ar_ptr = arena_for_chunk (p); _int_free (ar_ptr, p, 0); } @@ -3226,6 +3235,8 @@ __libc_realloc (void *oldmem, size_t bytes) return newp; } + if (mp_.arena_stickiness > 0) + arena_stickiness_track_free (oldp); __libc_lock_lock (ar_ptr->mutex); newp = _int_realloc (ar_ptr, oldp, oldsize, nb); @@ -3234,6 +3245,9 @@ __libc_realloc (void *oldmem, size_t bytes) assert (!newp || chunk_is_mmapped (mem2chunk (newp)) || ar_ptr == arena_for_chunk (mem2chunk (newp))); + if (mp_.arena_stickiness > 0) + arena_stickiness_track_alloc (newp); + if (newp == NULL) { /* Try harder to allocate memory in other arenas. */ @@ -3452,6 +3466,9 @@ __libc_calloc (size_t n, size_t elem_size) return mem; } + if (mp_.arena_stickiness > 0) + arena_stickiness_track_alloc (mem); + csz = chunksize (p); #if MORECORE_CLEARS @@ -5145,6 +5162,15 @@ do_set_arena_max (size_t value) return 1; } +static inline int +__always_inline +do_set_arena_stickiness (size_t value) +{ + LIBC_PROBE (memory_mallopt_arena_stickiness, 2, value, mp_.arena_stickiness); + mp_.arena_stickiness = value; + return 1; +} + #if USE_TCACHE static inline int __always_inline @@ -5237,6 +5263,11 @@ __libc_mallopt (int param_number, int value) if (value > 0) do_set_arena_max (value); break; + + case M_ARENA_STICKINESS: + if (value > 0) + do_set_arena_stickiness (value); + break; } __libc_lock_unlock (av->mutex); return res; diff --git a/malloc/malloc.h b/malloc/malloc.h index 339ab64..31bdb44 100644 --- a/malloc/malloc.h +++ b/malloc/malloc.h @@ -121,6 +121,7 @@ extern struct mallinfo mallinfo (void) __THROW; #define M_PERTURB -6 #define M_ARENA_TEST -7 #define M_ARENA_MAX -8 +#define M_ARENA_STICKINESS -9 /* General SVID/XPG interface to tunable parameters. */ extern int mallopt (int __param, int __val) __THROW; -- 2.9.3