On Wed, Jan 03, 2018 at 01:56:43PM +0100, Otto Moerbeek wrote:
> Hi,
>
> this is mostly kshe's work, with an additional change on the cache
> maintenance by me.
>
> I did not take the ffs change yet, since I want to measure the impact
> separately.
no feedback :-(
I'm going to commit this soon to force testing for machines running
snaps or current.
-Otto
>
> Index: malloc.c
> ===================================================================
> RCS file: /cvs/src/lib/libc/stdlib/malloc.c,v
> retrieving revision 1.238
> diff -u -p -r1.238 malloc.c
> --- malloc.c 1 Jan 2018 12:41:48 -0000 1.238
> +++ malloc.c 3 Jan 2018 12:52:14 -0000
> @@ -376,12 +376,11 @@ omalloc_parseopt(char opt)
> case 'X':
> mopts.malloc_xmalloc = 1;
> break;
> - default: {
> + default:
> dprintf(STDERR_FILENO, "malloc() warning: "
> "unknown char in MALLOC_OPTIONS\n");
> break;
> }
> - }
> }
>
> static void
> @@ -448,9 +447,6 @@ omalloc_init(void)
> ;
> }
>
> -/*
> - * Initialize a dir_info, which should have been cleared by caller
> - */
> static void
> omalloc_poolinit(struct dir_info **dp)
> {
> @@ -502,7 +498,7 @@ omalloc_grow(struct dir_info *d)
> size_t i;
> struct region_info *p;
>
> - if (d->regions_total > SIZE_MAX / sizeof(struct region_info) / 2 )
> + if (d->regions_total > SIZE_MAX / sizeof(struct region_info) / 2)
> return 1;
>
> newtotal = d->regions_total * 2;
> @@ -641,9 +637,9 @@ static void
> unmap(struct dir_info *d, void *p, size_t sz, int clear)
> {
> size_t psz = sz >> MALLOC_PAGESHIFT;
> - size_t rsz, tounmap;
> + size_t rsz;
> struct region_info *r;
> - u_int i, offset;
> + u_int i, offset, mask;
>
> if (sz != PAGEROUND(sz))
> wrterror(d, "munmap round");
> @@ -662,30 +658,34 @@ unmap(struct dir_info *d, void *p, size_
> STATS_SUB(d->malloc_used, sz);
> return;
> }
> - tounmap = 0;
> - if (psz > rsz)
> - tounmap = psz - rsz;
> offset = getrbyte(d);
> - for (i = 0; tounmap > 0 && i < mopts.malloc_cache; i++) {
> - r = &d->free_regions[(i + offset) & (mopts.malloc_cache - 1)];
> - if (r->p != NULL) {
> - rsz = r->size << MALLOC_PAGESHIFT;
> - if (munmap(r->p, rsz))
> - wrterror(d, "munmap %p", r->p);
> - r->p = NULL;
> - if (tounmap > r->size)
> - tounmap -= r->size;
> - else
> - tounmap = 0;
> - d->free_regions_size -= r->size;
> - r->size = 0;
> - STATS_SUB(d->malloc_used, rsz);
> + mask = mopts.malloc_cache - 1;
> + if (psz > rsz) {
> + size_t tounmap = psz - rsz;
> + i = 0;
> + for (;;) {
> + r = &d->free_regions[(i + offset) & mask];
> + if (r->p != NULL) {
> + rsz = r->size << MALLOC_PAGESHIFT;
> + if (munmap(r->p, rsz))
> + wrterror(d, "munmap %p", r->p);
> + r->p = NULL;
> + if (tounmap > r->size)
> + tounmap -= r->size;
> + else
> + tounmap = 0;
> + d->free_regions_size -= r->size;
> + STATS_SUB(d->malloc_used, rsz);
> + if (tounmap == 0) {
> + offset = i;
> + break;
> + }
> + }
> + i++;
> }
> }
> - if (tounmap > 0)
> - wrterror(d, "malloc cache underflow");
> - for (i = 0; i < mopts.malloc_cache; i++) {
> - r = &d->free_regions[(i + offset) & (mopts.malloc_cache - 1)];
> + for (i = 0; ; i++) {
> + r = &d->free_regions[(i + offset) & mask];
> if (r->p == NULL) {
> if (clear)
> memset(p, 0, sz - mopts.malloc_guard);
> @@ -702,8 +702,6 @@ unmap(struct dir_info *d, void *p, size_
> break;
> }
> }
> - if (i == mopts.malloc_cache)
> - wrterror(d, "malloc free slot lost");
> if (d->free_regions_size > mopts.malloc_cache)
> wrterror(d, "malloc cache overflow");
> }
> @@ -723,7 +721,6 @@ zapcacheregion(struct dir_info *d, void
> wrterror(d, "munmap %p", r->p);
> r->p = NULL;
> d->free_regions_size -= r->size;
> - r->size = 0;
> STATS_SUB(d->malloc_used, rsz);
> }
> }
> @@ -760,7 +757,6 @@ map(struct dir_info *d, void *hint, size
> if (r->size == psz) {
> p = r->p;
> r->p = NULL;
> - r->size = 0;
> d->free_regions_size -= psz;
> if (mopts.malloc_freeunmap)
> mprotect(p, sz, PROT_READ | PROT_WRITE);
> @@ -808,10 +804,7 @@ init_chunk_info(struct dir_info *d, stru
> int i;
>
> if (bits == 0) {
> - p->shift = 1;
> - i = MALLOC_MINSIZE - 1;
> - while (i >>= 1)
> - p->shift++;
> + p->shift = MALLOC_MINSHIFT;
> p->total = p->free = MALLOC_PAGESIZE >> p->shift;
> p->size = 0;
> p->offset = 0xdead;
> @@ -824,13 +817,9 @@ init_chunk_info(struct dir_info *d, stru
> p->canary = (u_short)d->canary1;
>
> /* set all valid bits in the bitmap */
> - for (i = 0; p->total - i >= MALLOC_BITS; i += MALLOC_BITS)
> - p->bits[i / MALLOC_BITS] = (u_short)~0U;
> -
> - if (i < p->total)
> - p->bits[i / MALLOC_BITS] = 0;
> - for (; i < p->total; i++)
> - p->bits[i / MALLOC_BITS] |= (u_short)1U << (i % MALLOC_BITS);
> + i = p->total - 1;
> + memset(p->bits, 0xff, sizeof(p->bits[0]) * (i / MALLOC_BITS));
> + p->bits[i / MALLOC_BITS] = (2U << (i % MALLOC_BITS)) - 1;
> }
>
> static struct chunk_info *
> @@ -907,23 +896,20 @@ err:
> static int
> find_chunksize(size_t size)
> {
> - int i, j;
> + int r;
> +
> + /* malloc(0) is special */
> + if (size == 0)
> + return 0;
>
> - /* Don't bother with anything less than this */
> - /* unless we have a malloc(0) requests */
> - if (size != 0 && size < MALLOC_MINSIZE)
> + if (size < MALLOC_MINSIZE)
> size = MALLOC_MINSIZE;
> + size--;
>
> - /* Find the right bucket */
> - if (size == 0)
> - j = 0;
> - else {
> - j = MALLOC_MINSHIFT;
> - i = (size - 1) >> (MALLOC_MINSHIFT - 1);
> - while (i >>= 1)
> - j++;
> - }
> - return j;
> + r = MALLOC_MINSHIFT;
> + while (size >> r)
> + r++;
> + return r;
> }
>
> static void
> @@ -1013,7 +999,7 @@ found:
> *lp ^= u;
>
> /* If there are no more free, remove from free-list */
> - if (!--bp->free)
> + if (--bp->free == 0)
> LIST_REMOVE(bp, entries);
>
> /* Adjust to the real offset of that chunk */
> @@ -1048,8 +1034,8 @@ validate_canary(struct dir_info *d, u_ch
> while (p < q) {
> if (*p != SOME_JUNK) {
> wrterror(d, "chunk canary corrupted %p %#tx@%#zx%s",
> - ptr, p - ptr, sz, *p == SOME_FREEJUNK ?
> - " (double free?)" : "");
> + ptr, p - ptr, sz,
> + *p == SOME_FREEJUNK ? " (double free?)" : "");
> }
> p++;
> }
> @@ -1173,8 +1159,7 @@ omalloc(struct dir_info *pool, size_t sz
> else
> memset(p, SOME_JUNK,
> psz - mopts.malloc_guard);
> - }
> - else if (mopts.chunk_canaries)
> + } else if (mopts.chunk_canaries)
> fill_canary(p, sz - mopts.malloc_guard,
> psz - mopts.malloc_guard);
> }
> @@ -1225,7 +1210,7 @@ _malloc_init(int from_rthreads)
> max = from_rthreads ? _MALLOC_MUTEXES : 1;
> if (((uintptr_t)&malloc_readonly & MALLOC_PAGEMASK) == 0)
> mprotect(&malloc_readonly, sizeof(malloc_readonly),
> - PROT_READ | PROT_WRITE);
> + PROT_READ | PROT_WRITE);
> for (i = 0; i < max; i++) {
> if (mopts.malloc_pool[i])
> continue;
> @@ -2030,8 +2015,7 @@ omemalign(struct dir_info *pool, size_t
> SOME_JUNK, psz - sz);
> else
> memset(p, SOME_JUNK, psz - mopts.malloc_guard);
> - }
> - else if (mopts.chunk_canaries)
> + } else if (mopts.chunk_canaries)
> fill_canary(p, sz - mopts.malloc_guard,
> psz - mopts.malloc_guard);
>
