/************** Debugging ***************************/
//debug = COLLECT_PRINTF; // turn on printf's
-//debug = MEMSTOMP; // stomp on memory
-//debug = SENTINEL; // add underrun/overrrun protection
//debug = PTRCHECK; // more pointer checking
//debug = PTRCHECK2; // thorough but slow pointer checking
/***************************************************/
import rt.gc.cdgc.bits: GCBits;
-import rt.gc.cdgc.stats: GCStats;
-import alloc = rt.gc.cdgc.alloc;
+import rt.gc.cdgc.stats: GCStats, Stats;
import rt.gc.cdgc.dynarray: DynArray;
+import alloc = rt.gc.cdgc.alloc;
+import opts = rt.gc.cdgc.opts;
import cstdlib = tango.stdc.stdlib;
import cstring = tango.stdc.string;
static import gcc.builtins; // for __builtin_unwind_int
}
-
struct BlkInfo
{
void* base;
uint attr;
}
+package enum BlkAttr : uint
+{
+ FINALIZE = 0b0000_0001,
+ NO_SCAN = 0b0000_0010,
+ NO_MOVE = 0b0000_0100,
+ ALL_BITS = 0b1111_1111
+}
+
private
{
- enum BlkAttr : uint
- {
- FINALIZE = 0b0000_0001,
- NO_SCAN = 0b0000_0010,
- NO_MOVE = 0b0000_0100,
- ALL_BITS = 0b1111_1111
- }
extern (C) void* rt_stackBottom();
extern (C) void* rt_stackTop();
const uint GCVERSION = 1; // increment every time we change interface
// to GC.
+Stats stats;
+
class GC
{
// For passing to debug code
void initialize()
{
+ opts.parse(cstdlib.getenv("D_GC_OPTS"));
gcLock = GCLock.classinfo;
gcx = cast(Gcx*) cstdlib.calloc(1, Gcx.sizeof);
if (!gcx)
onOutOfMemoryError();
gcx.initialize();
setStackBottom(rt_stackBottom());
+ stats = Stats(this);
}
uint go()
{
Pool* pool = gcx.findPool(p);
- uint oldb = 0;
+ uint old_attrs = 0;
if (pool)
{
- auto biti = cast(size_t)(p - pool.baseAddr) / 16;
+ auto bit_i = cast(size_t)(p - pool.baseAddr) / 16;
- oldb = gcx.getBits(pool, biti);
+ old_attrs = gcx.getAttr(pool, bit_i);
}
- return oldb;
+ return old_attrs;
}
if (!thread_needLock())
uint go()
{
Pool* pool = gcx.findPool(p);
- uint oldb = 0;
+ uint old_attrs = 0;
if (pool)
{
- auto biti = cast(size_t)(p - pool.baseAddr) / 16;
+ auto bit_i = cast(size_t)(p - pool.baseAddr) / 16;
- oldb = gcx.getBits(pool, biti);
- gcx.setBits(pool, biti, mask);
+ old_attrs = gcx.getAttr(pool, bit_i);
+ gcx.setAttr(pool, bit_i, mask);
}
- return oldb;
+ return old_attrs;
}
if (!thread_needLock())
uint go()
{
Pool* pool = gcx.findPool(p);
- uint oldb = 0;
+ uint old_attrs = 0;
if (pool)
{
- auto biti = cast(size_t)(p - pool.baseAddr) / 16;
+ auto bit_i = cast(size_t)(p - pool.baseAddr) / 16;
- oldb = gcx.getBits(pool, biti);
- gcx.clrBits(pool, biti, mask);
+ old_attrs = gcx.getAttr(pool, bit_i);
+ gcx.clrAttr(pool, bit_i, mask);
}
- return oldb;
+ return old_attrs;
}
if (!thread_needLock())
/**
*
*/
- void *malloc(size_t size, uint bits = 0)
+ void *malloc(size_t size, uint attrs, PointerMap ptrmap)
{
if (!size)
{
if (!thread_needLock())
{
- return mallocNoSync(size, bits);
+ return mallocNoSync(size, attrs, ptrmap.bits.ptr);
}
else synchronized (gcLock)
{
- return mallocNoSync(size, bits);
+ return mallocNoSync(size, attrs, ptrmap.bits.ptr);
}
}
//
//
//
- private void *mallocNoSync(size_t size, uint bits = 0)
+ private void *mallocNoSync(size_t size, uint attrs, size_t* pm_bitmask)
{
assert(size != 0);
+ stats.malloc_started(size, attrs, pm_bitmask);
+ scope (exit)
+ stats.malloc_finished(p);
+
void *p = null;
Bins bin;
assert(gcx);
- size += SENTINEL_EXTRA;
+ if (opts.options.sentinel)
+ size += SENTINEL_EXTRA;
+
+ bool has_pm = !(attrs & BlkAttr.NO_SCAN);
+ size_t pm_bitmask_size;
+ if (has_pm)
+ pm_bitmask_size = (size_t*).sizeof;
+ size += pm_bitmask_size;
// Compute size bin
// Cache previous binsize lookup - Dave Fladebo.
lastbin = bin;
}
+ size_t capacity; // to figure out where to store the bitmask
if (bin < B_PAGE)
{
p = gcx.bucket[bin];
}
p = gcx.bucket[bin];
}
+ capacity = binsize[bin];
// Return next item from free list
gcx.bucket[bin] = (cast(List*)p).next;
- if( !(bits & BlkAttr.NO_SCAN) )
- memset(p + size, 0, binsize[bin] - size);
- debug (MEMSTOMP) memset(p, 0xF0, size);
+ if( !(attrs & BlkAttr.NO_SCAN) )
+ memset(p + size, 0, capacity - size);
+ if (opts.options.mem_stomp)
+ memset(p, 0xF0, size);
}
else
{
p = gcx.bigAlloc(size);
if (!p)
onOutOfMemoryError();
+ // Round the size up to the number of pages needed to store it
+ size_t npages = (size + PAGESIZE - 1) / PAGESIZE;
+ capacity = npages * PAGESIZE;
+ }
+
+ // Store the bit mask AFTER SENTINEL_POST
+ // TODO: store it BEFORE, so the bitmask is protected too
+ if (has_pm) {
+ auto end_of_blk = cast(size_t**)(p + capacity - pm_bitmask_size);
+ *end_of_blk = pm_bitmask;
+ size -= pm_bitmask_size;
}
- size -= SENTINEL_EXTRA;
- p = sentinel_add(p);
- sentinel_init(p, size);
- if (bits)
+ if (opts.options.sentinel) {
+ size -= SENTINEL_EXTRA;
+ p = sentinel_add(p);
+ sentinel_init(p, size);
+ }
+
+ if (attrs)
{
Pool *pool = gcx.findPool(p);
assert(pool);
- gcx.setBits(pool, cast(size_t)(p - pool.baseAddr) / 16, bits);
+ gcx.setAttr(pool, cast(size_t)(p - pool.baseAddr) / 16, attrs);
}
return p;
}
/**
*
*/
- void *calloc(size_t size, uint bits = 0)
+ void *calloc(size_t size, uint attrs, PointerMap ptrmap)
{
if (!size)
{
if (!thread_needLock())
{
- return callocNoSync(size, bits);
+ return callocNoSync(size, attrs, ptrmap.bits.ptr);
}
else synchronized (gcLock)
{
- return callocNoSync(size, bits);
+ return callocNoSync(size, attrs, ptrmap.bits.ptr);
}
}
//
//
//
- private void *callocNoSync(size_t size, uint bits = 0)
+ private void *callocNoSync(size_t size, uint attrs, size_t* pm_bitmask)
{
assert(size != 0);
- void *p = mallocNoSync(size, bits);
+ void *p = mallocNoSync(size, attrs, pm_bitmask);
memset(p, 0, size);
return p;
}
/**
*
*/
- void *realloc(void *p, size_t size, uint bits = 0)
+ void *realloc(void *p, size_t size, uint attrs, PointerMap ptrmap)
{
if (!thread_needLock())
{
- return reallocNoSync(p, size, bits);
+ return reallocNoSync(p, size, attrs, ptrmap.bits.ptr);
}
else synchronized (gcLock)
{
- return reallocNoSync(p, size, bits);
+ return reallocNoSync(p, size, attrs, ptrmap.bits.ptr);
}
}
//
//
//
- private void *reallocNoSync(void *p, size_t size, uint bits = 0)
+ private void *reallocNoSync(void *p, size_t size, uint attrs,
+ size_t* pm_bitmask)
{
if (!size)
{
}
else if (!p)
{
- p = mallocNoSync(size, bits);
+ p = mallocNoSync(size, attrs, pm_bitmask);
}
else
{
- void *p2;
- size_t psize;
+ Pool* pool = gcx.findPool(p);
+ if (pool is null)
+ return null;
- version (SENTINEL)
+ // Set or retrieve attributes as appropriate
+ auto bit_i = cast(size_t)(p - pool.baseAddr) / 16;
+ if (attrs) {
+ gcx.clrAttr(pool, bit_i, BlkAttr.ALL_BITS);
+ gcx.setAttr(pool, bit_i, attrs);
+ }
+ else
+ attrs = gcx.getAttr(pool, bit_i);
+
+ void* blk_base_addr = gcx.findBase(p);
+ size_t blk_size = gcx.findSize(p);
+ bool has_pm = !(attrs & BlkAttr.NO_SCAN);
+ size_t pm_bitmask_size = 0;
+ if (has_pm) {
+ pm_bitmask_size = (size_t*).sizeof;
+ // Retrieve pointer map bit mask if appropriate
+ if (pm_bitmask is null) {
+ auto end_of_blk = cast(size_t**)(blk_base_addr +
+ blk_size - pm_bitmask_size);
+ pm_bitmask = *end_of_blk;
+ }
+ }
+
+ if (opts.options.sentinel)
{
sentinel_Invariant(p);
- psize = *sentinel_size(p);
- if (psize != size)
+ size_t sentinel_stored_size = *sentinel_size(p);
+ if (sentinel_stored_size != size)
{
- if (psize)
- {
- Pool *pool = gcx.findPool(p);
-
- if (pool)
- {
- auto biti = cast(size_t)(p - pool.baseAddr) / 16;
-
- if (bits)
- {
- gcx.clrBits(pool, biti, BlkAttr.ALL_BITS);
- gcx.setBits(pool, biti, bits);
- }
- else
- {
- bits = gcx.getBits(pool, biti);
- }
- }
- }
- p2 = mallocNoSync(size, bits);
- if (psize < size)
- size = psize;
+ void* p2 = mallocNoSync(size, attrs, pm_bitmask);
+ if (sentinel_stored_size < size)
+ size = sentinel_stored_size;
cstring.memcpy(p2, p, size);
p = p2;
}
}
else
{
- psize = gcx.findSize(p); // find allocated size
- if (psize >= PAGESIZE && size >= PAGESIZE)
+ size += pm_bitmask_size;
+ if (blk_size >= PAGESIZE && size >= PAGESIZE)
{
- auto psz = psize / PAGESIZE;
+ auto psz = blk_size / PAGESIZE;
auto newsz = (size + PAGESIZE - 1) / PAGESIZE;
if (newsz == psz)
return p;
- auto pool = gcx.findPool(p);
auto pagenum = (p - pool.baseAddr) / PAGESIZE;
if (newsz < psz)
// Shrink in place
synchronized (gcLock)
{
- debug (MEMSTOMP)
- memset(p + size, 0xF2, psize - size);
+ if (opts.options.mem_stomp)
+ memset(p + size - pm_bitmask_size, 0xF2,
+ blk_size - size - pm_bitmask_size);
pool.freePages(pagenum + newsz, psz - newsz);
}
+ if (has_pm) {
+ auto end_of_blk = cast(size_t**)(
+ blk_base_addr + (PAGESIZE * newsz) -
+ pm_bitmask_size);
+ *end_of_blk = pm_bitmask;
+ }
return p;
}
else if (pagenum + newsz <= pool.npages)
{
if (i == pagenum + newsz)
{
- debug (MEMSTOMP)
- memset(p + psize, 0xF0,
- size - psize);
+ if (opts.options.mem_stomp)
+ memset(p + blk_size - pm_bitmask_size,
+ 0xF0, size - blk_size
+ - pm_bitmask_size);
memset(pool.pagetable + pagenum +
psz, B_PAGEPLUS, newsz - psz);
+ if (has_pm) {
+ auto end_of_blk = cast(size_t**)(
+ blk_base_addr +
+ (PAGESIZE * newsz) -
+ pm_bitmask_size);
+ *end_of_blk = pm_bitmask;
+ }
return p;
}
if (i == pool.npages)
}
}
}
- if (psize < size || // if new size is bigger
- psize > size * 2) // or less than half
+ // if new size is bigger or less than half
+ if (blk_size < size || blk_size > size * 2)
{
- if (psize)
- {
- Pool *pool = gcx.findPool(p);
-
- if (pool)
- {
- auto biti = cast(size_t)(p - pool.baseAddr) / 16;
-
- if (bits)
- {
- gcx.clrBits(pool, biti, BlkAttr.ALL_BITS);
- gcx.setBits(pool, biti, bits);
- }
- else
- {
- bits = gcx.getBits(pool, biti);
- }
- }
- }
- p2 = mallocNoSync(size, bits);
- if (psize < size)
- size = psize;
+ size -= pm_bitmask_size;
+ blk_size -= pm_bitmask_size;
+ void* p2 = mallocNoSync(size, attrs, pm_bitmask);
+ if (blk_size < size)
+ size = blk_size;
cstring.memcpy(p2, p, size);
p = p2;
}
}
body
{
- version (SENTINEL)
- {
+ if (opts.options.sentinel)
return 0;
+
+ Pool* pool = gcx.findPool(p);
+ if (pool is null)
+ return 0;
+
+ // Retrieve attributes
+ auto bit_i = cast(size_t)(p - pool.baseAddr) / 16;
+ uint attrs = gcx.getAttr(pool, bit_i);
+
+ void* blk_base_addr = gcx.findBase(p);
+ size_t blk_size = gcx.findSize(p);
+ bool has_pm = !(attrs & BlkAttr.NO_SCAN);
+ size_t* pm_bitmask = null;
+ size_t pm_bitmask_size = 0;
+ if (has_pm) {
+ pm_bitmask_size = (size_t*).sizeof;
+ // Retrieve pointer map bit mask
+ auto end_of_blk = cast(size_t**)(blk_base_addr +
+ blk_size - pm_bitmask_size);
+ pm_bitmask = *end_of_blk;
}
- auto psize = gcx.findSize(p); // find allocated size
- if (psize < PAGESIZE)
- return 0; // cannot extend buckets
- auto psz = psize / PAGESIZE;
+ if (blk_size < PAGESIZE)
+ return 0; // cannot extend buckets
+
+ minsize += pm_bitmask_size;
+ maxsize += pm_bitmask_size;
+
+ auto psz = blk_size / PAGESIZE;
auto minsz = (minsize + PAGESIZE - 1) / PAGESIZE;
auto maxsz = (maxsize + PAGESIZE - 1) / PAGESIZE;
- auto pool = gcx.findPool(p);
auto pagenum = (p - pool.baseAddr) / PAGESIZE;
size_t sz;
}
if (sz < minsz)
return 0;
- debug (MEMSTOMP)
- memset(p + psize, 0xF0, (psz + sz) * PAGESIZE - psize);
+
+ size_t new_size = (psz + sz) * PAGESIZE;
+
+ if (opts.options.mem_stomp)
+ memset(p + blk_size - pm_bitmask_size, 0xF0,
+ new_size - blk_size - pm_bitmask_size);
memset(pool.pagetable + pagenum + psz, B_PAGEPLUS, sz);
gcx.p_cache = null;
gcx.size_cache = 0;
- return (psz + sz) * PAGESIZE;
+
+ if (has_pm) {
+ new_size -= pm_bitmask_size;
+ auto end_of_blk = cast(size_t**)(blk_base_addr + new_size);
+ *end_of_blk = pm_bitmask;
+ }
+ return new_size;
}
Pool* pool;
size_t pagenum;
Bins bin;
- size_t biti;
+ size_t bit_i;
// Find which page it is in
pool = gcx.findPool(p);
if (!pool) // if not one of ours
return; // ignore
- sentinel_Invariant(p);
- p = sentinel_sub(p);
+ if (opts.options.sentinel) {
+ sentinel_Invariant(p);
+ p = sentinel_sub(p);
+ }
pagenum = cast(size_t)(p - pool.baseAddr) / PAGESIZE;
- biti = cast(size_t)(p - pool.baseAddr) / 16;
- gcx.clrBits(pool, biti, BlkAttr.ALL_BITS);
+ bit_i = cast(size_t)(p - pool.baseAddr) / 16;
+ gcx.clrAttr(pool, bit_i, BlkAttr.ALL_BITS);
bin = cast(Bins)pool.pagetable[pagenum];
if (bin == B_PAGE) // if large alloc
size_t n = pagenum;
while (++n < pool.npages && pool.pagetable[n] == B_PAGEPLUS)
npages++;
- debug (MEMSTOMP) memset(p, 0xF2, npages * PAGESIZE);
+ if (opts.options.mem_stomp)
+ memset(p, 0xF2, npages * PAGESIZE);
pool.freePages(pagenum, npages);
}
else
// Add to free list
List *list = cast(List*)p;
- debug (MEMSTOMP) memset(p, 0xF2, binsize[bin]);
+ if (opts.options.mem_stomp)
+ memset(p, 0xF2, binsize[bin]);
list.next = gcx.bucket[bin];
gcx.bucket[bin] = list;
{
assert (p);
- version (SENTINEL)
- {
+ if (opts.options.sentinel)
p = sentinel_sub(p);
- size_t size = gcx.findSize(p);
+ Pool* pool = gcx.findPool(p);
+ if (pool is null)
+ return 0;
+
+ auto biti = cast(size_t)(p - pool.baseAddr) / 16;
+ uint attrs = gcx.getAttr(pool, biti);
+
+ size_t size = gcx.findSize(p);
+ bool has_pm = !(attrs & BlkAttr.NO_SCAN);
+ size_t pm_bitmask_size = 0;
+ if (has_pm)
+ pm_bitmask_size = (size_t*).sizeof;
+
+ if (opts.options.sentinel) {
// Check for interior pointer
// This depends on:
// 1) size is a power of 2 for less than PAGESIZE values
// 2) base of memory pool is aligned on PAGESIZE boundary
if (cast(size_t)p & (size - 1) & (PAGESIZE - 1))
- size = 0;
- return size ? size - SENTINEL_EXTRA : 0;
+ return 0;
+ return size - SENTINEL_EXTRA - pm_bitmask_size;
}
- else
- {
+ else {
if (p == gcx.p_cache)
return gcx.size_cache;
- size_t size = gcx.findSize(p);
-
// Check for interior pointer
// This depends on:
// 1) size is a power of 2 for less than PAGESIZE values
// 2) base of memory pool is aligned on PAGESIZE boundary
if (cast(size_t)p & (size - 1) & (PAGESIZE - 1))
- size = 0;
- else
- {
- gcx.p_cache = p;
- gcx.size_cache = size;
- }
+ return 0;
- return size;
+ gcx.p_cache = p;
+ gcx.size_cache = size - pm_bitmask_size;
+
+ return gcx.size_cache;
}
}
{
assert(p);
- sentinel_Invariant(p);
+ if (opts.options.sentinel)
+ sentinel_Invariant(p);
debug (PTRCHECK)
{
Pool* pool;
Bins bin;
size_t size;
- p = sentinel_sub(p);
+ if (opts.options.sentinel)
+ p = sentinel_sub(p);
pool = gcx.findPool(p);
assert(pool);
pagenum = cast(size_t)(p - pool.baseAddr) / PAGESIZE;
}
////////////////////////////////////////////////////////////////////
- // getBits
+ // getAttr
////////////////////////////////////////////////////////////////////
- info.attr = getBits(pool, cast(size_t)(offset / 16));
+ info.attr = getAttr(pool, cast(size_t)(offset / 16));
+ if (!(info.attr & BlkAttr.NO_SCAN))
+ info.size -= (size_t*).sizeof; // bitmask
}
return info;
}
memset(&pool.pagetable[pn + 1], B_PAGEPLUS, npages - 1);
p = pool.baseAddr + pn * PAGESIZE;
memset(cast(char *)p + size, 0, npages * PAGESIZE - size);
- debug (MEMSTOMP) memset(p, 0xF1, size);
+ if (opts.options.mem_stomp)
+ memset(p, 0xF1, size);
return p;
Lnomemory:
}
+ /**
+ * Marks a range of memory using the conservative bit mask. Used for
+ * the stack, for the data segment, and additional memory ranges.
+ */
+ void mark_conservative(void* pbot, void* ptop)
+ {
+ mark(pbot, ptop, PointerMap.init.bits.ptr);
+ }
+
+
/**
* Search a range of memory values and mark any pointers into the GC pool.
*/
- void mark(void *pbot, void *ptop)
+ void mark(void *pbot, void *ptop, size_t* pm_bitmask)
{
+ const BITS_PER_WORD = size_t.sizeof * 8;
+
void **p1 = cast(void **)pbot;
void **p2 = cast(void **)ptop;
size_t pcache = 0;
uint changes = 0;
+ // TODO: add option to be conservative
+ // force conservative scanning
+ //pm_bitmask = PointerMap.init.bits.ptr;
+
+ size_t type_size = pm_bitmask[0];
+ size_t* pm_bits = pm_bitmask + 1;
+
//printf("marking range: %p -> %p\n", pbot, ptop);
- for (; p1 < p2; p1++)
- {
- Pool *pool;
- byte *p = cast(byte *)(*p1);
+ for (; p1 + type_size <= p2; p1 += type_size) {
+ for (size_t n = 0; n < type_size; n++) {
+ // scan bit set for this word
+ if (!(pm_bits[n / BITS_PER_WORD] & (1 << (n % BITS_PER_WORD))))
+ continue;
+
+ void* p = *(p1 + n);
+
+ if (p < minAddr || p >= maxAddr)
+ continue;
- if (p >= minAddr && p < maxAddr)
- {
if ((cast(size_t)p & ~(PAGESIZE-1)) == pcache)
continue;
- pool = findPool(p);
+ Pool* pool = findPool(p);
if (pool)
{
size_t offset = cast(size_t)(p - pool.baseAddr);
- size_t biti;
+ size_t bit_i;
size_t pn = offset / PAGESIZE;
Bins bin = cast(Bins)pool.pagetable[pn];
// Adjust bit to be at start of allocated memory block
if (bin <= B_PAGE)
- biti = (offset & notbinsize[bin]) >> 4;
+ bit_i = (offset & notbinsize[bin]) >> 4;
else if (bin == B_PAGEPLUS)
{
do
--pn;
}
while (cast(Bins)pool.pagetable[pn] == B_PAGEPLUS);
- biti = pn * (PAGESIZE / 16);
+ bit_i = pn * (PAGESIZE / 16);
}
else
{
if (bin >= B_PAGE) // Cache B_PAGE and B_PAGEPLUS lookups
pcache = cast(size_t)p & ~(PAGESIZE-1);
- if (!pool.mark.test(biti))
+ if (!pool.mark.test(bit_i))
{
- pool.mark.set(biti);
- if (!pool.noscan.test(biti))
+ pool.mark.set(bit_i);
+ if (!pool.noscan.test(bit_i))
{
- pool.scan.set(biti);
+ pool.scan.set(bit_i);
changes = 1;
}
}
anychanges |= changes;
}
-
/**
* Return number of full pages free'd.
*/
size_t fullcollectshell()
{
+ stats.collection_started();
+ scope (exit)
+ stats.collection_finished();
+
// The purpose of the 'shell' is to ensure all the registers
// get put on the stack so they'll be scanned
void *sp;
debug(COLLECT_PRINTF) printf("Gcx.fullcollect()\n");
thread_suspendAll();
+ stats.world_stopped();
p_cache = null;
size_cache = 0;
pool.mark.copy(&pool.freebits);
}
- rt_scanStaticData( &mark );
+ rt_scanStaticData( &mark_conservative );
if (!noStack)
{
// Scan stacks and registers for each paused thread
- thread_scanAll( &mark, stackTop );
+ thread_scanAll( &mark_conservative, stackTop );
}
// Scan roots
debug(COLLECT_PRINTF) printf("scan roots[]\n");
- mark(roots.ptr, roots.ptr + roots.length);
+ mark_conservative(roots.ptr, roots.ptr + roots.length);
// Scan ranges
debug(COLLECT_PRINTF) printf("scan ranges[]\n");
for (n = 0; n < ranges.length; n++)
{
debug(COLLECT_PRINTF) printf("\t%x .. %x\n", ranges[n].pbot, ranges[n].ptop);
- mark(ranges[n].pbot, ranges[n].ptop);
+ mark_conservative(ranges[n].pbot, ranges[n].ptop);
}
//log--;
pn = cast(size_t)(o - pool.baseAddr) / PAGESIZE;
bin = cast(Bins)pool.pagetable[pn];
- if (bin < B_PAGE)
- {
- mark(o, o + binsize[bin]);
+ if (bin < B_PAGE) {
+ auto end_of_blk = cast(size_t**)(o + binsize[bin] -
+ (size_t*).sizeof);
+ size_t* pm_bitmask = *end_of_blk;
+ mark(o, end_of_blk, pm_bitmask);
}
else if (bin == B_PAGE || bin == B_PAGEPLUS)
{
pn--;
}
u = 1;
- while (pn + u < pool.npages && pool.pagetable[pn + u] == B_PAGEPLUS)
+ while (pn + u < pool.npages &&
+ pool.pagetable[pn + u] == B_PAGEPLUS)
u++;
- mark(o, o + u * PAGESIZE);
+
+ size_t blk_size = u * PAGESIZE;
+ auto end_of_blk = cast(size_t**)(o + blk_size -
+ (size_t*).sizeof);
+ size_t* pm_bitmask = *end_of_blk;
+ mark(o, end_of_blk, pm_bitmask);
}
}
}
}
thread_resumeAll();
+ stats.world_started();
// Free up everything not marked
debug(COLLECT_PRINTF) printf("\tfree'ing\n");
auto size = binsize[bin];
byte* p = pool.baseAddr + pn * PAGESIZE;
byte* ptop = p + PAGESIZE;
- size_t biti = pn * (PAGESIZE/16);
- size_t bitstride = size / 16;
+ size_t bit_i = pn * (PAGESIZE/16);
+ size_t bit_stride = size / 16;
version(none) // BUG: doesn't work because freebits() must also be cleared
{
// If free'd entire page
- if (bbase[0] == 0 && bbase[1] == 0 && bbase[2] == 0 && bbase[3] == 0 &&
- bbase[4] == 0 && bbase[5] == 0 && bbase[6] == 0 && bbase[7] == 0)
+ if (bbase[0] == 0 && bbase[1] == 0 && bbase[2] == 0 &&
+ bbase[3] == 0 && bbase[4] == 0 && bbase[5] == 0 &&
+ bbase[6] == 0 && bbase[7] == 0)
{
- for (; p < ptop; p += size, biti += bitstride)
+ for (; p < ptop; p += size, bit_i += bit_stride)
{
- if (pool.finals.nbits && pool.finals.testClear(biti))
- rt_finalize(cast(List *)sentinel_add(p), false/*noStack > 0*/);
- gcx.clrBits(pool, biti, BlkAttr.ALL_BITS);
+ if (pool.finals.nbits && pool.finals.testClear(bit_i)) {
+ if (opts.options.sentinel)
+ rt_finalize(cast(List *)sentinel_add(p), false/*noStack > 0*/);
+ else
+ rt_finalize(cast(List *)p, false/*noStack > 0*/);
+ }
+ gcx.clrAttr(pool, bit_i, BlkAttr.ALL_BITS);
List *list = cast(List *)p;
- debug (MEMSTOMP) memset(p, 0xF3, size);
+ if (opts.options.mem_stomp)
+ memset(p, 0xF3, size);
}
pool.pagetable[pn] = B_FREE;
freed += PAGESIZE;
continue;
}
}
- for (; p < ptop; p += size, biti += bitstride)
+ for (; p < ptop; p += size, bit_i += bit_stride)
{
- if (!pool.mark.test(biti))
+ if (!pool.mark.test(bit_i))
{
- sentinel_Invariant(sentinel_add(p));
-
- pool.freebits.set(biti);
- if (pool.finals.nbits && pool.finals.testClear(biti))
- rt_finalize(cast(List *)sentinel_add(p), false/*noStack > 0*/);
- clrBits(pool, biti, BlkAttr.ALL_BITS);
+ if (opts.options.sentinel)
+ sentinel_Invariant(sentinel_add(p));
+
+ pool.freebits.set(bit_i);
+ if (pool.finals.nbits && pool.finals.testClear(bit_i)) {
+ if (opts.options.sentinel)
+ rt_finalize(cast(List *)sentinel_add(p), false/*noStack > 0*/);
+ else
+ rt_finalize(cast(List *)p, false/*noStack > 0*/);
+ }
+ clrAttr(pool, bit_i, BlkAttr.ALL_BITS);
List *list = cast(List *)p;
- debug (MEMSTOMP) memset(p, 0xF3, size);
+ if (opts.options.mem_stomp)
+ memset(p, 0xF3, size);
freed += size;
}
}
else if (bin == B_PAGE)
{
- size_t biti = pn * (PAGESIZE / 16);
- if (!pool.mark.test(biti))
+ size_t bit_i = pn * (PAGESIZE / 16);
+ if (!pool.mark.test(bit_i))
{
byte *p = pool.baseAddr + pn * PAGESIZE;
- sentinel_Invariant(sentinel_add(p));
- if (pool.finals.nbits && pool.finals.testClear(biti))
- rt_finalize(sentinel_add(p), false/*noStack > 0*/);
- clrBits(pool, biti, BlkAttr.ALL_BITS);
+ if (opts.options.sentinel)
+ sentinel_Invariant(sentinel_add(p));
+ if (pool.finals.nbits && pool.finals.testClear(bit_i)) {
+ if (opts.options.sentinel)
+ rt_finalize(sentinel_add(p), false/*noStack > 0*/);
+ else
+ rt_finalize(p, false/*noStack > 0*/);
+ }
+ clrAttr(pool, bit_i, BlkAttr.ALL_BITS);
debug(COLLECT_PRINTF) printf("\tcollecting big %x\n", p);
pool.pagetable[pn] = B_FREE;
freedpages++;
- debug (MEMSTOMP) memset(p, 0xF3, PAGESIZE);
+ if (opts.options.mem_stomp)
+ memset(p, 0xF3, PAGESIZE);
while (pn + 1 < pool.npages && pool.pagetable[pn + 1] == B_PAGEPLUS)
{
pn++;
pool.pagetable[pn] = B_FREE;
freedpages++;
- debug (MEMSTOMP)
+ if (opts.options.mem_stomp)
{
p += PAGESIZE;
memset(p, 0xF3, PAGESIZE);
for (size_t pn = 0; pn < pool.npages; pn++)
{
Bins bin = cast(Bins)pool.pagetable[pn];
- size_t biti;
+ size_t bit_i;
size_t u;
if (bin < B_PAGE)
{
size_t size = binsize[bin];
- size_t bitstride = size / 16;
- size_t bitbase = pn * (PAGESIZE / 16);
- size_t bittop = bitbase + (PAGESIZE / 16);
+ size_t bit_stride = size / 16;
+ size_t bit_base = pn * (PAGESIZE / 16);
+ size_t bit_top = bit_base + (PAGESIZE / 16);
byte* p;
- biti = bitbase;
- for (biti = bitbase; biti < bittop; biti += bitstride)
+ bit_i = bit_base;
+ for (; bit_i < bit_top; bit_i += bit_stride)
{
- if (!pool.freebits.test(biti))
+ if (!pool.freebits.test(bit_i))
goto Lnotfree;
}
pool.pagetable[pn] = B_FREE;
p = pool.baseAddr + pn * PAGESIZE;
for (u = 0; u < PAGESIZE; u += size)
{
- biti = bitbase + u / 16;
- if (pool.freebits.test(biti))
+ bit_i = bit_base + u / 16;
+ if (pool.freebits.test(bit_i))
{
List *list = cast(List *)(p + u);
- if (list.next != bucket[bin]) // avoid unnecessary writes
+ // avoid unnecessary writes
+ if (list.next != bucket[bin])
list.next = bucket[bin];
bucket[bin] = list;
}
/**
*
*/
- uint getBits(Pool* pool, size_t biti)
+ uint getAttr(Pool* pool, size_t bit_i)
in
{
assert( pool );
}
body
{
- uint bits;
+ uint attrs;
if (pool.finals.nbits &&
- pool.finals.test(biti))
- bits |= BlkAttr.FINALIZE;
- if (pool.noscan.test(biti))
- bits |= BlkAttr.NO_SCAN;
+ pool.finals.test(bit_i))
+ attrs |= BlkAttr.FINALIZE;
+ if (pool.noscan.test(bit_i))
+ attrs |= BlkAttr.NO_SCAN;
// if (pool.nomove.nbits &&
-// pool.nomove.test(biti))
-// bits |= BlkAttr.NO_MOVE;
- return bits;
+// pool.nomove.test(bit_i))
+// attrs |= BlkAttr.NO_MOVE;
+ return attrs;
}
/**
*
*/
- void setBits(Pool* pool, size_t biti, uint mask)
+ void setAttr(Pool* pool, size_t bit_i, uint mask)
in
{
assert( pool );
{
if (!pool.finals.nbits)
pool.finals.alloc(pool.mark.nbits);
- pool.finals.set(biti);
+ pool.finals.set(bit_i);
}
if (mask & BlkAttr.NO_SCAN)
{
- pool.noscan.set(biti);
+ pool.noscan.set(bit_i);
}
// if (mask & BlkAttr.NO_MOVE)
// {
// if (!pool.nomove.nbits)
// pool.nomove.alloc(pool.mark.nbits);
-// pool.nomove.set(biti);
+// pool.nomove.set(bit_i);
// }
}
/**
*
*/
- void clrBits(Pool* pool, size_t biti, uint mask)
+ void clrAttr(Pool* pool, size_t bit_i, uint mask)
in
{
assert( pool );
body
{
if (mask & BlkAttr.FINALIZE && pool.finals.nbits)
- pool.finals.clear(biti);
+ pool.finals.clear(bit_i);
if (mask & BlkAttr.NO_SCAN)
- pool.noscan.clear(biti);
+ pool.noscan.clear(bit_i);
// if (mask & BlkAttr.NO_MOVE && pool.nomove.nbits)
-// pool.nomove.clear(biti);
+// pool.nomove.clear(bit_i);
}
}
/* ============================ SENTINEL =============================== */
-version (SENTINEL)
-{
- const size_t SENTINEL_PRE = cast(size_t) 0xF4F4F4F4F4F4F4F4UL; // 32 or 64 bits
- const ubyte SENTINEL_POST = 0xF5; // 8 bits
- const uint SENTINEL_EXTRA = 2 * size_t.sizeof + 1;
+const size_t SENTINEL_PRE = cast(size_t) 0xF4F4F4F4F4F4F4F4UL; // 32 or 64 bits
+const ubyte SENTINEL_POST = 0xF5; // 8 bits
+const uint SENTINEL_EXTRA = 2 * size_t.sizeof + 1;
- size_t* sentinel_size(void *p) { return &(cast(size_t *)p)[-2]; }
- size_t* sentinel_pre(void *p) { return &(cast(size_t *)p)[-1]; }
- ubyte* sentinel_post(void *p) { return &(cast(ubyte *)p)[*sentinel_size(p)]; }
+size_t* sentinel_size(void *p) { return &(cast(size_t *)p)[-2]; }
+size_t* sentinel_pre(void *p) { return &(cast(size_t *)p)[-1]; }
+ubyte* sentinel_post(void *p) { return &(cast(ubyte *)p)[*sentinel_size(p)]; }
- void sentinel_init(void *p, size_t size)
- {
- *sentinel_size(p) = size;
- *sentinel_pre(p) = SENTINEL_PRE;
- *sentinel_post(p) = SENTINEL_POST;
- }
-
-
- void sentinel_Invariant(void *p)
- {
- assert(*sentinel_pre(p) == SENTINEL_PRE);
- assert(*sentinel_post(p) == SENTINEL_POST);
- }
-
-
- void *sentinel_add(void *p)
- {
- return p + 2 * size_t.sizeof;
- }
-
-
- void *sentinel_sub(void *p)
- {
- return p - 2 * size_t.sizeof;
- }
-}
-else
+void sentinel_init(void *p, size_t size)
{
- const uint SENTINEL_EXTRA = 0;
-
-
- void sentinel_init(void *p, size_t size)
- {
- }
+ *sentinel_size(p) = size;
+ *sentinel_pre(p) = SENTINEL_PRE;
+ *sentinel_post(p) = SENTINEL_POST;
+}
- void sentinel_Invariant(void *p)
- {
- }
+void sentinel_Invariant(void *p)
+{
+ assert(*sentinel_pre(p) == SENTINEL_PRE);
+ assert(*sentinel_post(p) == SENTINEL_POST);
+}
- void *sentinel_add(void *p)
- {
- return p;
- }
+void *sentinel_add(void *p)
+{
+ return p + 2 * size_t.sizeof;
+}
- void *sentinel_sub(void *p)
- {
- return p;
- }
+void *sentinel_sub(void *p)
+{
+ return p - 2 * size_t.sizeof;
}
projects / software / dgc / cdgc.git / blobdiff