Add (optional) early collection support

When early collection is enabled, the collection will be triggered before
the memory is actually exhausted (the min_free option is used to determine
how early the collection should be triggered). This could remove a little
pressure from the GC when eager allocation is not available (or when eager
allocation makes the heap grow too much).

The option is disabled by default and can be disabled with the
early_collect option.

diff --git a/rt/gc/cdgc/gc.d b/rt/gc/cdgc/gc.d
index 2ec72c4b9ac273a046ca96eef200e5dce60fa5a7..d4fb0e24cb4583293764de551dbe7c4f166a0d42 100644 (file)
--- a/rt/gc/cdgc/gc.d
+++ b/rt/gc/cdgc/gc.d
@@ -701,7 +701,7 @@ void mark_range(void *pbot, void *ptop, size_t* pm_bitmask)
 /**
  * Return number of full pages free'd.
  */
-size_t fullcollectshell()
+size_t fullcollectshell(bool early = false)
 {
     gc.stats.collection_started();
     scope (exit)
@@ -769,7 +769,7 @@ size_t fullcollectshell()
         mov sp[EBP],ESP     ;
     }
     }
-    result = fullcollect(sp);
+    result = fullcollect(sp, early);
     version (GNU)
     {
         // nothing to do
@@ -792,35 +792,42 @@ size_t fullcollectshell()
 /**
  *
  */
-size_t fullcollect(void *stackTop)
-{
-    debug(COLLECT_PRINTF) printf("Gcx.fullcollect()\n");
-
-    // If eager allocation is used, we need to check first if there is a mark
-    // process running. If there isn't, we start a new one (see the next code
-    // block). If there is, we check if it's still running or already finished.
-    // If it's still running, we tell the caller process no memory has been
-    // recovered (it will allocated more to fulfill the current request).  If
-    // the mark process is done, we lunch the sweep phase and hope enough
-    // memory is freed (if that not the case, the caller will allocate more
-    // memory and the next time it's exhausted it will run a new collection).
-    if (opts.options.eager_alloc) {
-        if (collect_in_progress()) {
-            os.WRes r = os.wait_pid(gc.mark_proc_pid, false); // don't block
-            assert (r != os.WRes.ERROR);
-            switch (r) {
+size_t fullcollect(void *stackTop, bool early = false)
+{
+    debug(COLLECT_PRINTF) printf("Gcx.fullcollect(early=%d)\n",
+            cast(int) early);
+
+    // We will block the mutator only if eager allocation is not used and this
+    // is not an early collection.
+    bool block = !opts.options.eager_alloc && !early;
+
+    // If there is a mark process running, check if it already finished.  If
+    // that is the case, we lunch the sweep phase and hope enough memory is
+    // freed.  If it's still running, either we block until the mark phase is
+    // done (and then sweep to finish the collection), or we tell the caller
+    // process no memory has been recovered (it will allocated more to fulfill
+    // the current request if eager allocation is used) and let the mark phase
+    // keep running in parallel.
+    if (collect_in_progress()) {
+        os.WRes r = os.wait_pid(gc.mark_proc_pid, block);
+        assert (r != os.WRes.ERROR);
+        switch (r) {
             case os.WRes.DONE:
-                debug(COLLECT_PRINTF) printf("\t\tmark proc DONE\n");
+                debug(COLLECT_PRINTF) printf("\t\tmark proc DONE (block=%d)\n",
+                        cast(int) block);
                 gc.mark_proc_pid = 0;
                 return sweep();
             case os.WRes.RUNNING:
                 debug(COLLECT_PRINTF) printf("\t\tmark proc RUNNING\n");
-                return 0;
+                if (!block)
+                    return 0;
+                // Something went wrong, if block is true, wait() should never
+                // returned RUNNING.
+                goto case os.WRes.ERROR;
             case os.WRes.ERROR:
                 debug(COLLECT_PRINTF) printf("\t\tmark proc ERROR\n");
                 disable_fork(); // Try to keep going without forking
                 break;
-            }
         }
     }
 
@@ -830,15 +837,14 @@ size_t fullcollect(void *stackTop)
     gc.stats.world_stopped();
 
     // If forking is enabled, we fork() and start a new mark phase in the
-    // child. The parent process will tell the caller that no memory could be
-    // recycled if eager allocation is used, allowing the mutator to keep going
-    // almost instantly (at the expense of more memory consumption because
-    // a new allocation will be triggered to fulfill the current request). If
-    // no eager allocation is used, the parent will wait for the mark phase to
-    // finish before returning control to the mutator, but other threads are
-    // restarted and may run in parallel with the mark phase (unless they
-    // allocate or use the GC themselves, in which case the global GC lock will
-    // stop them).
+    // child. If the collection should not block, the parent process tells the
+    // caller no memory could be recycled immediately (if eager allocation is
+    // used, and this collection was triggered by an allocation, the caller
+    // should allocate more memory to fulfill the request). If the collection
+    // should block, the parent will wait for the mark phase to finish before
+    // returning control to the mutator, but other threads are restarted and
+    // may run in parallel with the mark phase (unless they allocate or use the
+    // GC themselves, in which case the global GC lock will stop them).
     if (opts.options.fork) {
         cstdio.fflush(null); // avoid duplicated FILE* output
         os.pid_t child_pid = os.fork();
@@ -852,16 +858,16 @@ size_t fullcollect(void *stackTop)
             cstdlib.exit(0);
             break; // bogus, will never reach here
         default: // parent process (i.e. the mutator)
-            // start the world again and wait for the mark phase to finish
             thread_resumeAll();
             gc.stats.world_started();
-            if (opts.options.eager_alloc) {
+            if (!block) {
                 gc.mark_proc_pid = child_pid;
                 return 0;
             }
             os.WRes r = os.wait_pid(child_pid); // block until it finishes
             assert (r == os.WRes.DONE);
-            debug(COLLECT_PRINTF) printf("\t\tmark proc DONE (block)\n");
+            debug(COLLECT_PRINTF) printf("\t\tmark proc DONE (block=%d)\n",
+                    cast(int) block);
             if (r == os.WRes.DONE)
                 return sweep();
             debug(COLLECT_PRINTF) printf("\tmark() proc ERROR\n");
@@ -1268,9 +1274,10 @@ body
 
 void disable_fork()
 {
-    // we have to disable both options, as eager_alloc assumes fork is enabled
+    // we have to disable all options that assume fork is enabled
     opts.options.fork = false;
     opts.options.eager_alloc = false;
+    opts.options.early_collect = false;
 }
 
 
@@ -1282,9 +1289,9 @@ void initialize()
     // If we are going to fork, make sure we have the needed OS support
     if (opts.options.fork)
         opts.options.fork = os.HAVE_SHARED && os.HAVE_FORK;
-    // Eager allocation is only possible when forking
+    // Disable fork()-related options if we don't have it
     if (!opts.options.fork)
-        opts.options.eager_alloc = false;
+        disable_fork();
     gc.lock = GCLock.classinfo;
     gc.inited = 1;
     setStackBottom(rt_stackBottom());
@@ -1297,6 +1304,18 @@ void initialize()
 }
 
 
+// Launch a parallel collection if we don't have enough free memory available
+// (we have less than min_free% of the total heap free).
+void early_collect()
+{
+    if (!opts.options.early_collect || collect_in_progress())
+        return;
+    double percent_free = gc.free_mem * 100.0 / gc.total_mem;
+    if (percent_free < opts.options.min_free)
+        fullcollectshell(true);
+}
+
+
 //
 //
 //
@@ -1428,9 +1447,10 @@ private void *malloc(size_t size, uint attrs, size_t* pm_bitmask)
 
     gc.free_mem -= capacity;
     if (collected) {
-        // If there is not enough free memory, allocate a new pool big enough
-        // to have at least the min_free% of the total heap free. If there is
-        // too much free memory, try to free some empty pools.
+        // If there is not enough free memory (after a collection), allocate
+        // a new pool big enough to have at least the min_free% of the total
+        // heap free. If the collection left too much free memory, try to free
+        // some empty pools.
         double percent_free = gc.free_mem * 100.0 / gc.total_mem;
         if (percent_free < opts.options.min_free) {
             auto pool_size = gc.total_mem * 1.0 / opts.options.min_free
@@ -1440,6 +1460,8 @@ private void *malloc(size_t size, uint attrs, size_t* pm_bitmask)
         else
             minimize(false);
     }
+    else
+        early_collect();
 
     return p;
 }
@@ -1561,6 +1583,7 @@ private void *realloc(void *p, size_t size, uint attrs,
                                 blk_base_addr + new_blk_size - pm_bitmask_size);
                         *end_of_blk = pm_bitmask;
                     }
+                    early_collect();
                     return p;
                 }
                 if (i == pool.npages)
@@ -1673,6 +1696,9 @@ body
         auto end_of_blk = cast(size_t**)(blk_base_addr + new_size);
         *end_of_blk = pm_bitmask;
     }
+
+    early_collect();
+
     return new_size;
 }
 

diff --git a/rt/gc/cdgc/opts.d b/rt/gc/cdgc/opts.d
index 7cd6784f9c7338673dbcd953d066bece6cfbeb2d..1ec5d974c6c95394a28b0163f942b89da3c88973 100644 (file)
--- a/rt/gc/cdgc/opts.d
+++ b/rt/gc/cdgc/opts.d
@@ -58,6 +58,7 @@ struct Options
     bool conservative = false;
     bool fork = true;
     bool eager_alloc = true;
+    bool early_collect = false;
     uint min_free = 5; // percent of the heap (0-100)
     size_t prealloc_psize = 0;
     size_t prealloc_npools = 0;
@@ -151,6 +152,8 @@ void process_option(char* opt_name, char* opt_value)
         options.fork = parse_bool(opt_value);
     else if (cstr_eq(opt_name, "eager_alloc"))
         options.eager_alloc = parse_bool(opt_value);
+    else if (cstr_eq(opt_name, "early_collect"))
+        options.early_collect = parse_bool(opt_value);
     else if (cstr_eq(opt_name, "min_free"))
         parse_min_free(opt_value);
     else if (cstr_eq(opt_name, "pre_alloc"))
@@ -218,6 +221,7 @@ unittest
         assert (conservative == false);
         assert (fork == true);
         assert (eager_alloc == true);
+        assert (early_collect == false);
         assert (prealloc_psize == 0);
         assert (prealloc_npools == 0);
         assert (min_free == 5);
@@ -231,6 +235,7 @@ unittest
         assert (conservative == false);
         assert (fork == true);
         assert (eager_alloc == true);
+        assert (early_collect == false);
         assert (prealloc_psize == 0);
         assert (prealloc_npools == 0);
         assert (min_free == 5);
@@ -244,6 +249,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 0);
         assert (prealloc_npools == 0);
         assert (min_free == 5);
@@ -257,11 +263,12 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 0);
         assert (prealloc_npools == 0);
         assert (min_free == 5);
     }
-    parse("pre_alloc:min_free=30");
+    parse("pre_alloc:min_free=30:early_collect");
     with (options) {
         assert (verbose == 1);
         assert (cstring.strcmp(log_file.ptr, "12345 67890".ptr) == 0);
@@ -270,11 +277,12 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == true);
         assert (prealloc_psize == 0);
         assert (prealloc_npools == 0);
         assert (min_free == 30);
     }
-    parse("pre_alloc=1");
+    parse("pre_alloc=1:early_collect=0");
     with (options) {
         assert (verbose == 1);
         assert (cstring.strcmp(log_file.ptr, "12345 67890".ptr) == 0);
@@ -283,6 +291,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 1 * 1024 * 1024);
         assert (prealloc_npools == 1);
         assert (min_free == 30);
@@ -296,6 +305,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 1 * 1024 * 1024);
         assert (prealloc_npools == 1);
         assert (min_free == 30);
@@ -309,6 +319,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 1 * 1024 * 1024);
         assert (prealloc_npools == 1);
         assert (min_free == 30);
@@ -322,6 +333,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 9 * 1024 * 1024);
         assert (prealloc_npools == 10);
         assert (min_free == 30);
@@ -335,6 +347,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 5 * 1024 * 1024);
         assert (prealloc_npools == 2);
         assert (min_free == 30);
@@ -348,6 +361,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 5 * 1024 * 1024);
         assert (prealloc_npools == 2);
         assert (min_free == 30);
@@ -361,6 +375,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 5 * 1024 * 1024);
         assert (prealloc_npools == 2);
         assert (min_free == 0);
@@ -374,6 +389,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 5 * 1024 * 1024);
         assert (prealloc_npools == 2);
         assert (min_free == 100);
@@ -387,6 +403,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 5 * 1024 * 1024);
         assert (prealloc_npools == 2);
         assert (min_free == 100);
@@ -400,6 +417,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 5 * 1024 * 1024);
         assert (prealloc_npools == 2);
         assert (min_free == 100);
@@ -413,6 +431,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 5 * 1024 * 1024);
         assert (prealloc_npools == 2);
         assert (min_free == 100);
@@ -426,6 +445,7 @@ unittest
         assert (conservative == true);
         assert (fork == false);
         assert (eager_alloc == false);
+        assert (early_collect == false);
         assert (prealloc_psize == 5 * 1024 * 1024);
         assert (prealloc_npools == 2);
         assert (min_free == 100);