From 9a4584a1740f063aafc635cb5cd0a34f3195399c Mon Sep 17 00:00:00 2001 From: Leandro Lucarella Date: Sun, 14 Nov 2010 19:33:32 -0300 Subject: [PATCH] Add scripts to run the benchmark for the web This set of scripts run the benchmark and generates graphs with the results suitable for publishing on the web. The main script is run.sh. --- histogram-plot.tpl.gpi | 56 ++++++++++ pause-plot.sh | 19 ++++ pause-run.sh | 120 +++++++++++++++++++++ run.sh | 9 ++ templite.py | 234 +++++++++++++++++++++++++++++++++++++++++ tests-args.sh | 20 ++++ time-plot.sh | 17 +++ time-run.sh | 100 ++++++++++++++++++ 8 files changed, 575 insertions(+) create mode 100644 histogram-plot.tpl.gpi create mode 100755 pause-plot.sh create mode 100755 pause-run.sh create mode 100755 run.sh create mode 100755 templite.py create mode 100644 tests-args.sh create mode 100755 time-plot.sh create mode 100755 time-run.sh diff --git a/histogram-plot.tpl.gpi b/histogram-plot.tpl.gpi new file mode 100644 index 0000000..daa0738 --- /dev/null +++ b/histogram-plot.tpl.gpi @@ -0,0 +1,56 @@ +#!/usr/bin/gnuplot + +# Terminal +${ if 'fmt' in globals() and fmt: }$ + ${ if fmt == 'png': }$ + set term png size 640,480 + ${: elif fmt == 'svg': }$ + set term svg size 1600 1200 dynamic + ${: elif fmt == 'eps': }$ + set term postscript eps monochrome size 18cm,27cm 16 + ${:}$ + set output "${output}$" +${: else: }$ +set term x11 persist +${:}$ + +# Style +set style histogram rows title offset character 2, 0.25, 0 +set style fill solid 1.00 border rgb 'black' +set style data histograms +set xtics border in scale 1,0.5 nomirror rotate by -45 +#set xtics font "/usr/share/fonts/truetype/msttcorefonts/arial.ttf,9" + +# Title +set title "${title}$ [${cpus}$ CPU${'s' if cpus > 1 else ''}$]" + +# Data format +set datafile separator ',' + +# Labels +set ylabel "Time (sec)" +set xlabel font "/usr/share/fonts/truetype/msttcorefonts/arial.ttf,9" + +# Plot +${ +def title(i, name): + emit('title "%s"' % name if i == 0 else 'notitle') +def end(i): + emit(', \\' if i != len(progs)-1 else '') +def at(i): + emit(i*4) +}$ +plot ${ for i, (name, file) in enumerate(progs.items()): }$ \ + newhistogram '${name}$' at ${at(i)}$, '${file}$' \ + using 2:xtic(1) ${title(i, 'min')}$ \ + lc rgb 'gray20', \ + '' using (abs($3-$2-$5)) notitle \ + lc rgb 'gray90', \ + '' using ($5) ${title(i, 'mean+/-std')}$ \ + lc rgb 'gray50', \ + '' using ($5) notitle \ + lc rgb 'gray50', \ + '' using (abs($4-$3-$5)) ${title(i, 'max')}$ \ + lc rgb 'gray90' ${end(i)}$ +${:}$ + diff --git a/pause-plot.sh b/pause-plot.sh new file mode 100755 index 0000000..670037b --- /dev/null +++ b/pause-plot.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +test -n "$1" && title=$1 && shift +test -n "$1" && fmt=$1 && shift +test -n "$1" && output=$1 && shift + +files="$@" +test -z "$files" && files=build/pause-*-${CPUS}cpu.csv + +for f in $files +do + p=`echo $f | sed "s,.*build/.*-\\(.*\\)-${CPUS}cpu\\.csv,\\1,"` + progs="$progs$p='$f'," +done + +./templite.py "fmt='$fmt', output='$output', title='$title', cpus=$CPUS, + progs=dict($progs)" \ + < histogram-plot.tpl.gpi | gnuplot + diff --git a/pause-run.sh b/pause-run.sh new file mode 100755 index 0000000..d664500 --- /dev/null +++ b/pause-run.sh @@ -0,0 +1,120 @@ +#!/bin/sh + +. ./tests-args.sh +factor_rnddata=0.1 + + +FORMATS=${FORMATS:-png svg eps} + +NAMES=${NAMES:-`echo ./micro/*.d | xargs -n1 sh -c 'basename $0 .d'` dil} + +TYPES=${TYPES:-stw fork ea} + +CPUS=${CPUS:-`grep '^processor' /proc/cpuinfo | wc -l`} + +PLOTONLY=${PLOTONLY:-0} + +NORUN=${NORUN:-} + +STRIP=${STRIP:-1} + +ARCH=${ARCH:-} + + +run() { + name=$1 + type=$2 + prog=$3 + eval "args=\"\$args_$name\"" + if test $type = "warm" + then + echo " WARM $name" + $prog $args > /dev/null + return 0 + fi + dst="./build/cdgc/pause/$name-$type-${CPUS}cpu.csv" + if test -f $dst + then + echo "$NORUN" | grep -q "cdgc\\|$name" && + continue + test $PLOTONLY -eq 1 && + continue + fi + gc_opts= + test $type = "stw" && gc_opts="fork=0" + test $type = "fork" && gc_opts="eager_alloc=0" + test $type = "ea" && gc_opts="" + pa="$args" + test ${#args} -gt 40 && + pa="`echo $args | cut -b1-40`..." + echo " RUN $name $pa > $dst" + D_GC_OPTS="$D_GC_OPTS:collect_stats_file=$dst:$gc_opts" \ + setarch i386 $ARCH $prog $args > /dev/null +} + + +make -srj4 micro-gc-build dil-gc-build GC=cdgc + +for name in $NAMES +do + prog="./build/cdgc/bin/$name" + test $STRIP -eq 1 && + strip $prog + for type in warm $TYPES + do + run $name $type $prog + done +done + +for name in $NAMES +do + for time in stw pause + do + dst=./build/$time-$name-${CPUS}cpu.csv + if test -f $dst + then + echo "$NORUN" | grep -q "$name" && + continue + test $PLOTONLY -eq 1 && + continue + mv $dst ./build/$time-$name-${CPUS}cpu-old.csv + fi + col=4 # Stop-the-world data column + test $time = "pause" && col=2 # Total pause data column + echo -n > $dst + for type in $TYPES + do + src="./build/cdgc/pause/$name-$type-${CPUS}cpu.csv" + eval "factor=\"\$factor_$name\"" + test -z "$factor" && + factor=1 + (echo -n $type,; awk -F, \ + "{if (FNR > 1 && \$$col > 0) + print \$$col*$factor}" $src \ + | ./stats.py) >> $dst + echo " STATS `tail -n1 $dst | tr , ' '` >> $dst" + done + done +done + +for time in stw pause +do + echo -n " PLOT $time ${CPUS}cpu > ./build/$time-${CPUS}cpu.{" + for fmt in $FORMATS + do + dst=./build/$time-${CPUS}cpu.$fmt + test -f $dst && + mv $dst ./build/$time-${CPUS}cpu-old.$fmt + echo -n "$fmt," + files='' + for name in $NAMES + do + files="$files ./build/$time-$name-${CPUS}cpu.csv" + done + test $time = "stw" && title="Stop-the-world Time" + test $time = "pause" && title="Pause Time" + ./pause-plot.sh "$title" $fmt $dst $files + done + echo '}' +done + diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..7e7aa8e --- /dev/null +++ b/run.sh @@ -0,0 +1,9 @@ +#!/bin/sh +cpus="1 2 4" +test -n "$1" && cpus="$@" +for c in $cpus +do + ./bench.sh $c ./time-run.sh + ./bench.sh $c ./pause-run.sh +done + diff --git a/templite.py b/templite.py new file mode 100755 index 0000000..fea3324 --- /dev/null +++ b/templite.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python +# +# Templite+ +# A light-weight, fully functional, general purpose templating engine +# +# Copyright (c) 2009 joonis new media +# Author: Thimo Kraemer +# +# Based on Templite - Tomer Filiba +# http://code.activestate.com/recipes/496702/ +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. +# + +import sys, re + +class Templite(object): + auto_emit = re.compile('(^[\'\"])|(^[a-zA-Z0-9_\[\]\'\"]+$)') + + def __init__(self, template, start='${', end='}$'): + if len(start) != 2 or len(end) != 2: + raise ValueError('each delimiter must be two characters long') + delimiter = re.compile('%s(.*?)%s' % (re.escape(start), re.escape(end)), re.DOTALL) + offset = 0 + tokens = [] + for i, part in enumerate(delimiter.split(template)): + part = part.replace('\\'.join(list(start)), start) + part = part.replace('\\'.join(list(end)), end) + if i % 2 == 0: + if not part: continue + part = part.replace('\\', '\\\\').replace('"', '\\"') + part = '\t' * offset + 'emit("""%s""")' % part + else: + part = part.rstrip() + if not part: continue + if part.lstrip().startswith(':'): + if not offset: + raise SyntaxError('no block statement to terminate: ${%s}$' % part) + offset -= 1 + part = part.lstrip()[1:] + if not part.endswith(':'): continue + elif self.auto_emit.match(part.lstrip()): + part = 'emit(%s)' % part.lstrip() + lines = part.splitlines() + margin = min(len(l) - len(l.lstrip()) for l in lines if l.strip()) + part = '\n'.join('\t' * offset + l[margin:] for l in lines) + if part.endswith(':'): + offset += 1 + tokens.append(part) + if offset: + raise SyntaxError('%i block statement(s) not terminated' % offset) + self.__code = compile('\n'.join(tokens), '' % template[:20], 'exec') + + def render(self, __namespace=None, **kw): + """ + renders the template according to the given namespace. + __namespace - a dictionary serving as a namespace for evaluation + **kw - keyword arguments which are added to the namespace + """ + namespace = {} + if __namespace: namespace.update(__namespace) + if kw: namespace.update(kw) + namespace['emit'] = self.write + + __stdout = sys.stdout + sys.stdout = self + self.__output = [] + eval(self.__code, namespace) + sys.stdout = __stdout + return ''.join(self.__output) + + def write(self, *args): + for a in args: + self.__output.append(str(a)) + + +if __name__ == '__main__': + + if '--demo' not in sys.argv: + vars = eval('dict(' + ' '.join(sys.argv[1:]) + ')') + sys.stdout.write(Templite(sys.stdin.read()).render(vars)) + sys.exit(0) + + template = r""" +This we already know: + + + ${ + def say_hello(arg): + emit("hello ", arg, "
") + }$ + + + ${ + for i in range(10): + emit("\n") + }$ +
") + say_hello(i) + emit("
+ + ${emit("hi")}$ + + tralala ${if x > 7: + say_hello("big x")}$ lala + + $\{this is escaped starting delimiter + + ${emit("this }\$ is an escaped ending delimiter")}$ + + ${# this is a python comment }$ + + + + +But this is completely new: +${if x > 7:}$ + x is ${emit('greater')}$ than ${print x-1}$ Well, the print statement produces a newline. +${:else:}$ + This terminates the previous code block and starts an else code block + Also this would work: $\{:end}\$$\{else:}\$, but not this: $\{:end}\$ $\{else:}\$ +${:this terminates the else-block +only the starting colon is essential}$ + +So far you had to write: +${ + if x > 3: + emit(''' + After a condition you could not continue your template. + You had to write pure python code. + The only way was to use %%-based substitutions %s + ''' % x) +}$ + +${if x > 6:}$ + Now you do not need to break your template ${print x}$ +${:elif x > 3:}$ + This is great +${:endif}$ + +${for i in range(x-1):}$ Of course you can use any type of block statement ${i}$ ${"fmt: %s" % (i*2)}$ +${:else:}$ +Single variables and expressions starting with quotes are substituted automatically. +Instead $\{emit(x)}\$ you can write $\{x}\$ or $\{'%s' % x}\$ or $\{"", x}\$ +Therefore standalone statements like break, continue or pass +must be enlosed by a semicolon: $\{continue;}\$ +The end +${:end-for}$ +""" + + t = Templite(template) + print t.render(x=8) + + + # Output is: + """ +This we already know: + + + + + + + + + + + + + + + + +
hello 0
hello 1
hello 2
hello 3
hello 4
hello 5
hello 6
hello 7
hello 8
hello 9
+ + hi + + tralala hello big x
lala + + ${this is escaped starting delimiter + + this }$ is an escaped ending delimiter + + + + + + +But this is completely new: + + x is greater than 7 + Well, the print statement produces a newline. + + +So far you had to write: + + After a condition you could not continue your template. + You had to write pure python code. + The only way was to use %-based substitutions 8 + + + + Now you do not need to break your template 8 + + + + Of course you can use any type of block statement 0 fmt: 0 + Of course you can use any type of block statement 1 fmt: 2 + Of course you can use any type of block statement 2 fmt: 4 + Of course you can use any type of block statement 3 fmt: 6 + Of course you can use any type of block statement 4 fmt: 8 + Of course you can use any type of block statement 5 fmt: 10 + Of course you can use any type of block statement 6 fmt: 12 + +Single variables and expressions starting with quotes are substituted automatically. +Instead ${emit(x)}$ you can write ${x}$ or ${'%s' % x}$ or ${"", x}$ +Therefore standalone statements like break, continue or pass +must be enlosed by a semicolon: ${continue;}$ +The end +""" + diff --git a/tests-args.sh b/tests-args.sh new file mode 100644 index 0000000..38c5b5b --- /dev/null +++ b/tests-args.sh @@ -0,0 +1,20 @@ + +args_voronoi="-n 30000" +args_sbtree="16" +args_split="micro/bible.txt 2" +args_em3d="-n 4000 -d 300 -i 74" +args_bh="-b 4000" +# Using up to -c 1048575 takes ~2.5s (and uses ~256KiB), +# using -c 1048576 takes ~9s (and uses ~512KiB) +args_tsp="-c 1000000" +# Same as tsp but the limit is between 209000 and 2100000, +# the memory usage and time doubles (from ~3s/~128KiB to ~6s/256KiB) +args_bisort="-s 2000000" +args_conalloc="40 4 micro/bible.txt" +args_concpu="40 4 micro/bible.txt" + +tango_files=`find ../tango/tango -name '*.d' -o -name '*.di' | grep -v invariant` +args_dil="ddoc /tmp/tangodoc -hl --kandil -version=Tango -version=TangoDoc" +args_dil="$args_dil -version=Posix -version=linux $tango_files" +factor_dil=0.1 + diff --git a/time-plot.sh b/time-plot.sh new file mode 100755 index 0000000..2d3abd4 --- /dev/null +++ b/time-plot.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +test -n "$1" && fmt=$1 && shift +test -n "$1" && output=$1 && shift + +files="$@" +test -z "$files" && files=build/time-*-${CPUS}cpu.csv +for f in $files +do + p=`echo $f | sed "s,.*build/time-\\(.*\\)-${CPUS}cpu\\.csv,\\1,"` + progs="$progs$p='$f'," +done + +./templite.py "fmt='$fmt', output='$output', title='Run Time ($N runs)', cpus=$CPUS, + progs=dict($progs)" \ + < histogram-plot.tpl.gpi | gnuplot + diff --git a/time-run.sh b/time-run.sh new file mode 100755 index 0000000..8003aa4 --- /dev/null +++ b/time-run.sh @@ -0,0 +1,100 @@ +#!/bin/sh + +. ./tests-args.sh + + +export N=${N:-3} + +TIME=${TIME:-/usr/bin/time} + +FORMATS=${FORMATS:-png svg eps} + +NAMES=${NAMES:-`echo ./micro/*.d | xargs -n1 sh -c 'basename $0 .d'` dil} + +GCS=${GCS:-basic cdgc} + +CPUS=${CPUS:-`grep '^processor' /proc/cpuinfo | wc -l`} + +PLOTONLY=${PLOTONLY:-0} + +NORUN=${NORUN:-} + +STRIP=${STRIP:-1} + +ARCH=${ARCH:-} + + +for gc in $GCS +do + make -srj4 micro-gc-build dil-gc-build GC=$gc + for name in $NAMES + do + prog="./build/$gc/bin/$name" + dst="./build/$gc/time/$name-${CPUS}cpu.csv" + if test -f $dst + then + echo "$NORUN" | grep -q "$gc\\|$name" && + continue + test $PLOTONLY -eq 1 && + continue + fi + eval "args=\"\$args_$name\"" + pa="$args" + test ${#args} -gt 40 && + pa="`echo $args | cut -b1-40`..." + test $STRIP -eq 1 && + strip $prog + echo -n " RUN $name $pa > $dst: " + echo -n > $dst + for i in `seq $N` + do + test $(($i % 5)) -eq 0 && + echo -n "$i" || + echo -n "." + setarch i386 $ARCH \ + $TIME -f%e -a -o $dst \ + $prog $args > /dev/null + done + echo + done +done + +for name in $NAMES +do + dst=./build/time-$name-${CPUS}cpu.csv + if test -f $dst + then + echo "$NORUN" | grep -q "$name" && + continue + test $PLOTONLY -eq 1 && + continue + mv $dst ./build/time-$name-${CPUS}cpu-old.csv + fi + echo -n > $dst + for gc in $GCS + do + src=./build/$gc/time/$name-${CPUS}cpu.csv + eval "factor=\"\$factor_$name\"" + test -z "$factor" && + factor=1 + (echo -n $gc,; awk "{print \$1*$factor}" $src | ./stats.py) >> $dst + echo " STATS `tail -n1 $dst | tr , ' '` >> $dst" + done +done + +echo -n " PLOT ${CPUS}cpu > ./build/time-${CPUS}cpu.{" +for fmt in $FORMATS +do + dst=./build/time-${CPUS}cpu.$fmt + test -f $dst && + mv $dst ./build/time-${CPUS}cpu-old.$fmt + echo -n "$fmt," + files='' + for name in $NAMES + do + files="$files ./build/time-$name-${CPUS}cpu.csv" + done + ./time-plot.sh $fmt $dst $files +done +echo '}' + -- 2.43.0