Parallel Bash #2

The first part of Parallel Bash dealt with creation of parallel processes, so now it’s time for a real parallel tool.

The Apache benchmark tool ab has the disadvantage of sending only one URL for requests. To emulate a real server load we need some tools which can download several different files (.jpg, .js, .php, .css, ..) where each of them causes different CPU loads and network througphut, as well as different sizes of tranfered data. This can easily be done with httrack or wget.

And here is how:

The parallel routine pwork runs a site-mirror and collects the elapsed time in a log file

...
HTLOG=/tmp/httime.$$.log

pwork () {
    TMPDIR=$(mktemp -d /tmp/htbench_instance_XXXXX)
    TMPFILE=$(mktemp /tmp/htbench_log_XXXX)
    ( cd $TMPDIR
      #  append elapsed time to HTLOG
      /usr/bin/time -f "%e" -o $TMPFILE  \
      httrack -c8 --mirror --depth=2 -Q -q $1
      if lockfile -1 -r12 $HTLOCK; then
          cat $TMPFILE >> $HTLOG
      fi
      rm -f $HTLOCK $TMPFILE
    )
    rm -rf "$TMPDIR"
}
...

Limit the benchmark to n loops:

...
while [ $COUNT != 0 ]
do      J=$(jobs -p)
        echo "jobs left: "$COUNT
        N=$(echo $J|wc -w)
        if [ -z "$N" -o "$N" -lt $PARALLEL ]; then
            pwork "$URI" &
            (( COUNT-- ))
        fi
        sleep .2
done

Show final results:

...
show_results () {

    MINRUN=$(sort -n $HTLOG|head -1|cut -d' ' -f1)
    MAXRUN=$(sort -n $HTLOG|tail -1|cut -d' ' -f1)
    AVGRUN=$(perl -e '$sum=$count=0;
                         while (<>) {
                             chomp;$sum+=$_;$count++; }
                          printf "%.2f\n", $sum/$count;' $HTLOG)
    rm $HTLOG

    echo "Benchmark Results"
    echo "================="
    echo
    echo "longest runtime:  "$MAXRUN
    echo "shortest runtime: "$MINRUN
    echo "average runtime:  "$AVGRUN
    echo
}

reaper () {
        echo "waiting to finish $RUNNING" >&2
        wait
        show_results
        exit 0
}

trap reaper 0 1 2 3 11 15
...

Download latest version:
htbench.sh