#!/usr/bin/perl
#
#  /-------[ LICENSE ]----------------------------------------------------------\
#  |										|
#  | Copyright 2003-2004 Patrick C. Audley <paudley@blackcat.ca>                |
#  |										|
#  | This program is free software; you can redistribute it and/or modify	|
#  | it under the terms of the GNU General Public License as published by	|
#  | the Free Software Foundation; either version 2 of the License, or		|
#  | (at your option) any later version.					|
#  |										|
#  | This program is distributed in the hope that it will be useful,		|
#  | but WITHOUT ANY WARRANTY; without even the implied warranty of		|
#  | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              |
#  | GNU General Public License for more details.                               |
#  |                                                                            |
#  | You should have received a copy of the GNU General Public License          |
#  | along with this program; if not, write to the Free Software                |
#  | Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA  |
#  | The full license can also be obtained from:                                |
#  |                                                                            |
#  |    http://www.gnu.org/licenses/gpl.txt                                     |
#  |                                                                            |
#  \----------------------------------------------------------------------------/
#
#  If you have any suggestions for this script, please send them to me at:
#       paudley@blackcat.ca
#
#  The definitive version of this script can be foundn at:
#    http://blackcat.ca/lifeline/query.php?tag=BENCHGCC
#
# _______________
# VERSION HISTORY
#
# v1.0 (2004/09/01)
#  - First public version
#
# v1.1 (2004/09/22)
#  - added reporting of changes in binary size for nxsty
#
##
use strict;
use warnings;

use Getopt::Long;
use IO::Handle;
use Time::HiRes qw( time );
use List::Util qw( min sum );
use File::stat;
use Carp;

my $msgio = new IO::Handle;
$msgio->fdopen(fileno(STDOUT),"w");

my %opts = (
	    "base" => "-O2",
	    "peak" => "-O3",
	    "compile" => "./configure; make",
	    "clean" => "make clean",
	    "run" => "000",
	    "iterate" => 1,
	    "method" => "bestof",
	    "progressdots" => 50,
	    "sizeof" => "",
	    "compbase" => "",
	    "comppeak" => "",
	   );

=head1 NAME

bench_gcc - benchmark gcc optimization flags on a given code base

=head1 SYNOPSIS

bench_gcc [options]

GCC compiler options are a twisty, dark maze of evil smelling
passages.  Finding the right options for a given system can be quite
tricky even without all the myths that surround compiler options.

This script aims to help people test a set of options for a given
program.  This script will not tell you the Perfect CFLAGS(tm) nor
help you find God or anything else.  What it will do is let you
determine quantitatively how different options change you code.

=head2 Options

 FLAG OPTIONS:
  --base <flags>       Sets the baseline options for comparison
  --peak <flags>       Sets the new options that you want to benchmark

 COMPILER OPTIONS:
    To alter the paths for base and peak compilers, use:
  --compiler_base <path>
  --compiler_peak <path>

 SOURCE OPTIONS:
  --compile <command>  The command needed to compile the code. (ex: "./configure; make")
  --clean <command>    The command to reset the code in the current directory. (ex: "make clean");
  --run <command>      The command to use for benchmarking.  This is very application dependent and must be set by the user. [required option]

 CODE SIZE OPTIONS:
  --sizeof <file>      The file to compare size changes of.  Specifying this flag enables code size reporting.

 STATISTICAL OPTIONS:
  --iterate <number>   How many times the run command is executed.
  --bestof             Select the best time of all the runs.
  --average            Select the average time (mean) of all the runs.

 OUTPUT OPTIONS:
  --dotevery <number>  Sets the number of lines of output per dot printed.

 HELP OPTIONS:
  --man         Show a man page for this program.
  --help        Print a short option summary.

=head1 TIPS

- To eliminate disk cache from the benchmarks, place the testing directory on a ramdisk.

- The final report is appended in CSV format to "bench_gcc.csv" in the main directory.  A simple perl script can be written to parse this or alternately it can be loaded into a spreadsheet program with Gnumeric.

- You can use --compiler_base and --compile_peak to set different paths for gcc and g++ in order to test differnt version of gcc with the same base and peak options.

- To test more than two combinations of flags you can run the script multiple times with all the options.  Ignore the reports and use the CSV file that will have all the runs results in it.

- More complex stats can be had by hacking this perl script to suit your specific needs.

- Very involved tests can be done with the run command by writing a small testing script (ex:  create foo.sh and use --run foo.sh).  For instance, if you are interested in benchmarking options on KMail, you could write a test script that starts KMail, uses dcop to do some KMail things and shut it down.

- Try to make your run commands reflect the behaviours that you want to see in the application.

=head1 EXAMPLES

To benchmark the differnces in the options "-01" and "-03" on MP3 encoding you might perform the following steps.

You need a suitable WAV file to run this example.  If you do not have one you can make one with "mpg123 -w foo.wav foo.mp3" from foo.mp3.  For this example I assume that you have a suitable WAV file already called "/tmp/foo.wav".


 # cd /tmp
 # tar zxvf /usr/portage/distfiles/lame-3.96.1.tar.gz
 # cd lame-3.96.1
 # bench_gcc \
  --compile "./configure --disable-gtk; make" \
  --run "frontend/lame --preset standard /tmp/foo.wav foo.mp3 >&/dev/null" \
  --clean "make clean" \
  --average --iterate 3 --base "-O1" --peak "-O3" \
  --sizeof frontend/lame

This might give you output similar to the screen scrape below that shows a 9.87% speedup in runtime.

 -------------------------------------------------------------------------------------
 Starting base benchmark...
  - compiling... (log available in base/compile.log)
  ................................................................
  -> compile took 40.37 seconds.
  - starting runtime test... (log available in base/run.log.*)
   -> iteration 1...      run took 33.34 seconds.
   -> iteration 2...      run took 34.81 seconds.
   -> iteration 3...      run took 33.58 seconds.
   => average time:  33.91
  - cleaning up... (log available in base/clean.log)
 Starting peak benchmark...
  - compiling... (log available in peak/compile.log)
  ................................................................
  -> compile took 48.57 seconds.
  - starting runtime test... (log available in peak/run.log.*)
   -> iteration 1...      run took 30.92 seconds.
   -> iteration 2...      run took 30.96 seconds.
   -> iteration 3...      run took 30.72 seconds.
   => average time:  30.86
  - cleaning up... (log available in peak/clean.log)

 Stats Method:          average
 Number of iterations:  3

 Base Path:             default
 Base Options:          -O1
 Base Compile Time:     40.37 seconds
 Base Run Time:         33.91 seconds
 Base Code Size:        319191 bytes

 Peak Path:             default
 Peak Options:          -O3
 Peak Compile Time:     48.57 seconds
 Peak Run Time:         30.86 seconds
 Peak Code Size:        328284 bytes

 Diff Compile           8.20 seconds  (16.88% increase)
 Diff Run:              3.05 seconds  (-9.87% decrease)
 Diff Code Size:        -9093 bytes (2.77% increase)
 -------------------------------------------------------------------------------------


Or perhaps you want to compare gcc 3.3 to gcc 3.4.  If you replace the last line in the above command with:

  --bestof --iterate 1 --base "-O2" --peak "-O2" --compiler_base /usr/i686-pc-linux-gnu/gcc-bin/3.3 --compiler_peak /usr/i686-pc-linux-gnu/gcc-bin/3.4

You might get something like this showing the expected increase in compile time and a very nice 15% decrease in run time with the same flags.

 ------------------------------------------------------------------------------------
 Starting base benchmark...
  - compiling... (log available in base/compile.log)
  ................................................................
  -> compile took 49.27 seconds.
  - starting runtime test... (log available in base/run.log.*)
   -> iteration 1...      run took 36.53 seconds.
   => best time:  36.53
  - cleaning up... (log available in base/clean.log)
 Starting peak benchmark...
  - compiling... (log available in peak/compile.log)
  ................................................................
  -> compile took 36.10 seconds.
  - starting runtime test... (log available in peak/run.log.*)
   -> iteration 1...      run took 31.68 seconds.
   => best time:  31.68
  - cleaning up... (log available in peak/clean.log)

 Stats Method:          bestof
 Number of iterations:  1

 Base Path:             /usr/i686-pc-linux-gnu/gcc-bin/3.3/
 Base Options:          -O2
 Base Compile Time:     49.27 seconds
 Base Run Time:         36.53 seconds
 Base Code Size:        not_run

 Peak Path:             /usr/i686-pc-linux-gnu/gcc-bin/3.4/
 Peak Options:          -O2
 Peak Compile Time:     36.10 seconds
 Peak Run Time:         31.68 seconds
 Peak Code Size:        not_run

 Diff Compile           -13.17 seconds  (-36.47% decrease)
 Diff Run:              4.85 seconds  (-15.31% decrease)
 Diff Code Size:        not_run
 ------------------------------------------------------------------------------------

After running both of the above examples, you would have a gcc_bench.csv that looked like:

 -------------------------------------------------------------------------------------
 path,opts,compile_time,run_time,size,method,iterations
 default,--bypass -O1,37.92,69.41,319191,average,3
 default,--bypass -O3,47.81,64.46,328284,average,3
 /usr/i686-pc-linux-gnu/gcc-bin/3.3/,-O2,46.11,74.91,not_run,bestof,1
 /usr/i686-pc-linux-gnu/gcc-bin/3.4/,-O2,34.45,67.71,not_run,bestof,1
 -------------------------------------------------------------------------------------

=head1 NOTES ON STATISTICS

Good statistics are hard to come by.  Really.  This script measures
wall time of the run command.  Compile times are not expected to be
terribly accurate but you can get reasonable accuracy from the run
times but using more iterations.

=head1 AUTHOURS

Written by Patrick Audley <paudley@blackcat.ca> http://blackcat.ca

=head1 BUGS

Please send me any comments, bug reports, suggestions or flame mail :)

=head1 COPYRIGHT

Copyright 2003,2004 by Patrick Audley <paudley@blackcat.ca>

This program is licensed under the terms of the B<GPL>.  If you did not
recieve a copy of the license, you can get it from
http://www.gnu.org/licenses/gpl.html

=cut

#
# Detect Pod::Usage and use it if found, otherwise grep out the options form the script text
#

eval "use Pod::Usage;";
if( $@ ) {
  # Hmmm..  no Pod::Usage, do something sneaky.
  eval <<'POD_USAGE_FUDGE';
sub pod2usage {
  print STDERR "Failure to load Pod::Usage.  I'll try to run pod2text, if that fails, please read the code instead.\n\n";
  system("pod2text $0");
  exit(0);
}
POD_USAGE_FUDGE
}

my %cmdopts = (
	       "base=s"      => \$opts{base},
	       "peak=s"      => \$opts{peak},
	       "compile=s"   => \$opts{compile},
	       "clean=s"     => \$opts{clean},
	       "run=s"       => \$opts{run},
	       "iterate=i"   => \$opts{iterate},
	       "bestof!"     => sub { $opts{method} = "bestof"; },
	       "average!"    => sub { $opts{method} = "average"; },
	       "dotevery=i"  => \$opts{progressdots},
	       "sizeof=s"    => \$opts{sizeof},

	       "man"         => sub { pod2usage(-exitstatus=>0,-verbose=>2); },
	       "help"        => sub { pod2usage(-verbose=>1); },

	       "compiler_base=s" => \$opts{compbase},
	       "compiler_peak=s" => \$opts{comppeak},
	      );
pod2usage() unless GetOptions( %cmdopts ) && $opts{run} ne "000";

sub status {
  my $msg = shift;
  $msgio->print("$msg");
  $msgio->flush();
}

sub capture_command {
  my ( $cmd, $progress, $file ) = @_;
  open( CMD, $cmd."|" ) or die "Failed to start $cmd: ";
  open( LOG, ">".$file ) or die "Failed to open $file: ";
  my $i = 0;
  while( <CMD> ) {
    print LOG $_;
    $i++;
    status "." if( $progress && $i % $opts{progressdots} )
  }
  close( LOG );
  close( CMD );
  status "\n" if $progress;
}

my %bench = ();

sub bench_opts {
  my ( $cflags, $dir ) = @_;
  if( -d $dir ) {
    die "Can't remove the benchmark directory \"$dir\" because it wasn't make by me.  Please delete/move it and run this script again." unless -f "$dir/.bench_gcc";
    system( "rm -rf $dir" );
  }
  mkdir( "$dir" ) or die "Can't make benchmark directory \"$dir\": ";
  system( "touch $dir/.bench_gcc" );
  status "Starting $dir benchmark...\n";

  $ENV{"CFLAGS"} = $ENV{CXXFLAGS} = $cflags;

  $opts{"comp".$dir} .= "/" unless $opts{"comp".$dir} eq "";
  $ENV{"CC"} = $opts{"comp".$dir}."gcc";
  $ENV{"CXX"} = $opts{"comp".$dir}."g++";

  status " - compiling... (log available in $dir/compile.log)\n";
  my $start = time();
  capture_command( $opts{compile}, 1, "$dir/compile.log" );
  my $delta = time() - $start;
  $bench{$dir."_compile"} = $delta;
  status sprintf " -> compile took %.2f seconds.\n", $delta;

  # Handle code size stats
  $bench{$dir."_size"} = -1;
  if( $opts{sizeof} ne "" ) {
    die "The file you specified for size checking doesn't exist:  ".$opts{sizeof}."\n" unless -f $opts{sizeof};
    my $sb = stat( $opts{sizeof} );
    # Save a copy of the exe for later examination
    system( "cp ".$opts{sizeof}." $dir/sizeof" );
    $bench{$dir."_size"} = $sb->size;
  }

  my @times = ();
  status " - starting runtime test... (log available in $dir/run.log.*)\n";
  for( 1..$opts{iterate} ) {
    status "  -> iteration $_...  ";
    $start = time();
    capture_command( $opts{run}, 0, "$dir/run.log.$_" );
    $delta = time() - $start;
    status sprintf "    run took %.2f seconds.\n", $delta;
    push @times, $delta;
  }
  $delta = 0;
  if( $opts{method} eq "bestof" ) {
    $delta = min @times;
    status sprintf "  => best time:  %.2f\n", $delta;
  } elsif( $opts{method} eq "average" ) {
    $delta = ( sum @times ) / @times;
    status sprintf "  => average time:  %.2f\n", $delta;
  } else {
    confess "Unknown stats method.\n";
  }
  $bench{$dir."_run"} = $delta;

  status " - cleaning up... (log available in $dir/clean.log)\n";
  capture_command( $opts{clean}, 0, "$dir/clean.log" );

  open(REP,">$dir/report.txt") or die "Can't open $dir/report: ";
  print REP "Options: $cflags\n";
  print REP "Compile: ".$bench{$dir."_compile"}." seconds\n";
  print REP "Run: ".$bench{$dir."_run"}." seconds\n";
  print REP "Size: ".$bench{$dir."_size"}." bytes\n" if $opts{sizeof} ne "";
  close(REP);
}

bench_opts( $opts{base}, "base" );
bench_opts( $opts{peak}, "peak" );

$bench{delta_compile} =  $bench{peak_compile} - $bench{base_compile};
$bench{delta_run} = $bench{base_run} - $bench{peak_run};

for ( qw/ base_compile base_run peak_compile peak_run delta_compile delta_run / ) {
  $bench{"s_".$_} = sprintf "%.2f", $bench{$_};
}

sub format_perc {
  my ( $base, $peak ) = @_;
  my $perc = ( 100 - $base / $peak * 100 );
  my $trend = $perc > 0 ? "increase" : "decrease";
  return sprintf "%.2f%% %s", $perc, $trend;
}

$opts{basepath} = $opts{compbase} eq "" ? "default" : $opts{compbase};
$opts{peakpath} = $opts{comppeak} eq "" ? "default" : $opts{comppeak};

if( $opts{sizeof} eq "" ) {
  $bench{base_size} = "not_run";
  $bench{peak_size} = "not_run";
  $bench{delta_size} = "not_run";
} else {
  my $dcs = $bench{base_size} - $bench{peak_size};
  $bench{delta_size} = "$dcs bytes (".( format_perc $bench{base_size}, $bench{peak_size} ).")";
  $bench{base_size} .= " bytes";
  $bench{peak_size} .= " bytes";
}

my $rep = "
Stats Method:          $opts{method}
Number of iterations:  $opts{iterate}

Base Path:             $opts{basepath}
Base Options:          $opts{base}
Base Compile Time:     $bench{s_base_compile} seconds
Base Run Time:         $bench{s_base_run} seconds
Base Code Size:        $bench{base_size}

Peak Path:             $opts{peakpath}
Peak Options:          $opts{peak}
Peak Compile Time:     $bench{s_peak_compile} seconds
Peak Run Time:         $bench{s_peak_run} seconds
Peak Code Size:        $bench{peak_size}

Diff Compile           $bench{s_delta_compile} seconds  (".( format_perc $bench{base_compile}, $bench{peak_compile} ).")
Diff Run:              $bench{s_delta_run} seconds  (".( format_perc $bench{base_run}, $bench{peak_run} ).")
Diff Code Size:        $bench{delta_size}

";
print $rep;

open( REP, ">>bench_gcc.log" ) or die "Can't open bench_gcc.log: ";
print REP $rep;
close( REP );

my $header = -f "bench_gcc.csv";
open( CSV, ">>bench_gcc.csv" ) or die "Can't open bench_gcc.csv: ";
print CSV "path,opts,compile_time,run_time,size,method,iterations\n" unless $header;
for ( qw/ base peak / ) {
  print CSV $opts{$_."path"}.",".$opts{$_}.",".$bench{"s_".$_."_compile"}.",".$bench{"s_".$_."_run"}.",".$bench{$_."_size"}.",".$opts{method}.",".$opts{iterate}."\n";
}
close( CSV );