From 877a2a0a7e9441e6d8e3f34df83a86032af54098 Mon Sep 17 00:00:00 2001 From: Andrea Arteaga Date: Wed, 15 Feb 2012 02:13:15 +0000 Subject: Rewritten code... New XML parser, better modules. Not completely tested! --- numbench/PortageUtils.py | 77 ++++---- numbench/__init__.py | 1 - numbench/basemodule.py | 23 ++- numbench/benchchildren.py | 30 ++++ numbench/benchconfig.py | 95 ++++------ numbench/blas_accuracy.py | 4 +- numbench/btlbase.py | 4 +- numbench/confinput/__init__.py | 11 ++ numbench/confinput/oldinput.py | 69 ++++++++ numbench/confinput/xmlinput.py | 144 +++++++++++++++ numbench/lapack_accuracy.py | 4 +- numbench/main.py | 299 ++++++++++---------------------- numbench/metis.py | 4 +- numbench/modules/__init__.py | 0 numbench/modules/blas.py | 48 +++++ numbench/modules/cblas.py | 48 +++++ numbench/modules/internal/__init__.py | 0 numbench/modules/internal/blasBase.py | 76 ++++++++ numbench/modules/internal/btlBase.py | 40 +++++ numbench/modules/internal/lapackBase.py | 56 ++++++ numbench/modules/lapack.py | 47 +++++ numbench/report.py | 81 +++++++++ numbench/utils/__init__.py | 0 numbench/utils/alternatives.py | 60 +++++++ numbench/utils/benchpkgconfig.py | 70 ++++++++ numbench/utils/btl.py | 240 +++++++++++++++++++++++++ 26 files changed, 1209 insertions(+), 322 deletions(-) create mode 100644 numbench/benchchildren.py create mode 100644 numbench/confinput/__init__.py create mode 100644 numbench/confinput/oldinput.py create mode 100644 numbench/confinput/xmlinput.py mode change 100755 => 100644 numbench/main.py create mode 100644 numbench/modules/__init__.py create mode 100644 numbench/modules/blas.py create mode 100644 numbench/modules/cblas.py create mode 100644 numbench/modules/internal/__init__.py create mode 100644 numbench/modules/internal/blasBase.py create mode 100644 numbench/modules/internal/btlBase.py create mode 100644 numbench/modules/internal/lapackBase.py create mode 100644 numbench/modules/lapack.py create mode 100644 numbench/report.py create mode 100644 numbench/utils/__init__.py create mode 100644 numbench/utils/alternatives.py create mode 100644 numbench/utils/benchpkgconfig.py create mode 100644 numbench/utils/btl.py diff --git a/numbench/PortageUtils.py b/numbench/PortageUtils.py index f4f6d64..abc2ce1 100644 --- a/numbench/PortageUtils.py +++ b/numbench/PortageUtils.py @@ -18,32 +18,37 @@ import commands as cmd import subprocess as sp import os, portage, shlex -from os.path import join as pjoin +from os.path import join as pjoin, dirname import benchutils class InstallException(Exception): - def __init__(self, command, logfile): + def __init__(self, package, command, logfile): + self.package = package self.command = command self.logfile = logfile def _getEnv(root='/', envAdds={}): - denv = os.environ.copy() + #denv = os.environ.copy() + denv = {} #PATH denv['PATH'] = ':'.join([pjoin(root, i) for i in ('bin', 'usr/bin')]) if os.environ.has_key('PATH'): denv['PATH'] += ':' + os.environ['PATH'] denv['ROOTPATH'] = denv['PATH'] + #LIBRARY_PATH denv['LIBRARY_PATH'] = ':'.join([pjoin(root, i) for i in \ ('usr/lib', 'usr/lib64', 'usr/lib32')]) if os.environ.has_key('LIBRARY_PATH'): denv['LIBRARY_PATH'] += ':' + os.environ['LIBRARY_PATH'] + #LD_LIBRARY_PATH denv['LD_LIBRARY_PATH'] = ':'.join([pjoin(root, i) for i in \ ('usr/lib', 'usr/lib64', 'usr/lib32')]) if os.environ.has_key('LD_LIBRARY_PATH'): denv['LD_LIBRARY_PATH'] += ':' + os.environ['LD_LIBRARY_PATH'] + #INCLUDE_PATH denv['INCLUDE_PATH'] = ':'.join([pjoin(root, i) for i in ('usr/include',)]) if os.environ.has_key('INCLUDE_PATH'): @@ -55,7 +60,7 @@ def _getEnv(root='/', envAdds={}): return denv -def available_packages(pattern): +def availablePackages(pattern): """Returns a list of packages matching the given pattern. The packages are returned as (category, package, version, revision) tuple. @@ -72,14 +77,14 @@ def normalize_cpv(cpv): cpv_[-1] cpv = cpv_ except: - cpv = available_packages(cpv)[-1] + cpv = availablePackages(cpv)[-1] if cpv[-1] != 'r0': return '%s/%s-%s-%s' % cpv else: return '%s/%s-%s' % cpv[:-1] -def get_dependencies(package, env={}, split=False): +def getDependencies(package, env={}, split=False): pkg = normalize_cpv(package) cmd = ['emerge', '--ignore-default-opts', '='+pkg, '-poq'] proc = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE, env=env) @@ -94,24 +99,23 @@ def get_dependencies(package, env={}, split=False): else: return [shlex.split(l.strip())[-1] for l in lines] -def install_dependencies(package, env={}, root='/', - pkgdir='usr/portage/packages', logdir=None): - if logdir is None: - logdir = "/var/log/benchmarks/.unordered" - + +def installDependencies(test): # Adjust environment - denv = _getEnv(root, dict(PKGDIR=pkgdir)) + denv = _getEnv(test['root'], dict(PKGDIR=test['pkgdir'])) # Retrieve dependencies - deps = get_dependencies(package, denv, False) + deps = getDependencies(test['package'], denv, False) for i,d in enumerate(deps): - logfile = pjoin(logdir, 'emergedep_%i.log' % i) - install_package(d, env, root, pkgdir, logfile) + logfile = pjoin(test['logdir'], 'emergedep_%i.log' % i) + installPackage(test, package=d, env=test['dependenv'], logfile=logfile) -def install_package(package, env={}, root='/', pkgdir='usr/portage/packages', - logfile=None): +#def installPackage(package, env={}, root='/', pkgdir='usr/portage/packages', +# logfile=None): +def installPackage(test, package=None, env=None, logfile=None): + # TODO: rewrite docstring """Emerge a package in the given root. package is the package to be emerged. It has to be a tuple @@ -130,45 +134,46 @@ def install_package(package, env={}, root='/', pkgdir='usr/portage/packages', The function has no return value and raises an exception in case of building or emerging failure. Note: dependencies will NOT be emerged! """ + if package is None: + package = test['package'] + if env is None: + env = test['emergeenv'] + if logfile is None: + logfile = pjoin(test['logdir'], 'emerge.log') + envAdds = env.copy() - envAdds['PKGDIR'] = pkgdir - denv = _getEnv(root, envAdds) + envAdds['PKGDIR'] = test['pkgdir'] + denv = _getEnv(test['root'], envAdds) del envAdds # Retrieve package string pkg = normalize_cpv(package) # Execute emerge command and log the results - if logfile is not None: - fout = file(logfile, 'w') - fout.flush() - else: - fout = sp.PIPE + benchutils.mkdir(dirname(logfile)) + fout = file(logfile, 'w') cmd = ['emerge', '--ignore-default-opts', '-OB', '=' + pkg] p = sp.Popen(cmd, env=denv, stdout=fout, stderr=sp.STDOUT) p.wait() if p.returncode != 0: # In case of error, print the whole emerge command - raise InstallException(' '.join(cmd), logfile) + raise InstallException(p, ' '.join(cmd), logfile) + + fout.write('\n\n' + 80*'#' + '\n\n') # Unpack package onto root - benchutils.mkdir(pkgdir) - benchutils.mkdir(root) - archive = pjoin(pkgdir, pkg+'.tbz2') - tarcmd = ['tar', 'xjvf', archive, '-C', root] - cl = ' '.join(tarcmd) - if logfile is not None: - fout.write('\n\n' + 80*'#' + '\n\n') - fout.write(cl + '\n' + 80*'-' + '\n') + benchutils.mkdir(test['root']) + tarcmd = ['tar', 'xjvf', test['archive'], '-C', test['root']] + fout.write(' '.join(tarcmd) + '\n' + 80*'-' + '\n') p = sp.Popen(tarcmd, stdout=fout, stderr=sp.STDOUT) p.wait() if p.returncode != 0: # In case of error, print the whole emerge command - raise InstallException(cl, logfile) + raise InstallException(pkg, ' '.join(tarcmd), logfile) - if logfile is not None: - fout.close() + # Close, return + fout.close() if __name__ == '__main__': # Just a test diff --git a/numbench/__init__.py b/numbench/__init__.py index 8d1c8b6..e69de29 100644 --- a/numbench/__init__.py +++ b/numbench/__init__.py @@ -1 +0,0 @@ - diff --git a/numbench/basemodule.py b/numbench/basemodule.py index bf55981..b13b879 100644 --- a/numbench/basemodule.py +++ b/numbench/basemodule.py @@ -17,7 +17,7 @@ # from os.path import join as pjoin, basename, dirname import subprocess as sp -import shlex, os +import shlex, os, sys import shutil as shu import benchconfig as cfg @@ -25,15 +25,14 @@ from htmlreport import HTMLreport import basemodule from benchutils import mkdir, run_cmd from benchprint import Print -import benchpkgconfig as pc +import utils.benchpkgconfig as pc from testdescr import testdescr -import benchload as load try: if not locals().has_key('initialized'): initialized = True - import matplotlib - matplotlib.use('Agg') + import matplotlib + matplotlib.use('Agg') import matplotlib.pyplot as plt import numpy as np with_images = True @@ -214,30 +213,29 @@ class BaseTest: # Alternatives-2 version with pkg-config def _get_flags(self): # 1. Run with no requires - pfile = pc.GetFile(self.libname, self.impl, self.root) - flags = pc.Run(pfile, self.root, False) + pfile = pc.getFile(self.libname, self.impl, self.root) + flags = pc.run(pfile, self.root, False) logfile = file(pjoin(self.logdir, 'pkg-config.log'), 'w') print >> logfile, "File:", pfile print >> logfile, "Result:", flags # 2. Get requires - requires = pc.Requires(pfile) + requires = pc.requires(pfile) print >> logfile, "Requires:", requires # 3.Substitute requires and add flags for r in requires: if r in self.changes.keys(): - pfile = pc.GetFile(r, self.changes[r]) - flags += ' ' + pc.Run(pfile) + pfile = pc.getFile(r, self.changes[r]) + flags += ' ' + pc.run(pfile) else: - flags += ' ' + pc.Run(r) + flags += ' ' + pc.run(r) print >> logfile, "Third run:", flags logfile.close() return shlex.split(flags) def run_test(self, changes={}): - load.start() self.changes = changes # Convenient renames and definition of report files @@ -286,6 +284,5 @@ class BaseTest: Print("Test successful") # Return - load.stop() return self._generateResults() diff --git a/numbench/benchchildren.py b/numbench/benchchildren.py new file mode 100644 index 0000000..1686639 --- /dev/null +++ b/numbench/benchchildren.py @@ -0,0 +1,30 @@ +#===================================================== +# Copyright (C) 2011 Andrea Arteaga +#===================================================== +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +try: + copy = procs + del copy +except: + procs = [] + +def terminate(): + for p in procs: + if p.poll() is None: + p.kill() + +def append(proc): + procs.append(proc) \ No newline at end of file diff --git a/numbench/benchconfig.py b/numbench/benchconfig.py index 865dfcc..7191892 100644 --- a/numbench/benchconfig.py +++ b/numbench/benchconfig.py @@ -15,25 +15,22 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # -import sys, os, time +import sys, os, time, subprocess as sp from os.path import join as pjoin -import subprocess as sp -#from benchutils import mkdir -import benchutils as bu try: - needsinitialization = not initialized + needsinitialization = not initialized #@UndefinedVariable except NameError: needsinitialization = True if needsinitialization: + initialized = True + isroot = os.getuid() == 0 - try: - modname = sys.argv[1] - except: - modname = '' + modulename = sys.argv[1] + inputfile = os.path.realpath(sys.argv[2]) # Script directories curdir = os.path.abspath('.') @@ -43,66 +40,48 @@ if needsinitialization: else: btldir = '/usr/include/btl' - # Library directory (lib32 vs. lib64) + # Library directory (lib vs. lib32 vs. lib64) libdir = sp.Popen \ - ('ABI=$(portageq envvar ABI); echo /usr/`portageq envvar LIBDIR_$ABI`/', \ + ('ABI=$(portageq envvar ABI); echo `portageq envvar LIBDIR_$ABI`', \ stdout=sp.PIPE, shell=True).communicate()[0].strip() - - # roots, tests, packages directories -- report, logs base directories - if isroot: - testsdir = "/var/tmp/benchmarks/tests/" - rootsdir = "/var/tmp/benchmarks/roots/" - pkgsdir = "/var/cache/benchmarks/packages/" - - reportdirb = "/var/cache/benchmarks/reports/" - logdirb = "/var/log/benchmarks/"+modname+"_"+time.strftime('%Y-%m-%d') + if not libdir: + libdir = '/usr/lib' else: - testsdir = os.environ['HOME'] + "/.benchmarks/tests/" - rootsdir = os.environ['HOME'] + "/.benchmarks/roots/" - pkgsdir = os.environ['HOME'] + "/.benchmarks/packages/" - reportdirb = os.environ['HOME'] + "/.benchmarks/reports/" - logdirb = pjoin(os.environ['HOME'], ".benchmarks/log/") - - # Report directory - reportdirb += modname + "_" + time.strftime('%Y-%m-%d') - if os.path.exists(reportdirb): + libdir = '/usr/' + libdir + while libdir[0] == '/': + libdir = libdir[1:] + + # Storage directories + basedirb = pjoin(os.environ['HOME'], '.numbench') \ + + '/run_' + modulename + '_' + time.strftime('%Y-%m-%d') + if os.path.exists(basedirb): n = 1 while True: - reportdir = reportdirb + "_%i" % n - if not os.path.exists(reportdir): + basedir = basedirb + "_%i" % n + if not os.path.exists(basedir): break n += 1 else: - reportdir = reportdirb - del reportdirb + basedir = basedirb + del basedirb - # Logs directory - logdirb += modname + "_" + time.strftime('%Y-%m-%d') - if os.path.exists(logdirb): - n = 1 - while True: - logdir = logdirb + "_%i"%n - if not os.path.exists(logdir): - break - n += 1 - else: - logdir = logdirb - del logdirb + testsdir, rootsdir, pkgsdir, reportdir, logdir = tuple([pjoin(basedir, i) \ + for i in ('tests', 'roots', 'packages', 'report', 'log')]) -def makedirs(): - bu.mkdir(rootsdir) - bu.mkdir(testsdir) - bu.mkdir(pkgsdir) - bu.mkdir(reportdir) - bu.mkdir(logdir) -def purgedirs(): - bu.rmdir(rootsdir) - bu.rmdir(testsdir) - bu.rmdir(pkgsdir) - bu.rmdir(pjoin(reportdir, '..')) - bu.rmdir(pjoin(logdir, '..')) +#def makedirs(): +# bu.mkdir(rootsdir) +# bu.mkdir(testsdir) +# bu.mkdir(pkgsdir) +# bu.mkdir(reportdir) +# bu.mkdir(logdir) +# +#def purgedirs(): +# bu.rmdir(rootsdir) +# bu.rmdir(testsdir) +# bu.rmdir(pkgsdir) +# bu.rmdir(pjoin(reportdir, '..')) +# bu.rmdir(pjoin(logdir, '..')) -inizialized = True diff --git a/numbench/blas_accuracy.py b/numbench/blas_accuracy.py index 9996b89..f194d31 100644 --- a/numbench/blas_accuracy.py +++ b/numbench/blas_accuracy.py @@ -24,7 +24,7 @@ from benchprint import Print from htmlreport import HTMLreport import basemodule import benchconfig as cfg -import benchchilds +import benchchildren class Module(basemodule.BaseModule): @@ -141,7 +141,7 @@ class BLAS_accuracyTest(basemodule.BaseTest): logfile.write(80*'-' + '\n') proc = sp.Popen(args, bufsize=1, stdout=sp.PIPE, stderr=sp.PIPE, env=self.runenv, cwd=self.testdir) - benchchilds.append(proc) + benchchildren.append(proc) # Interpret output Print.down() diff --git a/numbench/btlbase.py b/numbench/btlbase.py index 5fac13a..23f1c51 100644 --- a/numbench/btlbase.py +++ b/numbench/btlbase.py @@ -24,7 +24,7 @@ from benchprint import Print from htmlreport import HTMLreport import basemodule import benchconfig as cfg -import benchchilds +import benchchildren from testdescr import testdescr class BTLBase(basemodule.BaseModule): @@ -153,7 +153,7 @@ class BTLTest(basemodule.BaseTest): logfile.write(80*'-' + '\n') proc = sp.Popen(args, bufsize=1, stdout=sp.PIPE, stderr=sp.PIPE, env=self.runenv, cwd=self.testdir) - benchchilds.append(proc) + benchchildren.append(proc) # Interpret output while True: diff --git a/numbench/confinput/__init__.py b/numbench/confinput/__init__.py new file mode 100644 index 0000000..4fbee33 --- /dev/null +++ b/numbench/confinput/__init__.py @@ -0,0 +1,11 @@ +from os.path import basename + +def parseInput(fname): + term = basename(fname).rsplit('.')[-1] + + if term.lower() == 'xml': + import xmlinput as parser + else: + import oldinput as parser + + return parser.parseConf(fname) \ No newline at end of file diff --git a/numbench/confinput/oldinput.py b/numbench/confinput/oldinput.py new file mode 100644 index 0000000..8d903c0 --- /dev/null +++ b/numbench/confinput/oldinput.py @@ -0,0 +1,69 @@ +def readEnvFile(fname): + """Reads a bash file with void environment and returns the environment + at the end of the execution.""" + proc = sp.Popen('. '+fname+' &> /dev/null; env', \ + shell=True, stdout=sp.PIPE, env={}) + lines = proc.stdout.read().split('\n')[:-1] + env = dict([l.split('=', 1) for l in lines]) + + for k in ('SHLVL', 'PWD', '_'): + if env.has_key(k): + del env[k] + return env + +def parseConf(fname): + input = file(fname).read() + + tests = {} + for line in input.split('\n'): + line = line.strip() + spl = [i.strip() for i in shlex.split(line)] + if len(spl) < 2: + continue + if line[0] == '#': + continue + env = {} + skip = [] + change = {} + descr = None + fileenv = {} + + # Interpret arguments + for var in spl[2:]: + + # if begins with '-': skip implementation + if var[0] == '-': + skip.append(var[1:]) + + # if key:value, substitute pkg-config dependency + elif ':' in var and not '=' in var: + c_0, c_1 = var.split(':', 1) + change[c_0] = c_1 + + # if descr|text set description (for future use) + elif var[:6] == 'descr|': + descr = var[6:] + + # if @file: read bash script and set env + elif var[0] == '@': + fileenvNew = readEnvFile(pjoin(cfg.curdir, var[1:])) + fileenv = dict( fileenv.items() + fileenvNew.items() ) + del fileenvNew + + # Otherwise, assume key=value syntax + else: + e_0, e_1 = var.split('=', 1) + env[e_0] = e_1 + + # Set environment (argument overrides bash file) + env = dict( fileenv.items() + env.items() ) + + try: + # Insert test + avail = available_packages(spl[1])[-1] + tests[spl[0]] = {'package':avail , 'emergeenv':env, 'skip':skip, \ + 'requires':change, 'descr':descr} + except: + # Or trigger an non-fatal error + sys.stderr.write('Error: package ' + spl[1] + ' not found\n') + return tests \ No newline at end of file diff --git a/numbench/confinput/xmlinput.py b/numbench/confinput/xmlinput.py new file mode 100644 index 0000000..4d4516c --- /dev/null +++ b/numbench/confinput/xmlinput.py @@ -0,0 +1,144 @@ +import xml.dom.minidom +import os, portage, types +import subprocess as sp +from os.path import join as pjoin, dirname as pdirname, realpath as prealpath + +from .. import benchconfig as cfg +from .. import PortageUtils as pu + + +def readFile(fs): + result = {} + + # If fs is a filename, open it + if type(fs) != types.FileType: + fs = file(pjoin(cfg.curdir, fs)) + + # Read line by line + for l in fs.readlines(): + try: + k,v = l.split('=', 1) + result[k.strip()] = v.strip() + except: + pass + + return result + + +def readScript(fname): + fname = pjoin(cfg.curdir, fname) + + # Execute script with void environment + proc = sp.Popen('. ' + fname + ' &> /dev/null; env', shell=True, + stdout=sp.PIPE, env={}) + result = readFile(proc.stdout) + + # Remove useless variables + for k in ('SHLVL', 'PWD', '_'): + if result.has_key(k): + del result[k] + return result + +def getEnvFromNode(node, envName): + envs = node.getElementsByTagName(envName) + + # Check number of envs + if len(envs) > 1: + errstr = "Error: no more than one " + envName + " element is allowed!" + raise Exception(errstr) + elif len(envs) < 1: + return {} + + e = envs[0] + + # Check attribute "append" + if (e.attributes.has_key('append')): + append = e.attributes['append'].value == '1' + else: + append = False + + if append: + env = os.environ + else: + env = {} + + # Check attribute script + # the script is run with a void environment + if (e.attributes.has_key('script')): + for k,v in readScript(e.getAttribute('script')).items(): + env[k] = v + + # Check attribute file + # the file must contain lines with key=value pairs (each line one pair) + if (e.attributes.has_key('file')): + for k,v in readFile(e.getAttribute('file')).items(): + env[k] = v + + # Get Variables + for v in e.getElementsByTagName('var'): + envname = v.getAttribute('name') + envvalue = v.firstChild.data + env[envname] = envvalue + + return env + + +def parseConf(fname): + testNodes = xml.dom.minidom.parse(fname).getElementsByTagName('test') + + tests = {} + + for t in testNodes: + tid = t.getAttribute('id') + + # Get description + descr = None + if len(t.getElementsByTagName('descr')) != 0: + descr = t.getElementsByTagName('descr')[0].firstChild.data + + # Get package + pkg = portage.catpkgsplit( + t.getElementsByTagName('pkg')[0].firstChild.data) + normPkg = pu.normalize_cpv(pkg) + + # Skip implementations + skip = [i.firstChild.data for i in t.getElementsByTagName('skip')] + + # Requirements + requires = {} + for i in t.getElementsByTagName('required'): + requires[i.getAttribute('name').strip()] = i.firstChild.data.strip() + + # Environments + dependenv = getEnvFromNode(t, 'dependenv') + emergeenv = getEnvFromNode(t, 'emergeenv') + compileenv = getEnvFromNode(t, 'compileenv') + runenv = getEnvFromNode(t, 'runenv') + + # Adjust PATH + if runenv.has_key('PATH'): + runenv['PATH'] += ':' + os.environ['PATH'] + else: + runenv['PATH'] = os.environ['PATH'] + + # Build test dictionary + tests[tid] = dict( + descr = descr, + package = pkg, + normalizedPackage = normPkg, + skip = skip, + requires = requires, + + dependenv = dependenv, + emergeenv = emergeenv, + compileenv = compileenv, + runenv = runenv, + + pkgdir = pjoin(cfg.pkgsdir, tid), + archive = pjoin(cfg.pkgsdir, tid, normPkg+'.tbz2'), + root = pjoin(cfg.rootsdir, tid), + testdir = pjoin(cfg.testsdir, tid), + logdir = pjoin(cfg.logdir, tid) + ) + + return tests \ No newline at end of file diff --git a/numbench/lapack_accuracy.py b/numbench/lapack_accuracy.py index f4737c0..3e04aa3 100644 --- a/numbench/lapack_accuracy.py +++ b/numbench/lapack_accuracy.py @@ -24,7 +24,7 @@ from benchprint import Print from htmlreport import HTMLreport import basemodule import benchconfig as cfg -import benchchilds +import benchchildren class Module(basemodule.BaseModule): @@ -146,7 +146,7 @@ class LAPACK_accuracyTest(basemodule.BaseTest): logfile.write(80*'-' + '\n') proc = sp.Popen(args, bufsize=1, stdout=sp.PIPE, stderr=logfile, env=self.runenv, cwd=self.testdir) - benchchilds.append(proc) + benchchildren.append(proc) # Interpret output Print.down() diff --git a/numbench/main.py b/numbench/main.py old mode 100755 new mode 100644 index 105a699..466d6ec --- a/numbench/main.py +++ b/numbench/main.py @@ -18,14 +18,12 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # -import os, sys, signal, shlex, shutil, time -from os.path import join as pjoin -import subprocess as sp +import os, sys, signal +import benchchildren # Set the signal handler def close(*args): - load.close() - benchchilds.terminate() + benchchildren.terminate() Print._level = 0 Print() Print(80*'-') @@ -36,8 +34,9 @@ signal.signal(signal.SIGINT, close) def print_usage(): - print "Usage: numbench [blas|cblas|lapack|scalapack|fftw|metis|" + \ - "blas_accuracy|lapack_accuracy] file args" + print "Usage: numbench [blas|cblas|lapack|scalapack|fftw|metis|" \ + "blas_accuracy|lapack_accuracy] file args" + def print_help(): print "Usage: numbench module conffile [options]" @@ -45,188 +44,73 @@ def print_help(): print " numbench module [ -h | --help ]" print print "Options:" - print " [ -p | --purge ] - Remove old results, logs, tests and packages" print " [ -h | --help ] - Display an help message" print print "Modules:" print " blas - Test BLAS implementations" print " cblas - Test CBLAS implementations" print " lapack - Test LAPACK implementations" - print " scalapack - Test the ScaLAPACK library" - print " blas_accuracy - Test BLAS implementations for accuracy" - print " lapack_accuracy - Test LAPACK implementations for accuracy" - print " fftw - Test the FFTW library" - print " metis - Test the METIS tools" + #print " scalapack - Test the ScaLAPACK library" + #print " blas_accuracy - Test BLAS implementations for accuracy" + #print " lapack_accuracy - Test LAPACK implementations for accuracy" + #print " fftw - Test the FFTW library" + #print " metis - Test the METIS tools" print print "More information about a module is available through the command:" print " numbench module --help" -def readEnvFile(fname): - """Reads a bash file with void environment and returns the environment - at the end of the execution.""" - proc = sp.Popen('. '+fname+' &> /dev/null; env', \ - shell=True, stdout=sp.PIPE, env={}) - lines = proc.stdout.read().split('\n')[:-1] - env = dict([l.split('=', 1) for l in lines]) - - for k in ('SHLVL', 'PWD', '_'): - if env.has_key(k): - del env[k] - return env - - -def tests_from_input(input): - tests = {} - for line in input.split('\n'): - line = line.strip() - spl = [i.strip() for i in shlex.split(line)] - if len(spl) < 2: - continue - if line[0] == '#': - continue - env = {} - skip = [] - change = {} - descr = None - fileenv = {} - - # Interpret arguments - for var in spl[2:]: - - # if begins with '-': skip implementation - if var[0] == '-': - skip.append(var[1:]) - - # if key:value, substitute pkg-config dependency - elif ':' in var and not '=' in var: - c_0, c_1 = var.split(':', 1) - change[c_0] = c_1 - - # if descr|text set description (for future use) - elif var[:6] == 'descr|': - descr = var[6:] - - # if @file: read bash script and set env - elif var[0] == '@': - fileenvNew = readEnvFile(pjoin(cfg.curdir, var[1:])) - fileenv = dict( fileenv.items() + fileenvNew.items() ) - del fileenvNew - - # Otherwise, assume key=value syntax - else: - e_0, e_1 = var.split('=', 1) - env[e_0] = e_1 - - # Set environment (argument overrides bash file) - env = dict( fileenv.items() + env.items() ) - - try: - # Insert test - avail = available_packages(spl[1])[-1] - tests[spl[0]] = {'package':avail , 'env':env, 'skip':skip, \ - 'changes':change, 'descr':descr} - except: - # Or trigger an non-fatal error - sys.stderr.write('Error: package ' + spl[1] + ' not found\n') - return tests +def loadModule(modulename): + tmp = __import__('numbench.modules.'+modulename, fromlist = ['Module']) +# try: +# tmp = __import__('numbench.modules.'+modulename, fromlist = ['Module']) +# except ImportError as e: +# sys.stderr.write('Module ' + modulename + ' not found') +# exit(1) + + return tmp -########################## -# HERE BEGINS THE SCRIPT # -########################## -import benchconfig as cfg -import benchchilds -import benchutils as bu +## PRINT HELP IF NEEDED -# If no argument is given, print the help -if (len(sys.argv) < 2): +# If no argument or '-h' is given, print the help +if len(sys.argv) < 3 or sys.argv[1] in ('-h', '--help'): print_help() exit(0) +# If requested, print the module help +if sys.argv[2] in ('-h', '--help'): + tmp = loadModule(sys.argv[1]) + tmp.Module.printHelp() + exit(0) -# Import the desired module or print help and exit -try: - - # Print main help - if (sys.argv[1] in ('-h', '--help')): - print_help() - exit(0); - - cfg.modulename = sys.argv[1] - - # Print module help - if (sys.argv[2] in ('-h', '--help')): - cfg.inputfile = '' - tmp = __import__('numbench.'+cfg.modulename, fromlist = ['Module']) - tmp.Module.printHelp() - exit(0) - - # Normal run: import module - - # Catch command-line arguments - passargs = [] - purge = False - for v in sys.argv[3:]: - if v in ('-p', '--purge'): - purge = True - else: - passargs.append(v) - - cfg.inputfile = os.path.abspath(sys.argv[2]) - os.chdir(cfg.scriptdir) - tmp = __import__('numbench.'+cfg.modulename, fromlist = ['Module']) - mod = tmp.Module(passargs) - del tmp - if purge: - cfg.purgedirs() - cfg.makedirs() - -except ImportError as e: - print e - print_usage() - exit(1) -except IndexError: - print_usage() - exit(1) -from PortageUtils import * +## BEGIN THE TRUE SCRIPT + +# Import the packages +from os.path import join as pjoin +import benchconfig as cfg, benchutils as bu, confinput from benchprint import Print +import PortageUtils as pu +import report +#from PortageUtils import \ +# normalize_cpv, install_dependencies, install_package, InstallException -""" -The test is the main configuration variable. Every entry in this dictionary -represents a package that has to be tested with his special environment, -which can contain information about the compiler, the flags,... -The dictionary key (e.g. "abcde" here) is just an identification method for -the test; it is safe to generate a random key, with some attention to avoid -overlapping keys. -Every entry (which is a dictionary itself) has to contain the item "package" in -the form of a tuple (category, package, version, revision) [see -portage.catpkgsplit], and the item "env", which describes the environment to be -used at compile-time as dictionary (it can just be a void one). -After the tests every successful tested item will contain the item "result", -which can contain any type of data and will be used for the final report. -""" - - -""" -The test variable is generated from a string which can be read from the file. -Here is an example of the parsed input. -Every line contains a configuration and will be an entry in the tests -dictionary; the line has to contain: -- an identification string -- a package description, which can, but does not must to, contain a version -- a list of environment variables separated by means of spaces -""" +# Parse the configuration file if not os.path.exists(cfg.inputfile): sys.stderr.write("File not found: " + cfg.inputfile) print_usage() exit(1) -input = file(cfg.inputfile).read() -cfg.tests = tests_from_input(input) +cfg.tests = confinput.parseInput(cfg.inputfile) + +# Import the module +#os.chdir(cfg.scriptdir) +mod = loadModule(cfg.modulename).Module(sys.argv[3:]) +cfg.mod = mod + # Write summary Print._level = 0 @@ -236,19 +120,37 @@ Print("-------------------------------") Print() for tname, ttest in cfg.tests.items(): Print("Test: " + tname) + if ttest['descr'] is not None: Print(" - Description: " + ttest['descr']) - Print(" - Package: " + normalize_cpv(ttest['package'])) - if len(ttest['env']) != 0: - Print(" - Environment: " + \ - ' '.join([n+'="'+v+'"' for n,v in ttest['env'].items()])) + + Print(" - Package: " + ttest['normalizedPackage']) + + if len(ttest['dependenv']) != 0: + Print(" - Dependencies emerge environment: " + \ + ' '.join([n+'="'+v+'"' for n,v in ttest['dependenv'].items()])) + + if len(ttest['emergeenv']) != 0: + Print(" - Emerge environment: " + \ + ' '.join([n+'="'+v+'"' for n,v in ttest['emergeenv'].items()])) + + if len(ttest['compileenv']) != 0: + Print(" - Suite compile-time environment: " + \ + ' '.join([n+'="'+v+'"' for n,v in ttest['compileenv'].items()])) + + if len(ttest['runenv']) != 0: + Print(" - Suite run-time environment: " + \ + ' '.join([n+'="'+v+'"' for n,v in ttest['runenv'].items()])) + if len(ttest['skip']) != 0: Print(" - Skip implementations: " + ' '.join(ttest['skip'])) - if len(ttest['changes']) != 0: - Print(" - Dependency substitutions:", '') - for c_0, c_1 in ttest['changes'].items(): + + if len(ttest['requires']) != 0: + Print(" - Pkg-config requirements substitutions:", '') + for c_0, c_1 in ttest['requires'].items(): Print(c_0 + ':' + c_1, '') Print() + Print() Print(80*'=') Print() @@ -259,46 +161,34 @@ Print("The logs will be available in the directory " + cfg.logdir) Print("The results will be available in the directory " + cfg.reportdir) Print() + +# Main iteration for tn,(name,test) in enumerate(cfg.tests.items(),1): Print._level = 0 Print("BEGIN TEST %i - %s" % (tn, name)) - pkgdir = pjoin(cfg.pkgsdir, name) - root = pjoin(cfg.rootsdir, name) - tlogdir = pjoin(cfg.logdir, name) - os.path.exists(tlogdir) or os.makedirs(tlogdir) - # Emerge package Print.down() - package = normalize_cpv(test['package']) - archive = pjoin(pkgdir, package+".tbz2") - test['pkgdir'] = pkgdir - test['archive'] = archive - if os.path.exists(archive): + if os.path.exists(test['archive']): Print("Package already emerged - skipping") test['emergesuccess'] = True else: try: # Emerge dependencies Print("Emerging dependencies") - install_dependencies( \ - test['package'], root=root, pkgdir=pkgdir, \ - logdir=tlogdir) + pu.installDependencies(test) # Emerge pacakge - Print("Emerging package %s" % package) - logfile = pjoin(tlogdir, 'emerge.log') - Print("(Run 'tail -f " + logfile + "' on another terminal" \ - + " to see the progress)") - install_package( \ - test['package'], env=test['env'], root=root, pkgdir=pkgdir, \ - logfile=logfile - ) + Print("Emerging package %s" % test['normalizedPackage']) + logfile = pjoin(test['logdir'], 'emerge.log') + Print("(Run 'tail -f " + logfile + "' on another terminal " + "to see the progress)") + pu.installPackage(test) test['emergesuccess'] = True - except InstallException as e: + except pu.InstallException as e: test['emergesuccess'] = False - Print("Package %s failed to emerge" % package) + Print("Package %s failed to emerge" % e.package) Print("Error log: " + e.logfile) Print.up() print @@ -306,7 +196,7 @@ for tn,(name,test) in enumerate(cfg.tests.items(),1): Print("Package emerged") # Find implementations - impls = [i for i in mod.get_impls(root) if not i in test['skip']] + impls = [i for i in mod.getImplementations(test) if not i in test['skip']] test['implementations'] = impls # Test every implementation @@ -314,27 +204,24 @@ for tn,(name,test) in enumerate(cfg.tests.items(),1): if len(impls) == 0: Print("No implementation found") for impl in impls: + # Run the test suite Print("Testing " + impl) Print.down() - - # Run the test suite - testdir = os.path.join(cfg.testsdir, name, impl) - t = mod.getTest(root, impl, testdir, logdir=tlogdir) - test['results'][impl] = t.run_test(test['changes']) + test['results'][impl] = mod.runTest(test, impl) Print.up() - + # All implementations tested + Print.up() print -# Save the results (first re-order them) -results = {} -for (name,test) in cfg.tests.items(): - if test.has_key('implementations'): - for impl in test['implementations']: - results[(name, impl)] = test['results'][impl] +# Save the results +report.saveReport() + + -mod.save_results(results) +# TODO: reintroduce the instructions feature (and remove "exit) +exit(0) Print._level = 0 @@ -348,7 +235,7 @@ for name,test in cfg.tests.items(): Print(len(printstr)*'-') Print.down() Print("# PKGDIR=" + test['pkgdir'] + " emerge -K '=" + \ - normalize_cpv(test['package']) + "'") + test['normalizedPackage'] + "'") try: for impl in test['implementations']: Print("Implementation " + impl + ":") diff --git a/numbench/metis.py b/numbench/metis.py index aad3e68..4c7c36b 100644 --- a/numbench/metis.py +++ b/numbench/metis.py @@ -23,7 +23,7 @@ import basemodule import benchconfig as cfg from benchutils import mkdir from benchprint import Print -import benchchilds +import benchchildren inputsdir = pjoin(cfg.testsdir, 'metis-input') mkdir(inputsdir) @@ -135,7 +135,7 @@ class MetisTest: logname = pjoin(self.logdir, t + '_%i.log' % size) cmd = [exe, inputfile, parts] pr = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.STDOUT, env=env) - benchchilds.append(pr) + benchchildren.append(pr) lines = pr.communicate()[0].split('\n') # Interpret output diff --git a/numbench/modules/__init__.py b/numbench/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/numbench/modules/blas.py b/numbench/modules/blas.py new file mode 100644 index 0000000..49aa31c --- /dev/null +++ b/numbench/modules/blas.py @@ -0,0 +1,48 @@ +#===================================================== +# Copyright (C) 2012 Andrea Arteaga +#===================================================== +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +import internal.blasBase as base + +class Module: + libname = "blas" + descr = "Test module for BLAS implementations" + + __init__ = base.init + getImplementations = base.getImplementations + runTest = base.runTest + getTests = base.getTests + reportConf = base.reportConf + + + +if __name__ == '__main__': + import os + import benchconfig as cfg + + +# m = Module(('1', 'matrix_matrix')) + m = Module(('axpy',)) + mytest = dict( + root='/home/spiros/packages/sci-libs/blasroot', + testdir='/home/spiros/tests/blas-reference', + logdir='/home/spiros/tests/log/blas-reference', + compileenv = {}, + runenv = {'PATH':os.environ['PATH']} + ) + cfg.libdir = 'usr/lib' + cfg.btldir = '/home/spiros/hg/btl' + print m.runTest(mytest, 'reference') \ No newline at end of file diff --git a/numbench/modules/cblas.py b/numbench/modules/cblas.py new file mode 100644 index 0000000..bdc17a5 --- /dev/null +++ b/numbench/modules/cblas.py @@ -0,0 +1,48 @@ +#===================================================== +# Copyright (C) 2012 Andrea Arteaga +#===================================================== +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +import internal.blasBase as base + +class Module: + libname = "cblas" + descr = "Test module for CBLAS implementations" + + __init__ = base.init + getImplementations = base.getImplementations + runTest = base.runTest + getTests = base.getTests + reportConf = base.reportConf + + + +if __name__ == '__main__': + import os + import benchconfig as cfg + + +# m = Module(('1', 'matrix_matrix')) + m = Module(('axpy',)) + mytest = dict( + root='/home/spiros/packages/sci-libs/cblasroot', + testdir='/home/spiros/tests/cblas-reference', + logdir='/home/spiros/tests/log/cblas-reference', + compileenv = {}, + runenv = {'PATH':os.environ['PATH']} + ) + cfg.libdir = 'usr/lib' + cfg.btldir = '/home/spiros/hg/btl' + print m.runTest(mytest, 'reference') \ No newline at end of file diff --git a/numbench/modules/internal/__init__.py b/numbench/modules/internal/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/numbench/modules/internal/blasBase.py b/numbench/modules/internal/blasBase.py new file mode 100644 index 0000000..34f2912 --- /dev/null +++ b/numbench/modules/internal/blasBase.py @@ -0,0 +1,76 @@ +#===================================================== +# Copyright (C) 2012 Andrea Arteaga +#===================================================== +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +import numbench.utils.btl as btl +import numbench.utils.alternatives as alt +import btlBase +from os.path import join as pjoin + + +avail1 = ('axpy', 'axpby', 'rot') +avail2 = ('matrix_vector','atv','symv', 'ger', 'syr2', 'trisolve_vector') +avail3 = ('matrix_matrix', 'aat', 'trisolve_matrix', 'trmm') +availableTests = avail1 + avail2 + avail3 +defaultTests = ('axpy', 'matrix_vector', 'trisolve_vector', 'matrix_matrix') + + +def init(self, args): + passargs = [] + tests = [] + + if len(args) == 0: + self.tests = defaultTests + return + + for i in args: + if i == '1': + tests += avail1 + continue + if i == '2': + tests += avail2 + continue + if i == '3': + tests += avail3 + continue + passargs.append(i) + + self.tests = btl.selectTests(availableTests, passargs+tests) + + +def getImplementations(self, test): + return alt.getImplementations(test['root'], self.libname) + + +def runTest(self, test, implementation): + # Set up btlconfig + btlconfig = dict ( + source = 'libs/BLAS/main.cpp', + exe = pjoin(test['testdir'], implementation, "test"), + logdir = pjoin(test['logdir'], implementation), + testdir = pjoin(test['testdir'], implementation), + btlincludes = ('libs/BLAS',), + defines = ("CBLASNAME="+self.libname, self.libname.upper()+"_INTERFACE"), + flags = alt.getFlags(test, self.libname, implementation), + tests = self.tests + ) + + return btlBase.runTest(test, btlconfig) + +getTests btlBase.getTests +reportConf = btlBase.reportConf + +reportConf = btlBase.reportConf diff --git a/numbench/modules/internal/btlBase.py b/numbench/modules/internal/btlBase.py new file mode 100644 index 0000000..63106b7 --- /dev/null +++ b/numbench/modules/internal/btlBase.py @@ -0,0 +1,40 @@ +#===================================================== +# Copyright (C) 2012 Andrea Arteaga +#===================================================== +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +import numbench.utils.btl as btl + +def reportConf(): + return {'type':'semilogx', 'xlabel':'size', 'ylabel':'MFlops'} + +def runTest(test, btlconfig): + + ret = btl.compileTest(test, btlconfig) + if ret != 0: + print "Compilation failed with code:", ret + else: + print "Compilation successful" + + ret, result = btl.runTest(test, btlconfig) + if ret != 0: + print "Execution failed with code:", ret + else: + print "Execution successful" + + return result + +def getTests(self): + return self.tests \ No newline at end of file diff --git a/numbench/modules/internal/lapackBase.py b/numbench/modules/internal/lapackBase.py new file mode 100644 index 0000000..c8098fc --- /dev/null +++ b/numbench/modules/internal/lapackBase.py @@ -0,0 +1,56 @@ +#===================================================== +# Copyright (C) 2012 Andrea Arteaga +#===================================================== +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +import numbench.utils.btl as btl +import numbench.utils.alternatives as alt +import btlBase +from os.path import join as pjoin + + +availableTests = ('general_solve', 'least_squares', 'lu_decomp', 'cholesky', \ + 'qr_decomp', 'svd_decomp', 'syev', 'stev', 'symm_ev') +defaultTests = ('lu_decomp', 'cholesky', 'qr_decomp', 'svd_decomp', 'syev') + + +def init(self, args): + if len(args) == 0: + self.tests = defaultTests + else: + self.tests = btl.selectTests(availableTests, args) + + +def getImplementations(self, test): + return alt.getImplementations(test['root'], self.libname) + + +def runTest(self, test, implementation): + # Set up btlconfig + btlconfig = dict ( + source = 'libs/LAPACK/main.cpp', + exe = pjoin(test['testdir'], implementation, "test"), + logdir = pjoin(test['logdir'], implementation), + testdir = pjoin(test['testdir'], implementation), + btlincludes = ('libs/BLAS', 'libs/LAPACK'), + defines = ("LAPACKNAME="+self.libname, ), + flags = alt.getFlags(test, self.libname, implementation), + tests = self.tests + ) + + return btlBase.runTest(test, btlconfig) + +getTests btlBase.getTests +reportConf = btlBase.reportConf \ No newline at end of file diff --git a/numbench/modules/lapack.py b/numbench/modules/lapack.py new file mode 100644 index 0000000..073e51a --- /dev/null +++ b/numbench/modules/lapack.py @@ -0,0 +1,47 @@ +#===================================================== +# Copyright (C) 2012 Andrea Arteaga +#===================================================== +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +import internal.lapackBase as base + +class Module: + libname = "lapack" + descr = "Test module for LAPACK implementations" + + __init__ = base.init + getImplementations = base.getImplementations + runTest = base.runTest + getTests = base.getTests + reportConf = base.reportConf + + +if __name__ == '__main__': + import os + import benchconfig as cfg + + +# m = Module(('1', 'matrix_matrix')) + m = Module(('lu_decomp', 'qr_decomp')) + mytest = dict( + root='/home/spiros/packages/sci-libs/lapackroot', + testdir='/home/spiros/tests/lapack-reference', + logdir='/home/spiros/tests/log/lapack-reference', + compileenv = {}, + runenv = {'PATH':os.environ['PATH']} + ) + cfg.libdir = 'usr/lib' + cfg.btldir = '/home/spiros/hg/btl' + print m.runTest(mytest, 'reference') \ No newline at end of file diff --git a/numbench/report.py b/numbench/report.py new file mode 100644 index 0000000..da1b5de --- /dev/null +++ b/numbench/report.py @@ -0,0 +1,81 @@ +import cfg +from os.path import join as pjoin, basename +import numpy as np + +from htmlreport import HTMLreport +from testdescr import testdescr +from benchprint import Print + +def saveReport(): + + # Check whether pyplot is working + try: + plt.figure() + except: + Print("Unable to generate plots") + Print("Please make sure that X is running and $DISPLAY is set") + return + + # Read configuration + conf = mod.reportConf() + + if conf['type'] == 'plot': + plotf = plt.plot + elif conf['type'] == 'semilogx': + plotf = plt.semilogx + elif conf['type'] == 'semilogy': + plotf = plt.semilogy + elif conf['type'] == 'loglog': + plotf = plt.loglog + + if conf.has_key('xlabel'): + xlabel = conf['xlabel'] + else: + xlabel = '' + + if conf.has_key('ylabel'): + ylabel = conf['ylabel'] + else: + ylabel = '' + + + # Open HTML file + htmlfname = pjoin(cfg.reportdir, 'index.html') + html = HTMLreport(htmlfname) + + for operation in cfg.mod.getTests(): + plt.figure(figsize=(12,9), dpi=300) + + for tid,test in cfg.tests.items(): + for impl in test['implementations']: + x,y = np.loadtxt(test['results'][impl][operation], unpack=True) + plotf(x, y, label=tid+'/'+impl, hold=True) + + plt.legend(loc='best') + plt.xlabel(xlabel) + plt.ylabel(ylabel) + plt.grid(True) + + fname = pjoin(cfg.reportdir, operation+'.png') + plt.savefig(fname, format='png', bbox_inches='tight', transparent=True) + html.addFig(testdescr[operation], image=basename(fname)) + + # Close HTML file + html.close() + Print('HTML report generated: ' + htmlfname) + + +# Initialize module +try: + if not locals().has_key('initialized'): + initialized = True + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + import numpy as np + with_images = True +except ImportError: + sys.stderr.write('Error: matplotlib and numpy are needed' + \ + 'in order to generate the reports!\n') + sys.stderr.write('Continue anyway.\n\n') + with_images = False \ No newline at end of file diff --git a/numbench/utils/__init__.py b/numbench/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/numbench/utils/alternatives.py b/numbench/utils/alternatives.py new file mode 100644 index 0000000..d69937b --- /dev/null +++ b/numbench/utils/alternatives.py @@ -0,0 +1,60 @@ +#===================================================== +# Copyright (C) 2012 Andrea Arteaga +#===================================================== +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +import subprocess as sp +import benchpkgconfig as pc +import shlex + +def getImplementations(root, libname): + cmd = ['eselect', '--no-color', '--brief', libname, 'list'] + env = dict(ROOT=root) + output = sp.Popen(cmd, env=env, stdout=sp.PIPE).communicate()[0].strip() + if '(none found)' in output: + return [] + else: + return [i.split()[0] for i in output.split('\n')] + + +def getFlags(test, libname, impl): + root = test['root'] + + # 1. Run without requires + pfile = pc.getFile(libname, impl, root) + flags = pc.run(pfile, root, False) + + # TODO: add log +# logfile = file(pjoin(self.logdir, 'pkg-config.log'), 'w') +# print >> logfile, "File:", pfile +# print >> logfile, "Result:", flags + + # 2. Get requires + requires = pc.requires(pfile) + # TODO: add log print >> logfile, "Requires:", requires + + # 3. Substitute requires and add flags + if test.has_key('requires'): + for r in requires: + if r in test['requires'].keys(): + pfile = pc.getFile(r, test['requires'][r]) + flags += ' ' + pc.run(pfile) + else: + flags += ' ' + pc.run(r) + # TODO: add log +# print >> logfile, "Third run:", flags +# logfile.close() + + return shlex.split(flags) \ No newline at end of file diff --git a/numbench/utils/benchpkgconfig.py b/numbench/utils/benchpkgconfig.py new file mode 100644 index 0000000..2ecc10a --- /dev/null +++ b/numbench/utils/benchpkgconfig.py @@ -0,0 +1,70 @@ +#===================================================== +# Copyright (C) 2011 Andrea Arteaga +#===================================================== +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +import os, types +from os.path import join as pjoin, basename, dirname +import subprocess as sp + +from .. import benchconfig as cfg + +def getFile(pfile, impl, roots='/'): + if pfile[-3:] != '.pc': + fname = pfile + '.pc' + else: + fname = pfile + + libdir = cfg.libdir + while libdir[0] == '/': + libdir = libdir[1:] + + # Check alternatives + if type(roots) in types.StringTypes: + roots = (roots, ) + pkgcfgpath = ':'.join([pjoin(r,'etc/env.d/alternatives', pfile, impl, \ + libdir, 'pkgconfig', fname) for r in roots]) + + if os.path.exists(pkgcfgpath): + return pkgcfgpath + else: + raise Exception('pkg-config file "' + pfile + '" not found', pkgcfgpath) + +def requires(fname): + env = {'PKG_CONFIG_PATH' : dirname(fname)} + cmd = ['pkg-config', '--print-requires', basename(fname)[:-3]] + proc = sp.Popen(cmd, env=env, stdout=sp.PIPE) + return proc.communicate()[0].split() + + +def run(fname, root='/', requires=True): + if not requires: + lines = file(fname, 'r').readlines() + newlines = [l for l in lines if l[:10] != 'Requires: '] + file(fname, 'w').writelines(newlines) + + bname = basename(fname) + if bname[-3:] == '.pc': + bname = bname[:-3] + + env = {'PKG_CONFIG_PATH' : dirname(fname), 'PKG_CONFIG_SYSROOT_DIR' : root} + cmd = ['pkg-config', '--libs', '--cflags', bname] + proc = sp.Popen(cmd, env=env, stdout=sp.PIPE) + out = proc.communicate()[0].strip() + + if not requires: + file(fname, 'w').writelines(lines) + + return out \ No newline at end of file diff --git a/numbench/utils/btl.py b/numbench/utils/btl.py new file mode 100644 index 0000000..edb3a18 --- /dev/null +++ b/numbench/utils/btl.py @@ -0,0 +1,240 @@ +#===================================================== +# Copyright (C) 2012 Andrea Arteaga +#===================================================== +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +from .. import benchchildren, benchutils as bu, benchconfig as cfg +from ..benchprint import Print + +from os.path import join as pjoin, dirname +import shlex, subprocess as sp + +# BTL global flags +btlincludes = ('actions','generic_bench','generic_bench/utils','libs/STL') +btllibraries = ('rt',) +btldefines = ('NDEBUG',) + + + +def compileTest(test, btlconfig): + + # Include directories + includes = [pjoin(cfg.btldir, i) for i in btlincludes] + if btlconfig.has_key('btlincludes'): + includes += [pjoin(cfg.btldir, i) for i in btlconfig['btlincludes']] + if btlconfig.has_key('includes'): + includes += btlconfig['includes'] + + # Linked libraries + libraries = list(btllibraries) + if btlconfig.has_key('libraries'): + libraries += btlconfig['libraries'] + + # Library directories + libdirs = [pjoin(test['root'], cfg.libdir)] + if btlconfig.has_key('libdirs'): + libdirs += btlconfig['libdirs'] + + # Defined preprocessor macros + defines = list(btldefines) + if btlconfig.has_key('defines'): + defines += btlconfig['defines'] + + # Other flags + flags = [] + + # Interpret flags + interpret = shlex.split(bu.run_cmd(['portageq', 'envvar', 'CXXFLAGS'])) + if btlconfig.has_key('flags'): + interpret += btlconfig['flags'] + for flag in interpret: + flag = flag.strip() + if flag[:2] == '-I': + includes.append(flag[2:]) + elif flag[:2] == '-l': + libraries.append(flag[2:]) + elif flag[:2] == '-L': + libdirs.append(flag[2:]) + else: + flags.append(flag) + del interpret + + + # Set compile-time environment + compileenv = test['compileenv'].copy() + + if compileenv.has_key('INCLUDE_PATH'): + compileenv['INCLUDE_PATH'] += ':' + ':'.join(includes) + else: + compileenv['INCLUDE_PATH'] = ':'.join(includes) + + libenv = ':'.join(libdirs) + if compileenv.has_key('LIBRARY_PATH'): + compileenv['LIBRARY_PATH'] += ':' + libenv + else: + compileenv['LIBRARY_PATH'] = libenv + + if compileenv.has_key('LD_LIBRARY_PATH'): + compileenv['LD_LIBRARY_PATH'] += ':' + libenv + else: + compileenv['LD_LIBRARY_PATH'] = libenv + + pathenv = ':'.join([pjoin(test['root'], l) for l in ('bin', 'usr/bin')]) + if compileenv.has_key('PATH'): + compileenv['PATH'] += pathenv + else: + compileenv['PATH'] = pathenv + + # Set run-time environment + runenv = test['runenv'].copy() + + if runenv.has_key('LD_LIBRARY_PATH'): + runenv['LD_LIBRARY_PATH'] += ':' + libenv + else: + runenv['LD_LIBRARY_PATH'] = libenv + + pathenv = ':'.join([pjoin(test['root'], l) for l in ('bin', 'usr/bin')]) + if runenv.has_key('PATH'): + runenv['PATH'] += pathenv + else: + runenv['PATH'] = pathenv + + btlconfig['runenv'] = runenv + + + # Set C++ compiler + cxx = '/usr/bin/g++' + + portageq_cxx = bu.run_cmd(['portageq', 'envvar', 'CXX']) + if portageq_cxx.strip() != "": + cxx = portageq_cxx + del portageq_cxx + + if btlconfig.has_key('CXX'): + cxx = btlconfig['CXX'] + + if compileenv.has_key('CXX'): + cxx = compileenv['CXX'] + + + # Form command-line arguments + args = [cxx, pjoin(cfg.btldir, btlconfig['source']), '-o', btlconfig['exe']] + args += ['-I'+I for I in includes] + args += ['-l'+l for l in libraries] + args += ['-L'+L for L in libdirs] + args += ['-D'+D for D in defines] + args += flags + + # Open logfile and write environment + bu.mkdir(btlconfig['logdir']) + logfile = pjoin(btlconfig['logdir'], "btlCompile.log") + logfs = file(logfile, 'w') + logfs.write('\n'.join([n+'='+v for n,v in compileenv.items()])) + logfs.write(3*'\n' + ' '.join(args) + 3*'\n') + logfs.flush() + + # Execute compilation + bu.mkdir(dirname(btlconfig['exe'])) + proc = sp.Popen(args, stdout=logfs, stderr=sp.STDOUT, env=compileenv) + proc.wait() + logfs.flush() + retcode = proc.returncode + if retcode == 0: + logfs.write("\n\n<<< Compilation terminated successfully >>>") + else: + logfs.write("\n\n<<< Compilation failed >>>") + + # Close, return + logfs.close() + return retcode + + +def runTest(test, btlconfig): + runenv = btlconfig['runenv'] + + # Check linking + logfs = file(pjoin(btlconfig['logdir'], 'btlLinking.log'), 'w') + sp.Popen(['ldd', '-v', btlconfig['exe']], stdout=logfs, env=runenv).wait() + logfs.close() + + + # Prepare arguments + args = (btlconfig['exe'],) + tuple(btlconfig['tests']) + if btlconfig.has_key('preargs'): + args = btlconfig['preargs'] + args + if btlconfig.has_key('postargs'): + args = args + btlconfig['postargs'] + + # Open log + logfs = file(pjoin(btlconfig['logdir'], "btlRun.log"), 'w') + logfs.write('\n'.join([n+'='+v for n,v in runenv.items()])) + logfs.write(3*'\n' + ' '.join(args) + 3*'\n') + logfs.flush() + + # Open pipe + proc = sp.Popen(args, bufsize=1, stdout=sp.PIPE, stderr=sp.PIPE, \ + env=runenv, cwd=btlconfig['testdir']) + benchchildren.append(proc) + + result = {} + + # Interpret output + Print('Begin execution') + while True: + # Each operation test begins with a line on stderr + errline = proc.stderr.readline() + if not errline: + break + logfs.write(errline) + + resfile = errline.split()[-1] + operation = resfile.split('_', 1)[-1].rsplit('_', 1)[0] + result[operation] = resfile + Print(operation + " -> " + resfile) + + + # Many different sizes for each operation test + Print.down() + cur = 0 + tot = 1 + while cur != tot: + outline = proc.stdout.readline() + # If the line is void, something gone wrong + if not outline: + Print.up() + Print('Execution error') + return 1 + logfs.write(outline) + logfs.flush() + + # Interpret line + outline = outline.strip() + (cur, tot) = shlex.split(outline)[-1][1:-1].split('/') + cur = int(cur); tot = int(tot) + Print(outline) + + + Print.up() + proc.wait() + Print("Execution finished with return code " + str(proc.returncode)) + + # Close, return + logfs.close() + return proc.returncode, result + + + +def selectTests(availableTests, args): + return tuple([i for i in availableTests if i in args]) -- cgit v1.2.3-65-gdbad