From 1144c1b3f9570b30e7313ef2326856836c0edf9f Mon Sep 17 00:00:00 2001
From: rjones <rjones@f3b2605a-c512-4ea7-a41b-209d697bcdaa>
Date: Wed, 21 Aug 2013 23:40:18 +0000
Subject: [PATCH] first drafts of benchmark,py - numerical comparison of log
files regression.sh - script for cron
git-svn-id: svn://svn.icms.temple.edu/lammps-ro/trunk@10640 f3b2605a-c512-4ea7-a41b-209d697bcdaa
---
regress/benchmark.py | 235 ++++++++++++++++++++++++++++++++++++++++++
regress/regression.sh | 11 ++
2 files changed, 246 insertions(+)
create mode 100755 regress/benchmark.py
create mode 100755 regress/regression.sh
diff --git a/regress/benchmark.py b/regress/benchmark.py
new file mode 100755
index 0000000000..209c91f8d9
--- /dev/null
+++ b/regress/benchmark.py
@@ -0,0 +1,235 @@
+#!/usr/bin/env python
+"""
+ function: numerical comparisions of logs and corresponding benchmarks
+ usage: benchmark.py <nprocs> <njobs> <dirs>
+"""
+import sys
+import os
+import math
+import re
+from operator import itemgetter
+from glob import glob
+import time
+import multiprocessing as mp
+try:
+ import Queue as queue # 2.6
+except ImportError:
+ import queue # 3.0
+
+#====================================================
+### constants
+#====================================================
+thermo_pattern = re.compile("^Step "); # fragile
+data_pattern = re.compile("\s*\d"); # fragile
+fail_pattern = re.compile("FAIL");
+tol = 1.e-6 # 1.e-10
+arch = "openmpi"
+src_path = "../src/" #relative to home
+exe_path = "../"+src_path
+
+#====================================================
+### date
+#====================================================
+def date():
+ return time.asctime()
+
+#====================================================
+### timer
+#====================================================
+## NOTE these don't seem to work how I expect them to
+def start():
+ global dt
+ dt = -(time.clock())
+def stop():
+ global dt
+ dt += (time.clock())
+ return dt
+
+#====================================================
+### run a benchmark
+#====================================================
+def run_test(test):
+ input = "in."+test;
+ log = "log."+test
+ stdout = "stdout."+test
+ ref = (glob(log+"*."+str(np)))[0];
+ msg = "==== comparing "+log+" with "+ref+" ====\n"
+ if (os.path.isfile(log)): os.remove(log)
+ if (os.path.isfile(stdout)): os.remove(stdout)
+ os.system(lmps+input+" >& "+stdout);
+ if (not os.path.isfile(log)) :
+ msg += "!!! no "+log+"\n";
+ msg += "!!! test "+test+" FAILED\n"
+ return msg
+ [cdict,cdata] = extract_data(log);
+ [bdict,bdata] = extract_data(ref);
+ cols = range(len(bdict))
+ if (len(cdata) != len(bdata)):
+ msg += "!!! data size "+str(len(cdata))+" does not match data "+str(len(bdata))+" in "+ref+"\n";
+ msg += "!!! test "+test+" FAILED\n"
+ return msg
+ fail = False
+ i = 0
+ for name in bdict:
+ [passing,cmsg] = compare(name,cdata[cols[i]],bdata[cols[i]]);
+ i += 1
+ msg += cmsg
+ if (not passing) : fail = True
+ if (fail) :
+ msg += "!!! test "+test+" FAILED\n"
+ else :
+ msg += "*** test "+test+" passed\n"
+ return msg
+
+#====================================================
+### extract data from log file
+#====================================================
+def extract_data(file):
+ dictionary = [];
+ data = []
+ read = False
+ for line in open(file):
+ if (read and data_pattern.match(line)) :
+ cols = line.split();
+ data.append(cols)
+ else :
+ read = False
+ if (thermo_pattern.match(line)):
+ dictionary = line.split();
+ read = True
+ return [dictionary,data]
+
+#====================================================
+### compare columns of current and benchmark
+#====================================================
+def compare(name,col1,col2):
+ err = 0.
+ norm1 = 0.
+ norm2 = 0.
+ n = len(col2)
+ for i in range(n):
+ v1 = float(col1[i])
+ v2 = float(col2[i])
+ norm1 += v1*v1
+ norm2 += v2*v2
+ dv = v1-v2
+ err += dv*dv
+ norm1 /= n
+ norm2 /= n
+ err /= n
+ if (norm2 > tol) :
+ msg = "{0:7s} relative error {1:4} wrt norm {2:7}\n".format(name,err,norm2)
+ else :
+ msg = "{0:7s} error {1:4}\n" .format(name,err)
+ return [(err < tol),msg];
+
+#################################################################
+class Worker(mp.Process):
+ def __init__(self, work_queue, result_queue):
+ mp.Process.__init__(self)
+ self.work_queue = work_queue
+ self.result_queue = result_queue
+ def run(self):
+ while True:
+ try:
+ job = self.work_queue.get_nowait()
+ except queue.Empty:
+ break
+ #print(">>> starting " + str(job[1]) + " ...")
+ os.chdir(job[0])
+ start()
+ msg = run_test(job[1])
+ elapsed_time = stop()
+ msg += "elapsed time "+str(elapsed_time)+"\n"
+ os.chdir(home)
+ self.result_queue.put([job[1],msg])
+
+#====================================================
+### parse
+#====================================================
+def init() :
+ global np, njobs, ntests, lmps, arch, home
+ home = os.getcwd()
+ if (len(sys.argv) < 4) :
+ print "usage: benchmark.py <nprocs> <njobs> <test_dirs>"
+ sys.exit(1)
+ np = int(sys.argv[1])
+ njobs = int(sys.argv[2])
+ lmps = "../"+src_path+"lmp_"+arch+" -in "
+ if (np > 1):
+ lmps = "mpirun -np "+str(np)+" "+lmps
+ else:
+ arch = "serial"
+ lmps = exe_path+"lmp_"+arch+" -in "
+ pool = mp.Pool(njobs)
+ dirs = sys.argv[3:]
+ tests = []
+ for dir in dirs:
+ os.chdir(dir);
+ for path in glob("./in.*"):
+ test = path[5:];
+ tests.append([dir,test])
+ os.chdir(home)
+ ntests = len(tests)
+ print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+ print "start: ",date()
+ print "arch:",arch,
+ print "nprocs:",np
+ print "ntests:",ntests,
+ print "njobs:",njobs
+ print "relative tolerance:",tol
+ print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+ print
+ return tests
+
+#====================================================
+### build executable
+#====================================================
+def build(arch):
+ os.system("cd ..; svn update >& svn_update.log")
+ os.system("cd ../src; make no-atc >& /dev/null")
+ os.system("cd ../src; make clean-all >& /dev/null")
+ #os.system("cd ../src; make yes-all >& /dev/null")
+ os.system("cd ../src; make yes-dipole >& /dev/null")
+ sys.stdout.flush()
+ print "** building ",arch,"...",
+ os.system("cd "+src_path+"; make -j "+str(np)+" "+arch+" >& build_"+arch+".log")
+ if (not os.path.isfile(src_path+"lmp_"+arch)) :
+ print "!!! build ",arch," FAILED"
+ sys.exit()
+ else:
+ print "done"
+ print
+
+#====================================================
+### main
+#====================================================
+if __name__ == '__main__':
+ tests = init()
+ build(arch)
+ work_queue = mp.Queue()
+ for test in tests:
+ work_queue.put(test)
+ result_queue = mp.Queue()
+ nfails = 0
+ fail_list = []
+ for i in range(njobs):
+ w = Worker(work_queue, result_queue)
+ w.start()
+ for i in range(ntests):
+ [test,msg] = result_queue.get()
+ if (fail_pattern.search(msg)) :
+ nfails += 1
+ fail_list.append(test)
+ #print msg # can print only if failed
+ print msg # can print only if failed
+ #print test, " passed"
+ print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+ print "end:",date()
+ if (nfails == 0):
+ print "*** no failures ***"
+ else :
+ print "!!!",nfails,"of",ntests,"tests failed"
+ for test in fail_list:
+ print test
+ print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
diff --git a/regress/regression.sh b/regress/regression.sh
new file mode 100755
index 0000000000..8aac9612fb
--- /dev/null
+++ b/regress/regression.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+cd /code/lammps-atc/regression
+./benchmark.py 4 12 min dipole >& latest
+fail=`grep -c FAIL latest`
+addrs="rjones@sandia.gov jatempl@sandia.gov jzimmer@sandia.gov sjplimp@sandia.gov akohlmey@gmail.com"
+if [ $fail == 0 ] ; then
+ mhmail $addrs -subject "LAMMPS regression passes" < latest
+else
+ mhmail $addrs -subject "LAMMPS regression $fail tests failed" < latest
+fi
+
--
GitLab