2. Timing script

     1: #line 483 "./lpsrc/flx_perf.pak"
     2: import os
     3: import time
     4: import sys
     5: import random
     6: import signal
     7: import socket
     8: import glob
     9: import string
    10: 
    11: # utility to make a CLEAN directory
    12: def mkdir(p):
    13:   os.system('rm -rf ' + p)
    14:   os.mkdir(p)
    15: 
    16: env = {}
    17: try:
    18:   execfile('config'+os.sep+'xlators_local.py')
    19: except:
    20:   selected_xlators = []
    21: 
    22: xlators = []
    23: for f in glob.glob("speed"+os.sep+"xlators"+os.sep+"*.py"):
    24:   execfile(f)
    25:   name = string.split(f,os.sep)[2][0:-3]
    26:   if name in selected_xlators or not selected_xlators:
    27:     xlators.append((name,mk,lang))
    28: 
    29: # set up any environment needed, this is a temporary hack
    30: for k in env:
    31:   os.putenv(k,env[k])
    32: 
    33: tests = {}
    34: for f in glob.glob("speed"+os.sep+"specs"+os.sep+"*.py"):
    35:   execfile(f)
    36:   base = string.split(f,os.sep)[2][:-3]
    37:   tests[base]=(descr,min,max)
    38: 
    39: 
    40: #------ command line arguments -----------------
    41: max_test_time = 5.0
    42: try: max_test_time = float(sys.argv[1])
    43: except:pass
    44: 
    45: # maximum allowed time per test, seconds
    46: maxtime = 5.0
    47: try: maxtime = float(sys.argv[2])
    48: except:pass
    49: 
    50: mintime = 0.5
    51: try: mintime = float(sys.argv[3])
    52: except:pass
    53: 
    54: gracetime = 10.0
    55: try: gracetime = float(sys.argv[4])
    56: except:pass
    57: 
    58: 
    59: process_count = 1
    60: try: process_count = int(sys.argv[5])
    61: except:pass
    62: 
    63: 
    64: # ------------ report paramaters ------------------
    65: print "Batch time limit",max_test_time,"minutes"
    66: print "Individual test soft seek range: ",mintime,'-',maxtime,"seconds"
    67: print "Grace before cancellation",gracetime,"seconds"
    68: print "Processes to run in parallel",process_count
    69: 
    70: # make the output directories
    71: mkdir('speed/exes')
    72: for key,mk,src in xlators:
    73:   print "Using translator",key
    74:   mkdir('speed/exes/'+key)
    75: 
    76: for t in tests.keys():
    77:   descr,min,max=tests[t]
    78:   print "Running test",t,"initial nrange="+str(min)+":"+str(max)
    79: 
    80: hostname = socket.gethostname()
    81: print "Host",hostname
    82: 
    83: #--------- setup -----------------------------
    84: ntests = len(tests)
    85: 
    86: #convert to seconds
    87: max_test_time = max_test_time * 60.0
    88: 
    89: sleep_time = maxtime+gracetime
    90: 
    91: 
    92: print "COMPILING PROGRAMS"
    93: #compile the programs
    94: wxlats = []
    95: for xl,mk,src in xlators:
    96:   for test in tests.keys():
    97:     descr,fst,last = tests[test]
    98:     cmd = mk(xl,test)
    99:     print cmd
   100:     # won't work on Win98, result always 0 for command.com
   101:     result = os.system(cmd)
   102:     if result <> 0:
   103:       print "COMPILE FAILURE",xl,src,test
   104:     else:
   105:       print "Compile OK",xl,src,test
   106:       wxlats.append((xl,mk,src,test))
   107: 
   108: nwxlats = len(wxlats)
   109: 
   110: for xlat,f,src,tst in wxlats:
   111:   print xlat,src,tst
   112: 
   113: # fork extra copies of ourself now
   114: 
   115: tofork = process_count - 1
   116: while tofork > 0:
   117:   if os.fork() == 0:
   118:     print "PROCESS",tofork,"FORKED"
   119:     random.seed() # reseed the random number generator
   120:     break
   121:   tofork = tofork - 1
   122: 
   123: # -------------- run -----------------------
   124: #total time for testsing, seconds:
   125: start_time = time.time()
   126: 
   127: #test date
   128: etime = time.time()
   129: ltime = time.localtime(etime)
   130: date = time.strftime("%Y/%m/%d %H:%M",ltime)
   131: 
   132: pid = 0
   133: spid = 0
   134: xlt = {}
   135: 
   136: f = open("speed/results.dat","at")
   137: while time.time() - start_time < max_test_time:
   138:   # pick a random test and a random translator
   139:   ix = random.randint(0,nwxlats-1)
   140:   xl,mk,src,test = wxlats[ix]
   141:   descr,fst,lst = tests[test]
   142:   fst,lst = xlt.get((test,xl), (fst,lst))
   143:   xlt[(test,xl)] = (fst,lst)
   144: 
   145:   # pick an integer between fst and lst inclusive
   146:   n = random.randint(fst,lst)
   147: 
   148:   test_file = "speed/exes/"+xl+"/"+test
   149:   test_arg = "%d" % n
   150: 
   151:   print test_file,test_arg
   152:   start = time.time()
   153:   pid = os.spawnl(os.P_NOWAIT,test_file,"DUMMY",test_arg)
   154:   spid = os.spawnlp(os.P_NOWAIT,"sleep","DUMMY","%d" % int(sleep_time))
   155:   pidx,status = os.wait()
   156:   finish = time.time()
   157:   elapsed = finish - start
   158: 
   159:   if pidx == spid:
   160:     os.kill(pid,signal.SIGKILL)
   161:     print "TIMEOUT"
   162:     assert (elapsed > maxtime)
   163:   else:
   164:     os.kill(spid,signal.SIGKILL)
   165:   os.wait()
   166: 
   167: 
   168:   if  pidx == pid:
   169:     signalled = os.WIFSIGNALED(status)
   170:     exited = os.WIFEXITED(status)
   171:     if not (signalled or exited):
   172:       print "WHAT?? Neither exited nor signalled?"
   173:       exit(2)
   174: 
   175:     # Killed by signal, possibly timeout, no result
   176:     # can be recorded
   177:     if signalled:
   178:       sig = os.WTERMSIG(status)
   179:       if sig == signal.SIGINT:
   180:         raise KeyboardInterrupt
   181: 
   182:       if sig == signal.SIGSEGV:
   183:         print "SEGMENTATION FAULT, TERMINATING"
   184:         sys.exit(1)
   185: 
   186:       if sig != 0:
   187:         print "UNKNOWN SIGNAL",sig,": TERMINATING"
   188:         sys.exit(1)
   189:     else:
   190:       # the process terminated correctly, record a result even
   191:       # if it is out of range
   192:       if status == 0:
   193:         x = hostname + " " + "p"+str(process_count)+" "+date + " %s %s %d %6.4f" % (xl,test,n,elapsed)
   194:         print "#"+str(tofork),x
   195:         f.write(x+"\n")
   196:         f.flush()
   197:       else:
   198:         # the process terminated by gave an error
   199:         # exit because this should not happen
   200:         ret = os.WEXITSTATUS(status)
   201:         if ret != 0:
   202:           print "TEST RETURNED ERROR CODE ",ret,": TERMINATING"
   203:           sys.exit(1)
   204:         else:
   205:           print "WHAT?? Exit code is 0 and not 0?"
   206:           sys.exit(2)
   207: 
   208:   # overtime
   209:   if elapsed > maxtime:
   210:     lst = n - 1
   211:     if lst < fst: fst = lst
   212:     print 'OVERTIME: CUT max to',lst
   213: 
   214:   # undertime
   215:   elif elapsed < mintime:
   216:     fst = n + 1
   217:     if lst < fst: lst = fst
   218:     print 'UNDERTIME: BOOSTED min to',fst
   219: 
   220:   #minimum n too high
   221:   elif n == fst:
   222:     fst = fst - 1
   223:     if lst < fst: lst = fst
   224:     print "BORDERLINE min: CUT min down to",fst
   225: 
   226:   # maximum n too low
   227:   elif n == lst:
   228:     lst = lst + 1
   229:     if lst < fst: fst = lst
   230:     print "BORDERLINE max: BOOSTED max up to",lst
   231: 
   232:   if lst < 1: lst = 1
   233:   if fst < 1: fst = 1
   234:   if fst > lst: fst = lst
   235:   xlt[test,xl] = (fst,lst)
   236: 
   237: f.close()
   238: 

     1: #line 721 "./lpsrc/flx_perf.pak"
     2: # generate analysis
     3: # sort data by host, test, processor, n value
     4: import sys
     5: import os
     6: import string
     7: import math
     8: hostsd = {}
     9: 
    10: args = sys.argv
    11: args = args[1:]
    12: mintime = 0.5
    13: mindata = 3
    14: pload = "p1"
    15: 
    16: try:
    17:   mintime = float(args[0])
    18: except:pass
    19: 
    20: try:
    21:   mindata = int(args[1])
    22: except:pass
    23: 
    24: try:
    25:   pload = args[1]
    26: except:pass
    27: 
    28: print "Minimum time",mintime
    29: print "Minimum data",mindata
    30: print "Load        ",pload
    31: 
    32: args = args[3:]
    33: 
    34: def chkxl(x):
    35:   return args == [] or x in args
    36: 
    37: def mkdir(p):
    38:   os.system('rm -rf ' + p)
    39:   os.mkdir(p)
    40: 
    41: hosts = []
    42: mkdir("speed/machine")
    43: def add_host(h):
    44:   hosts.append(h)
    45:   mkdir("speed/machine/"+h)
    46: 
    47: def check_host(h):
    48:   if h not in hosts: add_host(h)
    49: 
    50: fin = open("speed/results.dat")
    51: for line in fin:
    52:   #print line,
    53:   hostname,kind,date,time,xl,tst,n,elapsed = string.split (line)
    54:   check_host(hostname)
    55:   n = int(n)
    56:   elapsed=float(elapsed)
    57:   if pload == kind:
    58:     print hostname,kind,date,time,xl,tst,n,elapsed
    59:     hostd = hostsd.get(hostname,{})
    60:     hostsd[hostname] = hostd
    61:     testd = hostd.get(tst,{})
    62:     hostd[tst] = testd
    63:     xld = testd.get(xl,{})
    64:     testd[xl] = xld
    65:     nd = xld.get(n,[])
    66:     xld[n] = nd
    67:     nd.append(elapsed)
    68: fin.close()
    69: 
    70: print "DATA"
    71: fi = open("speed/mkjpgs.gpl",'w')
    72: fi.write("set terminal jpeg\n")
    73: 
    74: for h,d in hostsd.iteritems():
    75:   print "host",h
    76:   root = 'speed/machine/'+h+'/'
    77:   mkdir(root+"images/")
    78:   mkdir(root+"plotters/")
    79:   for t,xd in d.iteritems():
    80:     print "  test",t
    81:     mkdir(root+'/'+t)
    82:     # one gnuplot file for each test
    83:     fp = open(root+'plotters/'+t+'.gpl','w')
    84:     gx = 'set title "'+h+' '+pload+' '+t+'"\n'
    85:     gx = gx + 'set xlabel "N"\n'
    86:     gx = gx + 'set ylabel "secs"\n'
    87:     gx = gx + 'set key left\n'
    88:     gx = gx + 'plot \\\n'
    89: 
    90:     # one summary per test
    91:     summary = {}
    92:     first = 1
    93:     xls = xd.keys()
    94:     xls.sort()
    95:     for xl in [x for x in xls if chkxl(x)]:
    96:       nd = xd[xl]
    97:       # one datafile for each test/xlator
    98:       fxl = open(root+'/'+t+'/'+xl+'.dat','w')
    99:       print "    xlator",xl
   100:       ns = nd.keys()
   101:       ns.sort()
   102:       lines = 0
   103:       for n in ns:
   104:         es = nd[n]
   105:         count = len(es)
   106:         sum = reduce (lambda x,y: x+y,es)
   107:         average = sum / float(count)
   108:         emin = reduce (lambda x,y:min(x,y),es)
   109:         emax = reduce (lambda x,y:max(x,y),es)
   110:         edev = 0
   111:         for x in es: edev = edev + (x - average) * (x - average)
   112:         edev = math.sqrt(edev/float(count))
   113:         print "        n %2d -[%3d]-> (%6.2f,%6.2f,%6.2f) sd=%6.2f" % (n,count,emin,average,emax,edev)
   114:         # one line for each n value
   115:         # skip too low values
   116:         if average >= mintime and count >= mindata:
   117:           fxl.write("%s %d %0.2f %d %6.2f\n" % (xl,n,average,count,edev))
   118:           lines = lines + 1
   119:         summary[(-n,average)]=(xl,count,edev)
   120:       fxl.close()
   121: 
   122:       # gnuplot is too dumb to handle empty files
   123:       if lines > 0:
   124:         if first:
   125:           first = 0
   126:         else:
   127:           gx = gx + ', \\\n'
   128:         gx = gx + '"'+root+'/'+t+'/'+xl+'.dat" using 2:3 title "'+xl+'" with linespoints'
   129:     fp.write(gx+'\n')
   130:     fp.close()
   131: 
   132:     fi.write('set output "'+root+'images/'+t+'.jpg"\n')
   133:     fi.write('load "'+root+'plotters/'+t+'.gpl"\n')
   134: 
   135:     keys = summary.keys()
   136:     keys.sort()
   137: 
   138:     prlimit = 10
   139:     i = 0
   140:     print "Rankings for",t,"on",h
   141:     for m,a in keys:
   142:       xl,count,edev = summary[m,a]
   143:       print "    %-16s %2d %6.2f [N=%3d, SD=%6.2f]" % (xl,-m,a,count,edev)
   144:       i = i + 1
   145:       if i >= prlimit: break
   146:     try: os.mkdir(root+'rankings')
   147:     except: pass
   148:     rf = open(root+'rankings/'+t+'.txt','w')
   149:     prlimit = 10
   150:     i = 0
   151:     rf.write("Rankings for "+t+' on '+h+'\n')
   152:     for m,a in keys:
   153:       xl,count,edev = summary[m,a]
   154:       rf.write("    %-16s %2d %6.2f [N=%3d SD=%2d%%]\n" % (xl,-m,a,count,int(edev*100/average)))
   155:       i = i + 1
   156:       if i >= prlimit: break
   157:     rf.close()
   158: fi.close()
   159: 
   160: # the client can view individual plots in an x11 console
   161: # with the command 'gnuplot' and then
   162: # type 'load "filename"'
   163: