trisurf-ng.git - Gitblit

Fixes error occuring when statistics.csv is empty.

Samo Penic

2016-05-17 d665c0b94c289f6c34597af3010d6b9b38a08ac4

commit \| author \| age
bc14fb	1	#!/usr/bin/python3
SP	2
	3	import configobj
bd6993	4	import xml.etree.ElementTree as ET
SP	5	import base64
	6	import zlib
	7	import io
c14bd6	8	import os
SP	9	from itertools import islice
	10	import mmap
7c45b1	11	import shlex
34600b	12	import psutil
f81585	13	import time
SP	14	import datetime
fca8d6	15	import subprocess
38cb4a	16	import shutil
bc14fb	17
055489	18	# Process status
SP	19	TS_NOLOCK=0 # lock file does not exist
	20	TS_NONEXISTANT=0 # process is not in the list of processes
	21	TS_STOPPED=1 # the process is listed, but is in stopped state
	22	TS_RUNNING=2 # process is running
3d0247	23	TS_COMPLETED=3 #simulation is completed
bc14fb	24
9154b3	25	class FileContent:
055489	26	'''
SP	27	Class is helpful for reading and writting the specific files.
	28	'''
9154b3	29	def __init__(self,filename):
055489	30	''' The instance is done by calling constructor FileContent(filename)
SP	31
	32	The object then reads filename if it exists, otherwise the data is empty string.
	33	User may force to reread file by calling the readline() method of the class.
	34
	35	Filename is stored in local variable for future file operations.
	36	'''
	37
9154b3	38	self.filename=filename
SP	39	self.readfile()
bc14fb	40
9154b3	41	def readfile(self):
055489	42	'''Force reread of the file and setting the data'''
SP	43	self.data=""
9154b3	44	try:
SP	45	with open (self.filename, "r") as myfile:
055489	46	self.data=myfile.read().replace('\n', '') #read the file and remove newline from the text
9154b3	47	except:
055489	48	pass # does nothing if error occurs
9154b3	49
SP	50
	51	def writefile(self, data, mode='w'):
055489	52	'''File may be updated by completely rewritting the file contents or appending the data to the end of the file.
SP	53	this is achieved by calling writefile(data, mode) method, where data is the string data to be written and
	54	mode can be 'a' for append and 'w' for writting the file anew.
	55	'''
9154b3	56	with open (self.filename, mode) as myfile:
SP	57	myfile.write(data)
	58
	59	def getText(self):
055489	60	'''
SP	61	Method getText() or calling the object itself returns string of data
	62	'''
9154b3	63	return self.data
SP	64
	65	def __str__(self):
055489	66	'''
SP	67	Method getText() or calling the object itself returns string of data
	68	'''
9154b3	69	return self.getText()
bc14fb	70
SP	71	class Tape:
055489	72	'''
SP	73	Special class that manages configuration of trisurf (commonly named tape). It can read and parse configuration from disk or parse it from string.
	74	'''
bc14fb	75
SP	76	def __init__(self):
055489	77	'''The object is instatiated by calling Tape() constructor without parameters'''
bc14fb	78	return
SP	79
	80	def readTape(self, tape='tape'):
055489	81	'''
SP	82	Tape is read and parsed by calling the readTape() method with optional tape parameter, which is full path to filename where the configuration is stored.
	83	If the tape cannot be read, it prints error and exits.
	84	'''
bc14fb	85	try:
SP	86	self.config=configobj.ConfigObj(tape)
	87	except:
	88	print("Error reading or parsing tape file!\n")
	89	exit(1)
	90
	91	def setTape(self, string):
055489	92	'''Method setTape(string) parses the string in memory that hold the tape contents.'''
bd6993	93	self.config=configobj.ConfigObj(io.StringIO(string))
bc14fb	94	return
SP	95
	96	def getValue(self,key):
055489	97	'''Method getValue(key) returns value of a single parsed setting named "key".'''
SP	98
bc14fb	99	return self.config[key]
SP	100
c14bd6	101	def __str__(self):
055489	102	'''Calling the object itself, it recreates the tape contents from parsed values in form of key=value.'''
c14bd6	103	retval=""
SP	104	for key,val in self.config.iteritems():
	105	retval=retval+str(key)+" = "+str(val)+"\n"
	106	return retval
bc14fb	107
c14bd6	108
SP	109
	110	class Directory:
055489	111	'''
SP	112	Class deals with the paths where the simulation is run and data is stored.
	113	'''
c14bd6	114	def __init__(self, maindir=".", simdir=""):
055489	115	'''Initialization Directory() takes two optional parameters, namely maindir and simdir. Defaults to current directory. It sets local variables maindir and simdir accordingly.'''
c14bd6	116	self.maindir=maindir
SP	117	self.simdir=simdir
	118	return
	119
	120	def fullpath(self):
055489	121	'''
SP	122	Method returns string of path where the data is stored. It combines values of maindir and simdir as maindir/simdir on Unix.
	123	'''
c14bd6	124	return os.path.join(self.maindir,self.simdir)
SP	125
	126	def exists(self):
055489	127	''' Method checks whether the directory specified by fullpath() exists. It return True/False on completion.'''
c14bd6	128	path=self.fullpath()
SP	129	if(os.path.exists(path)):
055489	130	return True
c14bd6	131	else:
055489	132	return False
c14bd6	133
SP	134	def make(self):
055489	135	''' Method make() creates directory. If it fails it exits the program with error message.'''
c14bd6	136	try:
SP	137	os.makedirs(self.fullpath())
	138	except:
	139	print("Cannot make directory "+self.fullpath()+"\n")
	140	exit(1)
	141	return
	142
	143	def makeifnotexist(self):
055489	144	'''Method makeifnotexist() creates directory if it does not exist.'''
c14bd6	145	if(self.exists()==0):
SP	146	self.make()
bd826d	147	return True
SP	148	else:
	149	return False
c14bd6	150
SP	151	def remove(self):
055489	152	'''Method remove() removes directory recursively. WARNING! No questions asked.'''
c14bd6	153	if(self.exists()):
SP	154	try:
	155	os.rmdir(self.fullpath())
	156	except:
	157	print("Cannot remove directory "+self.fullpath()+ "\n")
	158	exit(1)
	159	return
	160
	161	def goto(self):
055489	162	'''
SP	163	Method goto() moves current directory to the one specified by fullpath(). WARNING: when using the relative paths, do not call this function multiple times.
	164	'''
c14bd6	165	try:
SP	166	os.chdir(self.fullpath())
	167	except:
	168	print("Cannot go to directory "+self.fullpath()+"\n")
	169	return
	170
	171
	172	class Statistics:
055489	173	'''
SP	174	Class that deals with the statistics file from the simulations.
	175	File is generally large and not all data is needed, so it is dealt with in a specific way.
	176	'''
	177
c14bd6	178	def __init__(self,path,filename="statistics.csv"):
055489	179	'''
SP	180	At the initialization call it receives optional filename parameter specifying the path and filename of the statistics file.
	181
	182	The local variables path, filename, fullname (joined path and filename) and private check if the file exists are stored.
	183	'''
c14bd6	184	self.path=path
SP	185	self.filename=filename
	186	self.fullname=os.path.join(path,filename)
f81585	187	self.fileOK=self.read()
c14bd6	188	return
SP	189
	190	def exists(self):
055489	191	'''Method check if the statistics file exists.'''
c14bd6	192	if(os.path.isfile(self.fullname)):
SP	193	return True
	194	else:
	195	return False
	196
	197	def mapcount(self):
055489	198	'''
SP	199	Internal method for determining the number of the lines in the most efficient way. Is it really the most efficient?
	200	'''
c14bd6	201	f = open(self.fullname, "r+")
d665c0	202	try:
SP	203	buf = mmap.mmap(f.fileno(), 0)
	204	lines = 0
	205	readline = buf.readline
	206	while readline():
	207	lines += 1
	208	f.close()
	209	except:
	210	lines=0
	211	f.close()
c14bd6	212	return lines
d665c0	213
c14bd6	214	def read(self):
055489	215	'''
SP	216	Method read() reads the statistics if it exists. It sets local variable dT storing the time differential between two intervals of simulation (outer loops). It also stores last simulation loop and the start of the run.
	217	'''
c14bd6	218	if(self.exists()):
3d0247	219	# epoch1=0
SP	220	# epoch2=0
	221	# n1=0
	222	# n2=0
c14bd6	223	nlines=self.mapcount()
3d0247	224	if nlines<2:
SP	225	return(False)
c14bd6	226	try:
SP	227	with open(self.fullname, "r+") as fin:
	228	i=0;
	229	for line in fin:
	230	if(i==1):
7c45b1	231	#print (line)
SP	232	fields=shlex.split(line)
	233	epoch1=fields[0]
	234	n1=fields[1]
c14bd6	235	if(i==nlines-1):
7c45b1	236	fields=shlex.split(line)
SP	237	epoch2=fields[0]
	238	n2=fields[1]
c14bd6	239	i=i+1
SP	240	except:
f81585	241	#print("Cannot read statistics file in "+self.fullname+"\n")
7c45b1	242	return(False)
c14bd6	243	else:
f81585	244	#print("File "+self.fullname+" does not exists.\n")
7c45b1	245	return(False)
3d0247	246	try:
SP	247	self.dT=(int(epoch2)-int(epoch1))/(int(n2)-int(n1))
	248	except:
	249	self.dT=0
f81585	250	self.last=n2
SP	251	self.startDate=epoch1
7c45b1	252	return(True)
f81585	253
SP	254	def __str__(self):
055489	255	'''
SP	256	Prints the full path with filename of the statistics.csv file
	257	'''
f81585	258	return(str(self.fullname))
SP	259
7c45b1	260
bc14fb	261
SP	262	class Runner:
	263	'''
055489	264	Class Runner consists of a single running or terminated instance of the trisurf. It manages starting, stopping, verifying the running process and printing the reports of the configured instances.
bc14fb	265	'''
bd826d	266	def __init__(self, subdir='run0', tape=None, snapshot=None, runArgs=[]):
c14bd6	267	self.subdir=subdir
b122c4	268	self.runArgs=runArgs
47d80d	269	self.fromSnapshot=False
bd826d	270	if(tape!=None):
c14bd6	271	self.initFromTape(tape)
bd826d	272	if(snapshot!=None):
c14bd6	273	self.initFromSnapshot(snapshot)
SP	274	return
	275
	276
a99f2b	277	def initFromTape(self, tape):
bc14fb	278	self.tape=Tape()
SP	279	self.tape.readTape(tape)
bd826d	280	self.tapeFile=tape
bc14fb	281
bd6993	282	def initFromSnapshot(self, snapshotfile):
SP	283	try:
	284	tree = ET.parse(snapshotfile)
	285	except:
	286	print("Error reading snapshot file")
	287	exit(1)
47d80d	288	self.fromSnapshot=True
SP	289	self.snapshotFile=snapshotfile
bd6993	290	root = tree.getroot()
SP	291	tapetxt=root.find('tape')
	292	version=root.find('trisurfversion')
	293	self.tape=Tape()
	294	self.tape.setTape(tapetxt.text)
2ded03	295
9154b3	296	def getPID(self):
34600b	297	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
bd826d	298	#self.Dir.makeifnotexist()
34600b	299	try:
SP	300	fp = open(os.path.join(self.Dir.fullpath(),'.lock'))
	301	except IOError as e:
	302	return 0 #file probably does not exist. e==2??
9154b3	303	pid=fp.readline()
SP	304	fp.close()
055489	305	return int(pid)
9154b3	306
3d0247	307	def getLastIteration(self):
SP	308	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
	309	#self.Dir.makeifnotexist()
	310	try:
	311	fp = open(os.path.join(self.Dir.fullpath(),'.status'))
	312	except IOError as e:
	313	return -1 #file probably does not exist. e==2??
	314	status=fp.readline()
	315	fp.close()
	316	return int(status)
	317
	318	def isCompleted(self):
	319	if (int(self.tape.getValue("iterations"))==self.getLastIteration()+1):
	320	return True
	321	else:
	322	return False
	323
9154b3	324	def getStatus(self):
SP	325	pid=self.getPID()
3d0247	326	if(self.isCompleted()):
SP	327	return TS_COMPLETED
9154b3	328	if(pid==0):
055489	329	return TS_NOLOCK
34600b	330	if(psutil.pid_exists(int(pid))):
055489	331	proc= psutil.Process(int(pid))
56ba2f	332	#psutil.__version__ == '3.4.2' requires name() and status(), some older versions reguire name, status
9aa037	333	if(psutil.__version__>='2.0.0'):
SP	334	procname=proc.name()
	335	procstat=proc.status()
	336	else:
	337	procname=proc.name
	338	procstat=proc.status
	339	if procname=="trisurf":
	340	if procstat=="stopped":
055489	341	return TS_STOPPED
SP	342	else:
	343	return TS_RUNNING
9154b3	344	else:
055489	345	return TS_NONEXISTANT
34600b	346	else:
055489	347	return TS_NONEXISTANT
bc14fb	348
SP	349	def start(self):
3d0247	350	if(self.getStatus()==0 or self.getStatus()==TS_COMPLETED):
38cb4a	351	#check if executable exists
SP	352	if(shutil.which('trisurf')==None):
	353	print("Error. Trisurf executable not found in PATH. Please install trisurf prior to running trisurf manager.")
	354	exit(1)
c14bd6	355	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
bd826d	356	#Symlinks tape file to the directory or create tape file from snapshot in the direcory...
SP	357	if(self.Dir.makeifnotexist()):
	358	if(self.fromSnapshot==False):
	359	try:
	360	os.symlink(os.path.abspath(self.tapeFile), self.Dir.fullpath()+"/tape")
	361	except:
	362	print("Error while symlinking "+os.path.abspath(self.tapeFile)+" to "+self.Dir.fullpath()+"/tape")
	363	exit(1)
	364	else:
	365	try:
	366	with open (os.path.join(self.Dir.fullpath(),"tape"), "w") as myfile:
3b83db	367	myfile.write("#This is automatically generated tape file from snapshot")
bd826d	368	myfile.write(str(self.tape))
SP	369	except:
	370	print("Error -- cannot make tapefile "+ os.path.join(self.Dir.fullpath(),"tape")+" from the snapshot in the running directory")
	371	exit(1)
	372	try:
	373	os.symlink(os.path.abspath(self.snapshotFile), os.path.join(self.Dir.fullpath(),self.snapshotFile))
	374	except:
	375	print("Error while symlinking "+os.path.abspath(self.snapshotFile)+" to "+os.path.join(self.Dir.fullpath(),self.snapshotFile))
3d0247	376
SP	377	#check if the simulation has been completed. in this case notify user and stop executing.
	378	if(self.isCompleted() and ("--force-from-tape" not in self.runArgs) and ("--reset-iteration-count" not in self.runArgs)):
3b83db	379	print("The simulation was completed. Not starting executable in "+self.Dir.fullpath())
3d0247	380	return
SP	381
47d80d	382	cwd=Directory(maindir=os.getcwd())
SP	383	self.Dir.goto()
3b83db	384	print("Starting trisurf-ng executable in "+self.Dir.fullpath())
47d80d	385	if(self.fromSnapshot==True):
b122c4	386	params=["trisurf", "--restore-from-vtk",self.snapshotFile]+self.runArgs
47d80d	387	else:
3d0247	388	#veify if dump exists. If not it is a first run and shoud be run with --force-from-tape
SP	389	if(os.path.isfile("dump.bin")==False):
	390	self.runArgs.append("--force-from-tape")
b122c4	391	params=["trisurf"]+self.runArgs
fca8d6	392	subprocess.Popen (params, stdout=subprocess.DEVNULL)
47d80d	393	cwd.goto()
c14bd6	394	else:
3b83db	395	print("Process in "+self.Dir.fullpath()+" already running. Not starting.")
c14bd6	396	return
bc14fb	397
SP	398	def stop(self):
	399	pass
	400
c14bd6	401	def setMaindir(self,prefix,variables):
055489	402	maindir=""
c14bd6	403	for p,v in zip(prefix,variables):
SP	404	if(v=="xk0"):
	405	tv=str(round(float(self.tape.config[v])))
	406	else:
	407	tv=self.tape.config[v]
	408	maindir=maindir+p+tv
	409	self.maindir=maindir
	410	return
	411
	412	def setSubdir(self, subdir="run0"):
	413	self.subdir=subdir
	414	return
	415
	416	def getStatistics(self, statfile="statistics.csv"):
7ca5ae	417	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
34600b	418	self.statistics=Statistics(self.Dir.fullpath(), statfile)
9154b3	419	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
38cb4a	420	pid=self.getPID()
055489	421	status=self.getStatus()
25296b	422	if(self.statistics.fileOK):
SP	423	ETA=str(datetime.timedelta(microseconds=(int(self.tape.config['iterations'])-int(self.statistics.last))self.statistics.dT)1000000)
055489	424	if(status==TS_NONEXISTANT or status==TS_NOLOCK):
SP	425	statustxt="Not running"
9154b3	426	pid=""
38cb4a	427	ETA=""
055489	428	elif status==TS_STOPPED:
SP	429	statustxt="Stopped"
38cb4a	430	ETA="N/A"
3d0247	431	elif status==TS_COMPLETED:
SP	432	statustxt="Completed"
38cb4a	433	pid=""
SP	434	ETA=""
055489	435	else:
SP	436	statustxt="Running"
9154b3	437
SP	438	if(self.statistics.fileOK):
38cb4a	439	report=[time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(self.statistics.startDate))),ETA, statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
9154b3	440	else:
25296b	441	report=["N/A","N/A",statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
f81585	442	return report
c14bd6	443
055489	444	def writeComment(self, data, mode='w'):
9154b3	445	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
SP	446	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
055489	447	self.Comment.writefile(data,mode=mode)
9154b3	448
bc14fb	449	def __str__(self):
c14bd6	450	if(self.getStatus()==0):
SP	451	str=" not running."
	452	else:
	453	str=" running."
	454	return(self.Dir.fullpath()+str)
	455
bc14fb	456