trisurf-ng.git - Gitblit

Samo Penic

2016-05-15 bb207499ba467145aabb379b841094afd6d695c0

commit \| author \| age
bc14fb	1	#!/usr/bin/python3
SP	2
	3	import configobj
bd6993	4	import xml.etree.ElementTree as ET
SP	5	import base64
	6	import zlib
	7	import io
c14bd6	8	import os
SP	9	from itertools import islice
	10	import mmap
7c45b1	11	import shlex
34600b	12	import psutil
f81585	13	import time
SP	14	import datetime
fca8d6	15	import subprocess
bc14fb	16
055489	17	# Process status
SP	18	TS_NOLOCK=0 # lock file does not exist
	19	TS_NONEXISTANT=0 # process is not in the list of processes
	20	TS_STOPPED=1 # the process is listed, but is in stopped state
	21	TS_RUNNING=2 # process is running
3d0247	22	TS_COMPLETED=3 #simulation is completed
bc14fb	23
9154b3	24	class FileContent:
055489	25	'''
SP	26	Class is helpful for reading and writting the specific files.
	27	'''
9154b3	28	def __init__(self,filename):
055489	29	''' The instance is done by calling constructor FileContent(filename)
SP	30
	31	The object then reads filename if it exists, otherwise the data is empty string.
	32	User may force to reread file by calling the readline() method of the class.
	33
	34	Filename is stored in local variable for future file operations.
	35	'''
	36
9154b3	37	self.filename=filename
SP	38	self.readfile()
bc14fb	39
9154b3	40	def readfile(self):
055489	41	'''Force reread of the file and setting the data'''
SP	42	self.data=""
9154b3	43	try:
SP	44	with open (self.filename, "r") as myfile:
055489	45	self.data=myfile.read().replace('\n', '') #read the file and remove newline from the text
9154b3	46	except:
055489	47	pass # does nothing if error occurs
9154b3	48
SP	49
	50	def writefile(self, data, mode='w'):
055489	51	'''File may be updated by completely rewritting the file contents or appending the data to the end of the file.
SP	52	this is achieved by calling writefile(data, mode) method, where data is the string data to be written and
	53	mode can be 'a' for append and 'w' for writting the file anew.
	54	'''
9154b3	55	with open (self.filename, mode) as myfile:
SP	56	myfile.write(data)
	57
	58	def getText(self):
055489	59	'''
SP	60	Method getText() or calling the object itself returns string of data
	61	'''
9154b3	62	return self.data
SP	63
	64	def __str__(self):
055489	65	'''
SP	66	Method getText() or calling the object itself returns string of data
	67	'''
9154b3	68	return self.getText()
bc14fb	69
SP	70	class Tape:
055489	71	'''
SP	72	Special class that manages configuration of trisurf (commonly named tape). It can read and parse configuration from disk or parse it from string.
	73	'''
bc14fb	74
SP	75	def __init__(self):
055489	76	'''The object is instatiated by calling Tape() constructor without parameters'''
bc14fb	77	return
SP	78
	79	def readTape(self, tape='tape'):
055489	80	'''
SP	81	Tape is read and parsed by calling the readTape() method with optional tape parameter, which is full path to filename where the configuration is stored.
	82	If the tape cannot be read, it prints error and exits.
	83	'''
bc14fb	84	try:
SP	85	self.config=configobj.ConfigObj(tape)
	86	except:
	87	print("Error reading or parsing tape file!\n")
	88	exit(1)
	89
	90	def setTape(self, string):
055489	91	'''Method setTape(string) parses the string in memory that hold the tape contents.'''
bd6993	92	self.config=configobj.ConfigObj(io.StringIO(string))
bc14fb	93	return
SP	94
	95	def getValue(self,key):
055489	96	'''Method getValue(key) returns value of a single parsed setting named "key".'''
SP	97
bc14fb	98	return self.config[key]
SP	99
c14bd6	100	def __str__(self):
055489	101	'''Calling the object itself, it recreates the tape contents from parsed values in form of key=value.'''
c14bd6	102	retval=""
SP	103	for key,val in self.config.iteritems():
	104	retval=retval+str(key)+" = "+str(val)+"\n"
	105	return retval
bc14fb	106
c14bd6	107
SP	108
	109	class Directory:
055489	110	'''
SP	111	Class deals with the paths where the simulation is run and data is stored.
	112	'''
c14bd6	113	def __init__(self, maindir=".", simdir=""):
055489	114	'''Initialization Directory() takes two optional parameters, namely maindir and simdir. Defaults to current directory. It sets local variables maindir and simdir accordingly.'''
c14bd6	115	self.maindir=maindir
SP	116	self.simdir=simdir
	117	return
	118
	119	def fullpath(self):
055489	120	'''
SP	121	Method returns string of path where the data is stored. It combines values of maindir and simdir as maindir/simdir on Unix.
	122	'''
c14bd6	123	return os.path.join(self.maindir,self.simdir)
SP	124
	125	def exists(self):
055489	126	''' Method checks whether the directory specified by fullpath() exists. It return True/False on completion.'''
c14bd6	127	path=self.fullpath()
SP	128	if(os.path.exists(path)):
055489	129	return True
c14bd6	130	else:
055489	131	return False
c14bd6	132
SP	133	def make(self):
055489	134	''' Method make() creates directory. If it fails it exits the program with error message.'''
c14bd6	135	try:
SP	136	os.makedirs(self.fullpath())
	137	except:
	138	print("Cannot make directory "+self.fullpath()+"\n")
	139	exit(1)
	140	return
	141
	142	def makeifnotexist(self):
055489	143	'''Method makeifnotexist() creates directory if it does not exist.'''
c14bd6	144	if(self.exists()==0):
SP	145	self.make()
bd826d	146	return True
SP	147	else:
	148	return False
c14bd6	149
SP	150	def remove(self):
055489	151	'''Method remove() removes directory recursively. WARNING! No questions asked.'''
c14bd6	152	if(self.exists()):
SP	153	try:
	154	os.rmdir(self.fullpath())
	155	except:
	156	print("Cannot remove directory "+self.fullpath()+ "\n")
	157	exit(1)
	158	return
	159
	160	def goto(self):
055489	161	'''
SP	162	Method goto() moves current directory to the one specified by fullpath(). WARNING: when using the relative paths, do not call this function multiple times.
	163	'''
c14bd6	164	try:
SP	165	os.chdir(self.fullpath())
	166	except:
	167	print("Cannot go to directory "+self.fullpath()+"\n")
	168	return
	169
	170
	171	class Statistics:
055489	172	'''
SP	173	Class that deals with the statistics file from the simulations.
	174	File is generally large and not all data is needed, so it is dealt with in a specific way.
	175	'''
	176
c14bd6	177	def __init__(self,path,filename="statistics.csv"):
055489	178	'''
SP	179	At the initialization call it receives optional filename parameter specifying the path and filename of the statistics file.
	180
	181	The local variables path, filename, fullname (joined path and filename) and private check if the file exists are stored.
	182	'''
c14bd6	183	self.path=path
SP	184	self.filename=filename
	185	self.fullname=os.path.join(path,filename)
f81585	186	self.fileOK=self.read()
c14bd6	187	return
SP	188
	189	def exists(self):
055489	190	'''Method check if the statistics file exists.'''
c14bd6	191	if(os.path.isfile(self.fullname)):
SP	192	return True
	193	else:
	194	return False
	195
	196	def mapcount(self):
055489	197	'''
SP	198	Internal method for determining the number of the lines in the most efficient way. Is it really the most efficient?
	199	'''
c14bd6	200	f = open(self.fullname, "r+")
SP	201	buf = mmap.mmap(f.fileno(), 0)
	202	lines = 0
	203	readline = buf.readline
	204	while readline():
	205	lines += 1
	206	return lines
	207
	208	def read(self):
055489	209	'''
SP	210	Method read() reads the statistics if it exists. It sets local variable dT storing the time differential between two intervals of simulation (outer loops). It also stores last simulation loop and the start of the run.
	211	'''
c14bd6	212	if(self.exists()):
3d0247	213	# epoch1=0
SP	214	# epoch2=0
	215	# n1=0
	216	# n2=0
c14bd6	217	nlines=self.mapcount()
3d0247	218	if nlines<2:
SP	219	return(False)
c14bd6	220	try:
SP	221	with open(self.fullname, "r+") as fin:
	222	i=0;
	223	for line in fin:
	224	if(i==1):
7c45b1	225	#print (line)
SP	226	fields=shlex.split(line)
	227	epoch1=fields[0]
	228	n1=fields[1]
c14bd6	229	if(i==nlines-1):
7c45b1	230	fields=shlex.split(line)
SP	231	epoch2=fields[0]
	232	n2=fields[1]
c14bd6	233	i=i+1
SP	234	except:
f81585	235	#print("Cannot read statistics file in "+self.fullname+"\n")
7c45b1	236	return(False)
c14bd6	237	else:
f81585	238	#print("File "+self.fullname+" does not exists.\n")
7c45b1	239	return(False)
3d0247	240	try:
SP	241	self.dT=(int(epoch2)-int(epoch1))/(int(n2)-int(n1))
	242	except:
	243	self.dT=0
f81585	244	self.last=n2
SP	245	self.startDate=epoch1
7c45b1	246	return(True)
f81585	247
SP	248	def __str__(self):
055489	249	'''
SP	250	Prints the full path with filename of the statistics.csv file
	251	'''
f81585	252	return(str(self.fullname))
SP	253
7c45b1	254
bc14fb	255
SP	256	class Runner:
	257	'''
055489	258	Class Runner consists of a single running or terminated instance of the trisurf. It manages starting, stopping, verifying the running process and printing the reports of the configured instances.
bc14fb	259	'''
bd826d	260	def __init__(self, subdir='run0', tape=None, snapshot=None, runArgs=[]):
c14bd6	261	self.subdir=subdir
b122c4	262	self.runArgs=runArgs
47d80d	263	self.fromSnapshot=False
bd826d	264	if(tape!=None):
c14bd6	265	self.initFromTape(tape)
bd826d	266	if(snapshot!=None):
c14bd6	267	self.initFromSnapshot(snapshot)
SP	268	return
	269
	270
a99f2b	271	def initFromTape(self, tape):
bc14fb	272	self.tape=Tape()
SP	273	self.tape.readTape(tape)
bd826d	274	self.tapeFile=tape
bc14fb	275
bd6993	276	def initFromSnapshot(self, snapshotfile):
SP	277	try:
	278	tree = ET.parse(snapshotfile)
	279	except:
	280	print("Error reading snapshot file")
	281	exit(1)
47d80d	282	self.fromSnapshot=True
SP	283	self.snapshotFile=snapshotfile
bd6993	284	root = tree.getroot()
SP	285	tapetxt=root.find('tape')
	286	version=root.find('trisurfversion')
	287	self.tape=Tape()
	288	self.tape.setTape(tapetxt.text)
bd826d	289
9154b3	290	def getPID(self):
34600b	291	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
bd826d	292	#self.Dir.makeifnotexist()
34600b	293	try:
SP	294	fp = open(os.path.join(self.Dir.fullpath(),'.lock'))
	295	except IOError as e:
	296	return 0 #file probably does not exist. e==2??
9154b3	297	pid=fp.readline()
SP	298	fp.close()
055489	299	return int(pid)
9154b3	300
3d0247	301	def getLastIteration(self):
SP	302	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
	303	#self.Dir.makeifnotexist()
	304	try:
	305	fp = open(os.path.join(self.Dir.fullpath(),'.status'))
	306	except IOError as e:
	307	return -1 #file probably does not exist. e==2??
	308	status=fp.readline()
	309	fp.close()
	310	return int(status)
	311
	312	def isCompleted(self):
	313	if (int(self.tape.getValue("iterations"))==self.getLastIteration()+1):
	314	return True
	315	else:
	316	return False
	317
9154b3	318	def getStatus(self):
SP	319	pid=self.getPID()
3d0247	320	if(self.isCompleted()):
SP	321	return TS_COMPLETED
9154b3	322	if(pid==0):
055489	323	return TS_NOLOCK
34600b	324	if(psutil.pid_exists(int(pid))):
055489	325	proc= psutil.Process(int(pid))
bd826d	326	if proc.name()=="trisurf":
SP	327	if proc.status()=="stopped":
055489	328	return TS_STOPPED
SP	329	else:
	330	return TS_RUNNING
9154b3	331	else:
055489	332	return TS_NONEXISTANT
34600b	333	else:
055489	334	return TS_NONEXISTANT
bc14fb	335
SP	336	def start(self):
3d0247	337	if(self.getStatus()==0 or self.getStatus()==TS_COMPLETED):
c14bd6	338	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
bd826d	339	#Symlinks tape file to the directory or create tape file from snapshot in the direcory...
SP	340	if(self.Dir.makeifnotexist()):
	341	if(self.fromSnapshot==False):
	342	try:
	343	os.symlink(os.path.abspath(self.tapeFile), self.Dir.fullpath()+"/tape")
	344	except:
	345	print("Error while symlinking "+os.path.abspath(self.tapeFile)+" to "+self.Dir.fullpath()+"/tape")
	346	exit(1)
	347	else:
	348	try:
	349	with open (os.path.join(self.Dir.fullpath(),"tape"), "w") as myfile:
	350	myfile.write("#This is automatically generated tape file from snapshot\n")
	351	myfile.write(str(self.tape))
	352	except:
	353	print("Error -- cannot make tapefile "+ os.path.join(self.Dir.fullpath(),"tape")+" from the snapshot in the running directory")
	354	exit(1)
	355	try:
	356	os.symlink(os.path.abspath(self.snapshotFile), os.path.join(self.Dir.fullpath(),self.snapshotFile))
	357	except:
	358	print("Error while symlinking "+os.path.abspath(self.snapshotFile)+" to "+os.path.join(self.Dir.fullpath(),self.snapshotFile))
3d0247	359
SP	360	#check if the simulation has been completed. in this case notify user and stop executing.
	361	if(self.isCompleted() and ("--force-from-tape" not in self.runArgs) and ("--reset-iteration-count" not in self.runArgs)):
	362	print("The simulation was completed. Not starting executable at localhost in "+self.Dir.fullpath()+"\n")
	363	return
	364
47d80d	365	cwd=Directory(maindir=os.getcwd())
SP	366	self.Dir.goto()
3d0247	367	print("Starting trisurf-ng executable at localhost in "+self.Dir.fullpath()+"\n")
47d80d	368	if(self.fromSnapshot==True):
b122c4	369	params=["trisurf", "--restore-from-vtk",self.snapshotFile]+self.runArgs
47d80d	370	else:
3d0247	371	#veify if dump exists. If not it is a first run and shoud be run with --force-from-tape
SP	372	if(os.path.isfile("dump.bin")==False):
	373	self.runArgs.append("--force-from-tape")
b122c4	374	params=["trisurf"]+self.runArgs
fca8d6	375	subprocess.Popen (params, stdout=subprocess.DEVNULL)
47d80d	376	cwd.goto()
c14bd6	377	else:
SP	378	print("Process already running. Not starting\n")
	379	return
bc14fb	380
SP	381	def stop(self):
	382	pass
	383
c14bd6	384	def setMaindir(self,prefix,variables):
055489	385	maindir=""
c14bd6	386	for p,v in zip(prefix,variables):
SP	387	if(v=="xk0"):
	388	tv=str(round(float(self.tape.config[v])))
	389	else:
	390	tv=self.tape.config[v]
	391	maindir=maindir+p+tv
	392	self.maindir=maindir
	393	return
	394
	395	def setSubdir(self, subdir="run0"):
	396	self.subdir=subdir
	397	return
	398
	399	def getStatistics(self, statfile="statistics.csv"):
7ca5ae	400	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
34600b	401	self.statistics=Statistics(self.Dir.fullpath(), statfile)
9154b3	402	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
SP	403	pid=self.getPID();
055489	404	status=self.getStatus()
SP	405	if(status==TS_NONEXISTANT or status==TS_NOLOCK):
	406	statustxt="Not running"
9154b3	407	pid=""
055489	408	elif status==TS_STOPPED:
SP	409	statustxt="Stopped"
3d0247	410	elif status==TS_COMPLETED:
SP	411	statustxt="Completed"
055489	412	else:
SP	413	statustxt="Running"
9154b3	414
SP	415	if(self.statistics.fileOK):
fca8d6	416	report=[time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(self.statistics.startDate))),str(datetime.timedelta(microseconds=(int(self.tape.config['iterations'])-int(self.statistics.last))self.statistics.dT)1000000), statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
9154b3	417	else:
SP	418	report=["N/A","N/A\t",statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
f81585	419	return report
c14bd6	420
055489	421	def writeComment(self, data, mode='w'):
9154b3	422	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
SP	423	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
055489	424	self.Comment.writefile(data,mode=mode)
9154b3	425
bc14fb	426	def __str__(self):
c14bd6	427	if(self.getStatus()==0):
SP	428	str=" not running."
	429	else:
	430	str=" running."
	431	return(self.Dir.fullpath()+str)
	432
bc14fb	433