trisurf-ng.git - Gitblit

Samo Penic

2016-05-16 56ba2f3107066b9a449b83eba64cab8e43cae9f0

commit \| author \| age
bc14fb	1	#!/usr/bin/python3
SP	2
	3	import configobj
bd6993	4	import xml.etree.ElementTree as ET
SP	5	import base64
	6	import zlib
	7	import io
c14bd6	8	import os
SP	9	from itertools import islice
	10	import mmap
7c45b1	11	import shlex
34600b	12	import psutil
f81585	13	import time
SP	14	import datetime
fca8d6	15	import subprocess
38cb4a	16	import shutil
bc14fb	17
055489	18	# Process status
SP	19	TS_NOLOCK=0 # lock file does not exist
	20	TS_NONEXISTANT=0 # process is not in the list of processes
	21	TS_STOPPED=1 # the process is listed, but is in stopped state
	22	TS_RUNNING=2 # process is running
3d0247	23	TS_COMPLETED=3 #simulation is completed
bc14fb	24
9154b3	25	class FileContent:
055489	26	'''
SP	27	Class is helpful for reading and writting the specific files.
	28	'''
9154b3	29	def __init__(self,filename):
055489	30	''' The instance is done by calling constructor FileContent(filename)
SP	31
	32	The object then reads filename if it exists, otherwise the data is empty string.
	33	User may force to reread file by calling the readline() method of the class.
	34
	35	Filename is stored in local variable for future file operations.
	36	'''
	37
9154b3	38	self.filename=filename
SP	39	self.readfile()
bc14fb	40
9154b3	41	def readfile(self):
055489	42	'''Force reread of the file and setting the data'''
SP	43	self.data=""
9154b3	44	try:
SP	45	with open (self.filename, "r") as myfile:
055489	46	self.data=myfile.read().replace('\n', '') #read the file and remove newline from the text
9154b3	47	except:
055489	48	pass # does nothing if error occurs
9154b3	49
SP	50
	51	def writefile(self, data, mode='w'):
055489	52	'''File may be updated by completely rewritting the file contents or appending the data to the end of the file.
SP	53	this is achieved by calling writefile(data, mode) method, where data is the string data to be written and
	54	mode can be 'a' for append and 'w' for writting the file anew.
	55	'''
9154b3	56	with open (self.filename, mode) as myfile:
SP	57	myfile.write(data)
	58
	59	def getText(self):
055489	60	'''
SP	61	Method getText() or calling the object itself returns string of data
	62	'''
9154b3	63	return self.data
SP	64
	65	def __str__(self):
055489	66	'''
SP	67	Method getText() or calling the object itself returns string of data
	68	'''
9154b3	69	return self.getText()
bc14fb	70
SP	71	class Tape:
055489	72	'''
SP	73	Special class that manages configuration of trisurf (commonly named tape). It can read and parse configuration from disk or parse it from string.
	74	'''
bc14fb	75
SP	76	def __init__(self):
055489	77	'''The object is instatiated by calling Tape() constructor without parameters'''
bc14fb	78	return
SP	79
	80	def readTape(self, tape='tape'):
055489	81	'''
SP	82	Tape is read and parsed by calling the readTape() method with optional tape parameter, which is full path to filename where the configuration is stored.
	83	If the tape cannot be read, it prints error and exits.
	84	'''
bc14fb	85	try:
SP	86	self.config=configobj.ConfigObj(tape)
	87	except:
	88	print("Error reading or parsing tape file!\n")
	89	exit(1)
	90
	91	def setTape(self, string):
055489	92	'''Method setTape(string) parses the string in memory that hold the tape contents.'''
bd6993	93	self.config=configobj.ConfigObj(io.StringIO(string))
bc14fb	94	return
SP	95
	96	def getValue(self,key):
055489	97	'''Method getValue(key) returns value of a single parsed setting named "key".'''
SP	98
bc14fb	99	return self.config[key]
SP	100
c14bd6	101	def __str__(self):
055489	102	'''Calling the object itself, it recreates the tape contents from parsed values in form of key=value.'''
c14bd6	103	retval=""
SP	104	for key,val in self.config.iteritems():
	105	retval=retval+str(key)+" = "+str(val)+"\n"
	106	return retval
bc14fb	107
c14bd6	108
SP	109
	110	class Directory:
055489	111	'''
SP	112	Class deals with the paths where the simulation is run and data is stored.
	113	'''
c14bd6	114	def __init__(self, maindir=".", simdir=""):
055489	115	'''Initialization Directory() takes two optional parameters, namely maindir and simdir. Defaults to current directory. It sets local variables maindir and simdir accordingly.'''
c14bd6	116	self.maindir=maindir
SP	117	self.simdir=simdir
	118	return
	119
	120	def fullpath(self):
055489	121	'''
SP	122	Method returns string of path where the data is stored. It combines values of maindir and simdir as maindir/simdir on Unix.
	123	'''
c14bd6	124	return os.path.join(self.maindir,self.simdir)
SP	125
	126	def exists(self):
055489	127	''' Method checks whether the directory specified by fullpath() exists. It return True/False on completion.'''
c14bd6	128	path=self.fullpath()
SP	129	if(os.path.exists(path)):
055489	130	return True
c14bd6	131	else:
055489	132	return False
c14bd6	133
SP	134	def make(self):
055489	135	''' Method make() creates directory. If it fails it exits the program with error message.'''
c14bd6	136	try:
SP	137	os.makedirs(self.fullpath())
	138	except:
	139	print("Cannot make directory "+self.fullpath()+"\n")
	140	exit(1)
	141	return
	142
	143	def makeifnotexist(self):
055489	144	'''Method makeifnotexist() creates directory if it does not exist.'''
c14bd6	145	if(self.exists()==0):
SP	146	self.make()
bd826d	147	return True
SP	148	else:
	149	return False
c14bd6	150
SP	151	def remove(self):
055489	152	'''Method remove() removes directory recursively. WARNING! No questions asked.'''
c14bd6	153	if(self.exists()):
SP	154	try:
	155	os.rmdir(self.fullpath())
	156	except:
	157	print("Cannot remove directory "+self.fullpath()+ "\n")
	158	exit(1)
	159	return
	160
	161	def goto(self):
055489	162	'''
SP	163	Method goto() moves current directory to the one specified by fullpath(). WARNING: when using the relative paths, do not call this function multiple times.
	164	'''
c14bd6	165	try:
SP	166	os.chdir(self.fullpath())
	167	except:
	168	print("Cannot go to directory "+self.fullpath()+"\n")
	169	return
	170
	171
	172	class Statistics:
055489	173	'''
SP	174	Class that deals with the statistics file from the simulations.
	175	File is generally large and not all data is needed, so it is dealt with in a specific way.
	176	'''
	177
c14bd6	178	def __init__(self,path,filename="statistics.csv"):
055489	179	'''
SP	180	At the initialization call it receives optional filename parameter specifying the path and filename of the statistics file.
	181
	182	The local variables path, filename, fullname (joined path and filename) and private check if the file exists are stored.
	183	'''
c14bd6	184	self.path=path
SP	185	self.filename=filename
	186	self.fullname=os.path.join(path,filename)
f81585	187	self.fileOK=self.read()
c14bd6	188	return
SP	189
	190	def exists(self):
055489	191	'''Method check if the statistics file exists.'''
c14bd6	192	if(os.path.isfile(self.fullname)):
SP	193	return True
	194	else:
	195	return False
	196
	197	def mapcount(self):
055489	198	'''
SP	199	Internal method for determining the number of the lines in the most efficient way. Is it really the most efficient?
	200	'''
c14bd6	201	f = open(self.fullname, "r+")
SP	202	buf = mmap.mmap(f.fileno(), 0)
	203	lines = 0
	204	readline = buf.readline
	205	while readline():
	206	lines += 1
	207	return lines
	208
	209	def read(self):
055489	210	'''
SP	211	Method read() reads the statistics if it exists. It sets local variable dT storing the time differential between two intervals of simulation (outer loops). It also stores last simulation loop and the start of the run.
	212	'''
c14bd6	213	if(self.exists()):
3d0247	214	# epoch1=0
SP	215	# epoch2=0
	216	# n1=0
	217	# n2=0
c14bd6	218	nlines=self.mapcount()
3d0247	219	if nlines<2:
SP	220	return(False)
c14bd6	221	try:
SP	222	with open(self.fullname, "r+") as fin:
	223	i=0;
	224	for line in fin:
	225	if(i==1):
7c45b1	226	#print (line)
SP	227	fields=shlex.split(line)
	228	epoch1=fields[0]
	229	n1=fields[1]
c14bd6	230	if(i==nlines-1):
7c45b1	231	fields=shlex.split(line)
SP	232	epoch2=fields[0]
	233	n2=fields[1]
c14bd6	234	i=i+1
SP	235	except:
f81585	236	#print("Cannot read statistics file in "+self.fullname+"\n")
7c45b1	237	return(False)
c14bd6	238	else:
f81585	239	#print("File "+self.fullname+" does not exists.\n")
7c45b1	240	return(False)
3d0247	241	try:
SP	242	self.dT=(int(epoch2)-int(epoch1))/(int(n2)-int(n1))
	243	except:
	244	self.dT=0
f81585	245	self.last=n2
SP	246	self.startDate=epoch1
7c45b1	247	return(True)
f81585	248
SP	249	def __str__(self):
055489	250	'''
SP	251	Prints the full path with filename of the statistics.csv file
	252	'''
f81585	253	return(str(self.fullname))
SP	254
7c45b1	255
bc14fb	256
SP	257	class Runner:
	258	'''
055489	259	Class Runner consists of a single running or terminated instance of the trisurf. It manages starting, stopping, verifying the running process and printing the reports of the configured instances.
bc14fb	260	'''
bd826d	261	def __init__(self, subdir='run0', tape=None, snapshot=None, runArgs=[]):
c14bd6	262	self.subdir=subdir
b122c4	263	self.runArgs=runArgs
47d80d	264	self.fromSnapshot=False
bd826d	265	if(tape!=None):
c14bd6	266	self.initFromTape(tape)
bd826d	267	if(snapshot!=None):
c14bd6	268	self.initFromSnapshot(snapshot)
SP	269	return
	270
	271
a99f2b	272	def initFromTape(self, tape):
bc14fb	273	self.tape=Tape()
SP	274	self.tape.readTape(tape)
bd826d	275	self.tapeFile=tape
bc14fb	276
bd6993	277	def initFromSnapshot(self, snapshotfile):
SP	278	try:
	279	tree = ET.parse(snapshotfile)
	280	except:
	281	print("Error reading snapshot file")
	282	exit(1)
47d80d	283	self.fromSnapshot=True
SP	284	self.snapshotFile=snapshotfile
bd6993	285	root = tree.getroot()
SP	286	tapetxt=root.find('tape')
	287	version=root.find('trisurfversion')
	288	self.tape=Tape()
	289	self.tape.setTape(tapetxt.text)
2ded03	290
9154b3	291	def getPID(self):
34600b	292	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
bd826d	293	#self.Dir.makeifnotexist()
34600b	294	try:
SP	295	fp = open(os.path.join(self.Dir.fullpath(),'.lock'))
	296	except IOError as e:
	297	return 0 #file probably does not exist. e==2??
9154b3	298	pid=fp.readline()
SP	299	fp.close()
055489	300	return int(pid)
9154b3	301
3d0247	302	def getLastIteration(self):
SP	303	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
	304	#self.Dir.makeifnotexist()
	305	try:
	306	fp = open(os.path.join(self.Dir.fullpath(),'.status'))
	307	except IOError as e:
	308	return -1 #file probably does not exist. e==2??
	309	status=fp.readline()
	310	fp.close()
	311	return int(status)
	312
	313	def isCompleted(self):
	314	if (int(self.tape.getValue("iterations"))==self.getLastIteration()+1):
	315	return True
	316	else:
	317	return False
	318
9154b3	319	def getStatus(self):
SP	320	pid=self.getPID()
3d0247	321	if(self.isCompleted()):
SP	322	return TS_COMPLETED
9154b3	323	if(pid==0):
055489	324	return TS_NOLOCK
34600b	325	if(psutil.pid_exists(int(pid))):
055489	326	proc= psutil.Process(int(pid))
56ba2f	327	#psutil.__version__ == '3.4.2' requires name() and status(), some older versions reguire name, status
2ded03	328	if proc.name()=="trisurf":
SP	329	if proc.status()=="stopped":
055489	330	return TS_STOPPED
SP	331	else:
	332	return TS_RUNNING
9154b3	333	else:
055489	334	return TS_NONEXISTANT
34600b	335	else:
055489	336	return TS_NONEXISTANT
bc14fb	337
SP	338	def start(self):
3d0247	339	if(self.getStatus()==0 or self.getStatus()==TS_COMPLETED):
38cb4a	340	#check if executable exists
SP	341	if(shutil.which('trisurf')==None):
	342	print("Error. Trisurf executable not found in PATH. Please install trisurf prior to running trisurf manager.")
	343	exit(1)
c14bd6	344	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
bd826d	345	#Symlinks tape file to the directory or create tape file from snapshot in the direcory...
SP	346	if(self.Dir.makeifnotexist()):
	347	if(self.fromSnapshot==False):
	348	try:
	349	os.symlink(os.path.abspath(self.tapeFile), self.Dir.fullpath()+"/tape")
	350	except:
	351	print("Error while symlinking "+os.path.abspath(self.tapeFile)+" to "+self.Dir.fullpath()+"/tape")
	352	exit(1)
	353	else:
	354	try:
	355	with open (os.path.join(self.Dir.fullpath(),"tape"), "w") as myfile:
3b83db	356	myfile.write("#This is automatically generated tape file from snapshot")
bd826d	357	myfile.write(str(self.tape))
SP	358	except:
	359	print("Error -- cannot make tapefile "+ os.path.join(self.Dir.fullpath(),"tape")+" from the snapshot in the running directory")
	360	exit(1)
	361	try:
	362	os.symlink(os.path.abspath(self.snapshotFile), os.path.join(self.Dir.fullpath(),self.snapshotFile))
	363	except:
	364	print("Error while symlinking "+os.path.abspath(self.snapshotFile)+" to "+os.path.join(self.Dir.fullpath(),self.snapshotFile))
3d0247	365
SP	366	#check if the simulation has been completed. in this case notify user and stop executing.
	367	if(self.isCompleted() and ("--force-from-tape" not in self.runArgs) and ("--reset-iteration-count" not in self.runArgs)):
3b83db	368	print("The simulation was completed. Not starting executable in "+self.Dir.fullpath())
3d0247	369	return
SP	370
47d80d	371	cwd=Directory(maindir=os.getcwd())
SP	372	self.Dir.goto()
3b83db	373	print("Starting trisurf-ng executable in "+self.Dir.fullpath())
47d80d	374	if(self.fromSnapshot==True):
b122c4	375	params=["trisurf", "--restore-from-vtk",self.snapshotFile]+self.runArgs
47d80d	376	else:
3d0247	377	#veify if dump exists. If not it is a first run and shoud be run with --force-from-tape
SP	378	if(os.path.isfile("dump.bin")==False):
	379	self.runArgs.append("--force-from-tape")
b122c4	380	params=["trisurf"]+self.runArgs
fca8d6	381	subprocess.Popen (params, stdout=subprocess.DEVNULL)
47d80d	382	cwd.goto()
c14bd6	383	else:
3b83db	384	print("Process in "+self.Dir.fullpath()+" already running. Not starting.")
c14bd6	385	return
bc14fb	386
SP	387	def stop(self):
	388	pass
	389
c14bd6	390	def setMaindir(self,prefix,variables):
055489	391	maindir=""
c14bd6	392	for p,v in zip(prefix,variables):
SP	393	if(v=="xk0"):
	394	tv=str(round(float(self.tape.config[v])))
	395	else:
	396	tv=self.tape.config[v]
	397	maindir=maindir+p+tv
	398	self.maindir=maindir
	399	return
	400
	401	def setSubdir(self, subdir="run0"):
	402	self.subdir=subdir
	403	return
	404
	405	def getStatistics(self, statfile="statistics.csv"):
7ca5ae	406	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
34600b	407	self.statistics=Statistics(self.Dir.fullpath(), statfile)
9154b3	408	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
38cb4a	409	pid=self.getPID()
055489	410	status=self.getStatus()
25296b	411	if(self.statistics.fileOK):
SP	412	ETA=str(datetime.timedelta(microseconds=(int(self.tape.config['iterations'])-int(self.statistics.last))self.statistics.dT)1000000)
055489	413	if(status==TS_NONEXISTANT or status==TS_NOLOCK):
SP	414	statustxt="Not running"
9154b3	415	pid=""
38cb4a	416	ETA=""
055489	417	elif status==TS_STOPPED:
SP	418	statustxt="Stopped"
38cb4a	419	ETA="N/A"
3d0247	420	elif status==TS_COMPLETED:
SP	421	statustxt="Completed"
38cb4a	422	pid=""
SP	423	ETA=""
055489	424	else:
SP	425	statustxt="Running"
9154b3	426
SP	427	if(self.statistics.fileOK):
38cb4a	428	report=[time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(self.statistics.startDate))),ETA, statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
9154b3	429	else:
25296b	430	report=["N/A","N/A",statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
f81585	431	return report
c14bd6	432
055489	433	def writeComment(self, data, mode='w'):
9154b3	434	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
SP	435	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
055489	436	self.Comment.writefile(data,mode=mode)
9154b3	437
bc14fb	438	def __str__(self):
c14bd6	439	if(self.getStatus()==0):
SP	440	str=" not running."
	441	else:
	442	str=" running."
	443	return(self.Dir.fullpath()+str)
	444
bc14fb	445