trisurf-manager.git

Samo Penic

2019-08-14 8f46a7926d3517bf5405fc5a8c723e6cc318ad58

commit \| author \| age
8ab985	1	import configobj
SP	2	import xml.etree.ElementTree as ET
	3	import base64
	4	import zlib
	5	import sys,io
	6	import os
	7	from itertools import islice
	8	import mmap
	9	import shlex
	10	import psutil
	11	import time
	12	import datetime
	13	import subprocess
	14	import shutil
	15
	16	# Process status
	17	TS_NOLOCK=0 # lock file does not exist
	18	TS_NONEXISTANT=0 # process is not in the list of processes
	19	TS_STOPPED=1 # the process is listed, but is in stopped state
	20	TS_RUNNING=2 # process is running
	21	TS_COMPLETED=3 #simulation is completed
	22
c1504d	23	#namespace variable. Seems the best place to put the variable in :)
SP	24	_analysis_list={}
	25
8ab985	26	class FileContent:
SP	27	'''
	28	Class is helpful for reading and writting the specific files.
	29	'''
	30	def __init__(self,filename):
	31	''' The instance is done by calling constructor FileContent(filename)
	32
	33	The object then reads filename if it exists, otherwise the data is empty string.
	34	User may force to reread file by calling the readline() method of the class.
	35
	36	Filename is stored in local variable for future file operations.
	37	'''
	38
	39	self.filename=filename
	40	self.readfile()
	41
	42	def readfile(self):
	43	'''Force reread of the file and setting the data'''
	44	self.data=""
	45	try:
	46	with open (self.filename, "r") as myfile:
	47	self.data=myfile.read().replace('\n', '') #read the file and remove newline from the text
	48	except:
	49	pass # does nothing if error occurs
	50
	51
	52	def writefile(self, data, mode='w'):
	53	'''File may be updated by completely rewritting the file contents or appending the data to the end of the file.
	54	this is achieved by calling writefile(data, mode) method, where data is the string data to be written and
	55	mode can be 'a' for append and 'w' for writting the file anew.
	56	'''
	57	with open (self.filename, mode) as myfile:
	58	myfile.write(data)
	59
	60	def getText(self):
	61	'''
	62	Method getText() or calling the object itself returns string of data
	63	'''
	64	return self.data
	65
	66	def __str__(self):
	67	'''
	68	Method getText() or calling the object itself returns string of data
	69	'''
	70	return self.getText()
	71
	72	class Tape:
	73	'''
	74	Special class that manages configuration of trisurf (commonly named tape). It can read and parse configuration from disk or parse it from string.
	75	'''
	76
	77	def __init__(self):
	78	'''The object is instatiated by calling Tape() constructor without parameters'''
	79	return
	80
	81	def readTape(self, tape='tape'):
	82	'''
	83	Tape is read and parsed by calling the readTape() method with optional tape parameter, which is full path to filename where the configuration is stored.
	84	If the tape cannot be read, it prints error and exits.
	85	'''
	86	try:
	87	self.config=configobj.ConfigObj(tape)
	88	with open (tape, "r") as myfile:
	89	self.rawText=myfile.read() #read the file
	90
	91	except:
	92	print("Error reading or parsing tape file!\n")
	93	exit(1)
	94
	95	def setTape(self, string):
	96	'''Method setTape(string) parses the string in memory that hold the tape contents.'''
	97	self.config=configobj.ConfigObj(io.StringIO(string))
	98	self.rawText=string
	99	return
	100
	101	def getValue(self,key):
	102	'''Method getValue(key) returns value of a single parsed setting named "key".'''
	103
	104	return self.config[key]
	105
	106	def __str__(self):
	107	'''Calling the object itself, it recreates the tape contents from parsed values in form of key=value.'''
	108	retval=""
	109	for key,val in self.config.iteritems():
	110	retval=retval+str(key)+" = "+str(val)+"\n"
	111	return retval
	112
	113
	114
	115	class Directory:
	116	'''
	117	Class deals with the paths where the simulation is run and data is stored.
	118	'''
	119	def __init__(self, maindir=".", simdir=""):
	120	'''Initialization Directory() takes two optional parameters, namely maindir and simdir. Defaults to current directory. It sets local variables maindir and simdir accordingly.'''
	121	self.maindir=maindir
	122	self.simdir=simdir
	123	return
	124
	125	def fullpath(self):
	126	'''
	127	Method returns string of path where the data is stored. It combines values of maindir and simdir as maindir/simdir on Unix.
	128	'''
	129	return os.path.join(self.maindir,self.simdir)
	130
	131	def exists(self):
	132	''' Method checks whether the directory specified by fullpath() exists. It return True/False on completion.'''
	133	path=self.fullpath()
	134	if(os.path.exists(path)):
	135	return True
	136	else:
	137	return False
	138
	139	def make(self):
	140	''' Method make() creates directory. If it fails it exits the program with error message.'''
	141	try:
	142	os.makedirs(self.fullpath())
	143	except:
	144	print("Cannot make directory "+self.fullpath()+"\n")
	145	exit(1)
	146	return
	147
	148	def makeifnotexist(self):
	149	'''Method makeifnotexist() creates directory if it does not exist.'''
	150	if(self.exists()==0):
	151	self.make()
	152	return True
	153	else:
	154	return False
	155
	156	def remove(self):
	157	'''Method remove() removes directory recursively. WARNING! No questions asked.'''
	158	if(self.exists()):
	159	try:
	160	os.rmdir(self.fullpath())
	161	except:
	162	print("Cannot remove directory "+self.fullpath()+ "\n")
	163	exit(1)
	164	return
	165
	166	def goto(self):
	167	'''
	168	Method goto() moves current directory to the one specified by fullpath(). WARNING: when using the relative paths, do not call this function multiple times.
	169	'''
	170	try:
	171	os.chdir(self.fullpath())
	172	except:
	173	print("Cannot go to directory "+self.fullpath()+"\n")
	174	return
	175
	176
	177	class Statistics:
	178	'''
	179	Class that deals with the statistics file from the simulations.
	180	File is generally large and not all data is needed, so it is dealt with in a specific way.
	181	'''
	182
	183	def __init__(self,path,filename="statistics.csv"):
	184	'''
	185	At the initialization call it receives optional filename parameter specifying the path and filename of the statistics file.
	186
	187	The local variables path, filename, fullname (joined path and filename) and private check if the file exists are stored.
	188	'''
	189	self.path=path
	190	self.filename=filename
	191	self.fullname=os.path.join(path,filename)
	192	return
	193
	194	def exists(self):
	195	'''Method check if the statistics file exists.'''
	196	if(os.path.isfile(self.fullname)):
	197	return True
	198	else:
	199	return False
	200
5795ac	201	def lineCount(self):
8ab985	202	'''
SP	203	Internal method for determining the number of the lines in the most efficient way. Is it really the most efficient?
	204	'''
	205	f = open(self.fullname, "r+")
	206	try:
	207	buf = mmap.mmap(f.fileno(), 0)
	208	lines = 0
	209	readline = buf.readline
	210	while readline():
	211	lines += 1
	212	f.close()
	213	except:
	214	lines=0
	215	f.close()
	216	return lines
	217
5795ac	218	def tail(self):
SP	219	with open(self.fullname,'r') as myfile:
8ab985	220	lines=myfile.readlines()
SP	221	return [lines[len(lines)-2].replace('\n',''),lines[len(lines)-1].replace('\n','')]
	222
5795ac	223	def checkFileValidity(self):
8ab985	224	try:
5795ac	225	lines=self.tail()
8ab985	226	except:
SP	227	return(False)
	228	if len(lines)<2:
	229	return(False)
5795ac	230	return(True)
SP	231
	232	def getSimulationDeltaTime(self):
	233	try:
	234	lines=self.tail()
	235	except:
	236	return 0
	237
8ab985	238	fields=shlex.split(lines[0])
SP	239	epoch1=fields[0]
	240	n1=fields[1]
	241
	242	fields=shlex.split(lines[1])
	243	epoch2=fields[0]
	244	n2=fields[1]
	245	try:
5795ac	246	dT=int(epoch2)-int(epoch1)
8ab985	247	except:
5795ac	248	return 0
SP	249	return dT
	250
	251	def getLastIterationInStatistics(self):
	252	try:
	253	lines=self.tail()
	254	except:
	255	return 0
	256
	257	fields=shlex.split(lines[0])
	258	epoch1=fields[0]
	259	n1=fields[1]
	260
	261	fields=shlex.split(lines[1])
	262	epoch2=fields[0]
	263	return (fields[1])
99bb43	264
SP	265	def getColumn(self,n=0):
	266	lines=self.readText().splitlines()
	267	col=[]
	268	name=lines[0].split()[n]
	269	for line in lines:
	270	col.append(line.split()[n])
	271	return col[2:],name
	272
	273	def getTable(self):
	274	lines=self.readText().splitlines()
	275	keys=lines[0].split()
	276	table={}
	277	def num(s):
	278	try:
	279	return int(s)
	280	except ValueError:
	281	return float(s)
	282	for key in keys:
	283	table[key]=[]
	284	for line in lines:
	285	fields=line.split()
f0131d	286	for i in range(0,len(keys)):
99bb43	287	try:
SP	288	value=num(fields[i])
	289	table[keys[i]].append(value)
	290	except:
	291	pass
	292	return table
5795ac	293
8ab985	294
SP	295	def readText(self):
	296	with open(self.fullname, 'r+') as fin:
	297	cont=fin.read()
	298	return cont
	299
	300	def __str__(self):
	301	'''
	302	Prints the full path with filename of the statistics.csv file
	303	'''
	304	return(str(self.fullname))
	305
	306
	307
	308	class Runner:
	309	'''
	310	Class Runner consists of a single running or terminated instance of the trisurf. It manages starting, stopping, verifying the running process and printing the reports of the configured instances.
	311	'''
	312
	313	@property
	314	def Dir(self):
	315	return Directory(maindir=self.maindir,simdir=self.subdir)
	316
	317
	318	@property
	319	def Statistics(self):
	320	return Statistics(self.Dir.fullpath(), "statistics.csv")
	321
	322	def __init__(self, subdir='run0', tape=None, snapshot=None, runArgs=[]):
	323	self.subdir=subdir
	324	self.runArgs=runArgs
	325	self.isFromSnapshot=False
	326	if(tape!=None):
	327	self.initFromTape(tape)
	328	if(snapshot!=None):
	329	self.initFromSnapshot(snapshot)
	330	return
	331
	332
	333	def initFromTape(self, tape):
	334	self.Tape=Tape()
	335	self.Tape.readTape(tape)
	336	self.tapeFilename=tape
	337
	338	def initFromSnapshot(self, snapshotfile):
	339	try:
	340	tree = ET.parse(snapshotfile)
	341	except:
	342	print("Error reading snapshot file")
	343	exit(1)
	344	self.isFromSnapshot=True
	345	self.snapshotFile=snapshotfile
	346	root = tree.getroot()
	347	tapetxt=root.find('tape')
	348	version=root.find('trisurfversion')
	349	self.Tape=Tape()
	350	self.Tape.setTape(tapetxt.text)
	351
	352	def getPID(self):
	353	try:
	354	fp = open(os.path.join(self.Dir.fullpath(),'.lock'))
	355	except IOError as e:
	356	return 0 #file probably does not exist. e==2??
	357	pid=fp.readline()
	358	fp.close()
	359	return int(pid)
	360
	361	def getLastIteration(self):
	362	try:
	363	fp = open(os.path.join(self.Dir.fullpath(),'.status'))
	364	except IOError as e:
	365	return -1 #file probably does not exist. e==2??
	366	status=fp.readline()
	367	fp.close()
	368	return int(status)
	369
	370	def isCompleted(self):
	371	if int(self.Tape.getValue("iterations"))+int(self.Tape.getValue("inititer"))==self.getLastIteration()+1:
	372	return True
	373	else:
	374	return False
5795ac	375
SP	376
	377	def getStartTime(self):
	378	try:
	379	return os.path.getmtime(os.path.join(self.Dir.fullpath(),'.lock'))
	380	except:
	381	return -1
8ab985	382
SP	383	def getStatus(self):
	384	pid=self.getPID()
	385	if(self.isCompleted()):
	386	return TS_COMPLETED
	387	if(pid==0):
	388	return TS_NOLOCK
	389	if(psutil.pid_exists(int(pid))):
	390	proc= psutil.Process(int(pid))
	391	#psutil.__version__ == '3.4.2' requires name() and status(), some older versions reguire name, status
	392	if(psutil.__version__>='2.0.0'):
	393	procname=proc.name()
	394	procstat=proc.status()
	395	else:
	396	procname=proc.name
	397	procstat=proc.status
	398	if procname=="trisurf":
	399	if procstat=="stopped":
	400	return TS_STOPPED
	401	else:
	402	return TS_RUNNING
	403	else:
	404	return TS_NONEXISTANT
	405	else:
	406	return TS_NONEXISTANT
	407
	408	def start(self):
	409	if(self.getStatus()==0 or self.getStatus()==TS_COMPLETED):
	410	#check if executable exists
	411	if(shutil.which('trisurf')==None):
	412	print("Error. Trisurf executable not found in PATH. Please install trisurf prior to running trisurf manager.")
	413	exit(1)
	414	#Symlinks tape file to the directory or create tape file from snapshot in the direcory...
	415	if(self.Dir.makeifnotexist()):
	416	if(self.isFromSnapshot==False):
	417	try:
	418	os.symlink(os.path.abspath(self.tapeFilename), self.Dir.fullpath()+"/tape")
	419	except:
	420	print("Error while symlinking "+os.path.abspath(self.tapeFilename)+" to "+self.Dir.fullpath()+"/tape")
	421	exit(1)
	422	else:
	423	try:
	424	with open (os.path.join(self.Dir.fullpath(),"tape"), "w") as myfile:
	425	#myfile.write("#This is automatically generated tape file from snapshot")
	426	myfile.write(str(self.Tape.rawText))
	427	except:
	428	print("Error -- cannot make tapefile "+ os.path.join(self.Dir.fullpath(),"tape")+" from the snapshot in the running directory")
	429	exit(1)
	430	try:
	431	os.symlink(os.path.abspath(self.snapshotFile), os.path.join(self.Dir.fullpath(),"initial_snapshot.vtu"))
	432	except:
	433	print("Error while symlinking "+os.path.abspath(self.snapshotFile)+" to "+os.path.join(self.Dir.fullpath(),self.snapshotFile))
	434
	435	#check if the simulation has been completed. in this case notify user and stop executing.
	436	if(self.isCompleted() and ("--force-from-tape" not in self.runArgs) and ("--reset-iteration-count" not in self.runArgs)):
	437	print("The simulation was completed. Not starting executable in "+self.Dir.fullpath())
	438	return
	439
	440	cwd=Directory(maindir=os.getcwd())
	441	lastVTU=self.getLastVTU() #we get last VTU file in case we need to continue the simulation from last snapshot. Need to be done before the Dir.goto() call.
	442	self.Dir.goto()
	443	print("Starting trisurf-ng executable in "+self.Dir.fullpath())
	444	if(self.isFromSnapshot==True):
	445	#here we try to determine whether we should continue the simulation or start from last known VTU snapshot.
	446	if(lastVTU==None):
	447	initSnap="initial_snapshot.vtu"
	448	else:
	449	initSnap=lastVTU
	450	print("WARNING: Not using initial snapshot as starting point, but selecting "+initSnap+" as a starting vesicle")
	451	params=["trisurf", "--restore-from-vtk",initSnap]+self.runArgs
	452	print("InitSnap is: "+initSnap)
	453	else:
	454	#veify if dump exists. If not it is a first run and shoud be run with --force-from-tape
	455	if(os.path.isfile("dump.bin")==False):
	456	self.runArgs.append("--force-from-tape")
	457	params=["trisurf"]+self.runArgs
	458	subprocess.Popen (params, stdout=subprocess.DEVNULL)
	459	cwd.goto()
	460	else:
	461	print("Process in "+self.Dir.fullpath()+" already running. Not starting.")
	462	return
	463
	464
	465	def setMaindir(self,prefix,variables):
	466	maindir=""
	467	for p,v in zip(prefix,variables):
84450d	468	tv=str(self.Tape.config[v])
8ab985	469	maindir=maindir+p+tv
SP	470	self.maindir=maindir
	471	return
	472
	473	def setSubdir(self, subdir="run0"):
	474	self.subdir=subdir
	475	return
	476
	477	def getStatistics(self, statfile="statistics.csv"):
	478	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
	479	pid=self.getPID()
	480	status=self.getStatus()
5795ac	481	if(self.Statistics.checkFileValidity()):
SP	482	ETA=str(datetime.timedelta(microseconds=(int(self.Tape.config['iterations'])-int(self.Statistics.getLastIterationInStatistics()))self.Statistics.getSimulationDeltaTime())1000000)
8ab985	483	if(status==TS_NONEXISTANT or status==TS_NOLOCK):
SP	484	statustxt="Not running"
	485	pid=""
	486	ETA=""
	487	elif status==TS_STOPPED:
	488	statustxt="Stopped"
	489	ETA="N/A"
	490	elif status==TS_COMPLETED:
	491	statustxt="Completed"
	492	pid=""
	493	ETA=""
	494	else:
	495	statustxt="Running"
	496
5795ac	497	if(self.Statistics.checkFileValidity()):
SP	498	report=[time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(self.getStartTime()))),ETA, statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
8ab985	499	else:
SP	500	report=["N/A","N/A",statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
	501	return report
	502
	503
	504	def stop(self):
	505	try:
	506	p=psutil.Process(self.getPID())
	507	p.kill()
	508	except:
	509	print("Could not stop the process. Is the process running? Do you have sufficient privileges?")
	510
	511
	512	def writeComment(self, data, mode='w'):
	513	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
	514	self.Comment.writefile(data,mode=mode)
	515
	516
f49271	517	def getLastVTUold(self):
8ab985	518	vtuidx=self.getLastIteration()-int(self.Tape.getValue("inititer"))
SP	519	if vtuidx<0:
	520	return None
	521	else:
	522	return 'timestep_{:06d}.vtu'.format(vtuidx)
	523
f49271	524	def getLastVTU(self):
SP	525	flist=[]
	526	for file in os.listdir(self.Dir.fullpath()):
	527	if file.endswith(".vtu"):
	528	flist.append(file)
0d561c	529	flist.sort()
SP	530	if(len(flist)==0):
	531	return(-1)
	532	else:
	533	return(flist[-1])
f49271	534
8ab985	535	def __str__(self):
SP	536	if(self.getStatus()==0):
	537	str=" not running."
	538	else:
	539	str=" running."
	540	return(self.Dir.fullpath()+str)
	541
57b1a6	542	def __repr__(self):
SP	543	return("Instance of trisurf in "+self.Dir.fullpath())