trisurf-manager.git

Samo Penic

2017-11-28 824b3e562b5e0e7f9450b06a45deabda6ebe20df

commit \| author \| age
8ab985	1	import configobj
SP	2	import xml.etree.ElementTree as ET
	3	import base64
	4	import zlib
	5	import sys,io
	6	import os
	7	from itertools import islice
	8	import mmap
	9	import shlex
	10	import psutil
	11	import time
	12	import datetime
	13	import subprocess
	14	import shutil
	15
	16	# Process status
	17	TS_NOLOCK=0 # lock file does not exist
	18	TS_NONEXISTANT=0 # process is not in the list of processes
	19	TS_STOPPED=1 # the process is listed, but is in stopped state
	20	TS_RUNNING=2 # process is running
	21	TS_COMPLETED=3 #simulation is completed
	22
	23	class FileContent:
	24	'''
	25	Class is helpful for reading and writting the specific files.
	26	'''
	27	def __init__(self,filename):
	28	''' The instance is done by calling constructor FileContent(filename)
	29
	30	The object then reads filename if it exists, otherwise the data is empty string.
	31	User may force to reread file by calling the readline() method of the class.
	32
	33	Filename is stored in local variable for future file operations.
	34	'''
	35
	36	self.filename=filename
	37	self.readfile()
	38
	39	def readfile(self):
	40	'''Force reread of the file and setting the data'''
	41	self.data=""
	42	try:
	43	with open (self.filename, "r") as myfile:
	44	self.data=myfile.read().replace('\n', '') #read the file and remove newline from the text
	45	except:
	46	pass # does nothing if error occurs
	47
	48
	49	def writefile(self, data, mode='w'):
	50	'''File may be updated by completely rewritting the file contents or appending the data to the end of the file.
	51	this is achieved by calling writefile(data, mode) method, where data is the string data to be written and
	52	mode can be 'a' for append and 'w' for writting the file anew.
	53	'''
	54	with open (self.filename, mode) as myfile:
	55	myfile.write(data)
	56
	57	def getText(self):
	58	'''
	59	Method getText() or calling the object itself returns string of data
	60	'''
	61	return self.data
	62
	63	def __str__(self):
	64	'''
	65	Method getText() or calling the object itself returns string of data
	66	'''
	67	return self.getText()
	68
	69	class Tape:
	70	'''
	71	Special class that manages configuration of trisurf (commonly named tape). It can read and parse configuration from disk or parse it from string.
	72	'''
	73
	74	def __init__(self):
	75	'''The object is instatiated by calling Tape() constructor without parameters'''
	76	return
	77
	78	def readTape(self, tape='tape'):
	79	'''
	80	Tape is read and parsed by calling the readTape() method with optional tape parameter, which is full path to filename where the configuration is stored.
	81	If the tape cannot be read, it prints error and exits.
	82	'''
	83	try:
	84	self.config=configobj.ConfigObj(tape)
	85	with open (tape, "r") as myfile:
	86	self.rawText=myfile.read() #read the file
	87
	88	except:
	89	print("Error reading or parsing tape file!\n")
	90	exit(1)
	91
	92	def setTape(self, string):
	93	'''Method setTape(string) parses the string in memory that hold the tape contents.'''
	94	self.config=configobj.ConfigObj(io.StringIO(string))
	95	self.rawText=string
	96	return
	97
	98	def getValue(self,key):
	99	'''Method getValue(key) returns value of a single parsed setting named "key".'''
	100
	101	return self.config[key]
	102
	103	def __str__(self):
	104	'''Calling the object itself, it recreates the tape contents from parsed values in form of key=value.'''
	105	retval=""
	106	for key,val in self.config.iteritems():
	107	retval=retval+str(key)+" = "+str(val)+"\n"
	108	return retval
	109
	110
	111
	112	class Directory:
	113	'''
	114	Class deals with the paths where the simulation is run and data is stored.
	115	'''
	116	def __init__(self, maindir=".", simdir=""):
	117	'''Initialization Directory() takes two optional parameters, namely maindir and simdir. Defaults to current directory. It sets local variables maindir and simdir accordingly.'''
	118	self.maindir=maindir
	119	self.simdir=simdir
	120	return
	121
	122	def fullpath(self):
	123	'''
	124	Method returns string of path where the data is stored. It combines values of maindir and simdir as maindir/simdir on Unix.
	125	'''
	126	return os.path.join(self.maindir,self.simdir)
	127
	128	def exists(self):
	129	''' Method checks whether the directory specified by fullpath() exists. It return True/False on completion.'''
	130	path=self.fullpath()
	131	if(os.path.exists(path)):
	132	return True
	133	else:
	134	return False
	135
	136	def make(self):
	137	''' Method make() creates directory. If it fails it exits the program with error message.'''
	138	try:
	139	os.makedirs(self.fullpath())
	140	except:
	141	print("Cannot make directory "+self.fullpath()+"\n")
	142	exit(1)
	143	return
	144
	145	def makeifnotexist(self):
	146	'''Method makeifnotexist() creates directory if it does not exist.'''
	147	if(self.exists()==0):
	148	self.make()
	149	return True
	150	else:
	151	return False
	152
	153	def remove(self):
	154	'''Method remove() removes directory recursively. WARNING! No questions asked.'''
	155	if(self.exists()):
	156	try:
	157	os.rmdir(self.fullpath())
	158	except:
	159	print("Cannot remove directory "+self.fullpath()+ "\n")
	160	exit(1)
	161	return
	162
	163	def goto(self):
	164	'''
	165	Method goto() moves current directory to the one specified by fullpath(). WARNING: when using the relative paths, do not call this function multiple times.
	166	'''
	167	try:
	168	os.chdir(self.fullpath())
	169	except:
	170	print("Cannot go to directory "+self.fullpath()+"\n")
	171	return
	172
	173
	174	class Statistics:
	175	'''
	176	Class that deals with the statistics file from the simulations.
	177	File is generally large and not all data is needed, so it is dealt with in a specific way.
	178	'''
	179
	180	def __init__(self,path,filename="statistics.csv"):
	181	'''
	182	At the initialization call it receives optional filename parameter specifying the path and filename of the statistics file.
	183
	184	The local variables path, filename, fullname (joined path and filename) and private check if the file exists are stored.
	185	'''
	186	self.path=path
	187	self.filename=filename
	188	self.fullname=os.path.join(path,filename)
	189	return
	190
	191	def exists(self):
	192	'''Method check if the statistics file exists.'''
	193	if(os.path.isfile(self.fullname)):
	194	return True
	195	else:
	196	return False
	197
5795ac	198	def lineCount(self):
8ab985	199	'''
SP	200	Internal method for determining the number of the lines in the most efficient way. Is it really the most efficient?
	201	'''
	202	f = open(self.fullname, "r+")
	203	try:
	204	buf = mmap.mmap(f.fileno(), 0)
	205	lines = 0
	206	readline = buf.readline
	207	while readline():
	208	lines += 1
	209	f.close()
	210	except:
	211	lines=0
	212	f.close()
	213	return lines
	214
5795ac	215	def tail(self):
SP	216	with open(self.fullname,'r') as myfile:
8ab985	217	lines=myfile.readlines()
SP	218	return [lines[len(lines)-2].replace('\n',''),lines[len(lines)-1].replace('\n','')]
	219
5795ac	220	def checkFileValidity(self):
8ab985	221	try:
5795ac	222	lines=self.tail()
8ab985	223	except:
SP	224	return(False)
	225	if len(lines)<2:
	226	return(False)
5795ac	227	return(True)
SP	228
	229	def getSimulationDeltaTime(self):
	230	try:
	231	lines=self.tail()
	232	except:
	233	return 0
	234
8ab985	235	fields=shlex.split(lines[0])
SP	236	epoch1=fields[0]
	237	n1=fields[1]
	238
	239	fields=shlex.split(lines[1])
	240	epoch2=fields[0]
	241	n2=fields[1]
	242	try:
5795ac	243	dT=int(epoch2)-int(epoch1)
8ab985	244	except:
5795ac	245	return 0
SP	246	return dT
	247
	248	def getLastIterationInStatistics(self):
	249	try:
	250	lines=self.tail()
	251	except:
	252	return 0
	253
	254	fields=shlex.split(lines[0])
	255	epoch1=fields[0]
	256	n1=fields[1]
	257
	258	fields=shlex.split(lines[1])
	259	epoch2=fields[0]
	260	return (fields[1])
99bb43	261
SP	262	def getColumn(self,n=0):
	263	lines=self.readText().splitlines()
	264	col=[]
	265	name=lines[0].split()[n]
	266	for line in lines:
	267	col.append(line.split()[n])
	268	return col[2:],name
	269
	270	def getTable(self):
	271	lines=self.readText().splitlines()
	272	keys=lines[0].split()
	273	table={}
	274	def num(s):
	275	try:
	276	return int(s)
	277	except ValueError:
	278	return float(s)
	279	for key in keys:
	280	table[key]=[]
	281	for line in lines:
	282	fields=line.split()
f0131d	283	for i in range(0,len(keys)):
99bb43	284	try:
SP	285	value=num(fields[i])
	286	table[keys[i]].append(value)
	287	except:
	288	pass
	289	return table
5795ac	290
8ab985	291
SP	292	def readText(self):
	293	with open(self.fullname, 'r+') as fin:
	294	cont=fin.read()
	295	return cont
	296
	297	def __str__(self):
	298	'''
	299	Prints the full path with filename of the statistics.csv file
	300	'''
	301	return(str(self.fullname))
	302
	303
	304
	305	class Runner:
	306	'''
	307	Class Runner consists of a single running or terminated instance of the trisurf. It manages starting, stopping, verifying the running process and printing the reports of the configured instances.
	308	'''
	309
	310	@property
	311	def Dir(self):
	312	return Directory(maindir=self.maindir,simdir=self.subdir)
	313
	314
	315	@property
	316	def Statistics(self):
	317	return Statistics(self.Dir.fullpath(), "statistics.csv")
	318
	319	def __init__(self, subdir='run0', tape=None, snapshot=None, runArgs=[]):
	320	self.subdir=subdir
	321	self.runArgs=runArgs
	322	self.isFromSnapshot=False
	323	if(tape!=None):
	324	self.initFromTape(tape)
	325	if(snapshot!=None):
	326	self.initFromSnapshot(snapshot)
	327	return
	328
	329
	330	def initFromTape(self, tape):
	331	self.Tape=Tape()
	332	self.Tape.readTape(tape)
	333	self.tapeFilename=tape
	334
	335	def initFromSnapshot(self, snapshotfile):
	336	try:
	337	tree = ET.parse(snapshotfile)
	338	except:
	339	print("Error reading snapshot file")
	340	exit(1)
	341	self.isFromSnapshot=True
	342	self.snapshotFile=snapshotfile
	343	root = tree.getroot()
	344	tapetxt=root.find('tape')
	345	version=root.find('trisurfversion')
	346	self.Tape=Tape()
	347	self.Tape.setTape(tapetxt.text)
	348
	349	def getPID(self):
	350	try:
	351	fp = open(os.path.join(self.Dir.fullpath(),'.lock'))
	352	except IOError as e:
	353	return 0 #file probably does not exist. e==2??
	354	pid=fp.readline()
	355	fp.close()
	356	return int(pid)
	357
	358	def getLastIteration(self):
	359	try:
	360	fp = open(os.path.join(self.Dir.fullpath(),'.status'))
	361	except IOError as e:
	362	return -1 #file probably does not exist. e==2??
	363	status=fp.readline()
	364	fp.close()
	365	return int(status)
	366
	367	def isCompleted(self):
	368	if int(self.Tape.getValue("iterations"))+int(self.Tape.getValue("inititer"))==self.getLastIteration()+1:
	369	return True
	370	else:
	371	return False
5795ac	372
SP	373
	374	def getStartTime(self):
	375	try:
	376	return os.path.getmtime(os.path.join(self.Dir.fullpath(),'.lock'))
	377	except:
	378	return -1
8ab985	379
SP	380	def getStatus(self):
	381	pid=self.getPID()
	382	if(self.isCompleted()):
	383	return TS_COMPLETED
	384	if(pid==0):
	385	return TS_NOLOCK
	386	if(psutil.pid_exists(int(pid))):
	387	proc= psutil.Process(int(pid))
	388	#psutil.__version__ == '3.4.2' requires name() and status(), some older versions reguire name, status
	389	if(psutil.__version__>='2.0.0'):
	390	procname=proc.name()
	391	procstat=proc.status()
	392	else:
	393	procname=proc.name
	394	procstat=proc.status
	395	if procname=="trisurf":
	396	if procstat=="stopped":
	397	return TS_STOPPED
	398	else:
	399	return TS_RUNNING
	400	else:
	401	return TS_NONEXISTANT
	402	else:
	403	return TS_NONEXISTANT
	404
	405	def start(self):
	406	if(self.getStatus()==0 or self.getStatus()==TS_COMPLETED):
	407	#check if executable exists
	408	if(shutil.which('trisurf')==None):
	409	print("Error. Trisurf executable not found in PATH. Please install trisurf prior to running trisurf manager.")
	410	exit(1)
	411	#Symlinks tape file to the directory or create tape file from snapshot in the direcory...
	412	if(self.Dir.makeifnotexist()):
	413	if(self.isFromSnapshot==False):
	414	try:
	415	os.symlink(os.path.abspath(self.tapeFilename), self.Dir.fullpath()+"/tape")
	416	except:
	417	print("Error while symlinking "+os.path.abspath(self.tapeFilename)+" to "+self.Dir.fullpath()+"/tape")
	418	exit(1)
	419	else:
	420	try:
	421	with open (os.path.join(self.Dir.fullpath(),"tape"), "w") as myfile:
	422	#myfile.write("#This is automatically generated tape file from snapshot")
	423	myfile.write(str(self.Tape.rawText))
	424	except:
	425	print("Error -- cannot make tapefile "+ os.path.join(self.Dir.fullpath(),"tape")+" from the snapshot in the running directory")
	426	exit(1)
	427	try:
	428	os.symlink(os.path.abspath(self.snapshotFile), os.path.join(self.Dir.fullpath(),"initial_snapshot.vtu"))
	429	except:
	430	print("Error while symlinking "+os.path.abspath(self.snapshotFile)+" to "+os.path.join(self.Dir.fullpath(),self.snapshotFile))
	431
	432	#check if the simulation has been completed. in this case notify user and stop executing.
	433	if(self.isCompleted() and ("--force-from-tape" not in self.runArgs) and ("--reset-iteration-count" not in self.runArgs)):
	434	print("The simulation was completed. Not starting executable in "+self.Dir.fullpath())
	435	return
	436
	437	cwd=Directory(maindir=os.getcwd())
	438	lastVTU=self.getLastVTU() #we get last VTU file in case we need to continue the simulation from last snapshot. Need to be done before the Dir.goto() call.
	439	self.Dir.goto()
	440	print("Starting trisurf-ng executable in "+self.Dir.fullpath())
	441	if(self.isFromSnapshot==True):
	442	#here we try to determine whether we should continue the simulation or start from last known VTU snapshot.
	443	if(lastVTU==None):
	444	initSnap="initial_snapshot.vtu"
	445	else:
	446	initSnap=lastVTU
	447	print("WARNING: Not using initial snapshot as starting point, but selecting "+initSnap+" as a starting vesicle")
	448	params=["trisurf", "--restore-from-vtk",initSnap]+self.runArgs
	449	print("InitSnap is: "+initSnap)
	450	else:
	451	#veify if dump exists. If not it is a first run and shoud be run with --force-from-tape
	452	if(os.path.isfile("dump.bin")==False):
	453	self.runArgs.append("--force-from-tape")
	454	params=["trisurf"]+self.runArgs
	455	subprocess.Popen (params, stdout=subprocess.DEVNULL)
	456	cwd.goto()
	457	else:
	458	print("Process in "+self.Dir.fullpath()+" already running. Not starting.")
	459	return
	460
	461
	462	def setMaindir(self,prefix,variables):
	463	maindir=""
	464	for p,v in zip(prefix,variables):
824b3e	465	#if(v=="xk0"):
SP	466	# tv=str(round(float(self.Tape.config[v])))
	467	# if sys.version_info<(3,0):
	468	# tv=str(int(float(self.Tape.config[v])))
	469	#else:
	470	tv=str(self.Tape.config[v])
8ab985	471	maindir=maindir+p+tv
SP	472	self.maindir=maindir
	473	return
	474
	475	def setSubdir(self, subdir="run0"):
	476	self.subdir=subdir
	477	return
	478
	479	def getStatistics(self, statfile="statistics.csv"):
	480	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
	481	pid=self.getPID()
	482	status=self.getStatus()
5795ac	483	if(self.Statistics.checkFileValidity()):
SP	484	ETA=str(datetime.timedelta(microseconds=(int(self.Tape.config['iterations'])-int(self.Statistics.getLastIterationInStatistics()))self.Statistics.getSimulationDeltaTime())1000000)
8ab985	485	if(status==TS_NONEXISTANT or status==TS_NOLOCK):
SP	486	statustxt="Not running"
	487	pid=""
	488	ETA=""
	489	elif status==TS_STOPPED:
	490	statustxt="Stopped"
	491	ETA="N/A"
	492	elif status==TS_COMPLETED:
	493	statustxt="Completed"
	494	pid=""
	495	ETA=""
	496	else:
	497	statustxt="Running"
	498
5795ac	499	if(self.Statistics.checkFileValidity()):
SP	500	report=[time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(self.getStartTime()))),ETA, statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
8ab985	501	else:
SP	502	report=["N/A","N/A",statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
	503	return report
	504
	505
	506	def stop(self):
	507	try:
	508	p=psutil.Process(self.getPID())
	509	p.kill()
	510	except:
	511	print("Could not stop the process. Is the process running? Do you have sufficient privileges?")
	512
	513
	514	def writeComment(self, data, mode='w'):
	515	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
	516	self.Comment.writefile(data,mode=mode)
	517
	518
f49271	519	def getLastVTUold(self):
8ab985	520	vtuidx=self.getLastIteration()-int(self.Tape.getValue("inititer"))
SP	521	if vtuidx<0:
	522	return None
	523	else:
	524	return 'timestep_{:06d}.vtu'.format(vtuidx)
	525
f49271	526	def getLastVTU(self):
SP	527	flist=[]
	528	for file in os.listdir(self.Dir.fullpath()):
	529	if file.endswith(".vtu"):
	530	flist.append(file)
52e871	531	flist.sort()
SP	532	if(len(flist)==0):
	533	return -1
	534	else:
	535	return(flist[-1])
f49271	536
8ab985	537	def __str__(self):
SP	538	if(self.getStatus()==0):
	539	str=" not running."
	540	else:
	541	str=" running."
	542	return(self.Dir.fullpath()+str)
	543
57b1a6	544	def __repr__(self):
SP	545	return("Instance of trisurf in "+self.Dir.fullpath())