trisurf-manager.git

Samo Penic

2017-01-05 5795acb1babc896b7edb508deed2fe7edbb02fc1

commit \| author \| age
8ab985	1	import configobj
SP	2	import xml.etree.ElementTree as ET
	3	import base64
	4	import zlib
	5	import sys,io
	6	import os
	7	from itertools import islice
	8	import mmap
	9	import shlex
	10	import psutil
	11	import time
	12	import datetime
	13	import subprocess
	14	import shutil
	15
	16	# Process status
	17	TS_NOLOCK=0 # lock file does not exist
	18	TS_NONEXISTANT=0 # process is not in the list of processes
	19	TS_STOPPED=1 # the process is listed, but is in stopped state
	20	TS_RUNNING=2 # process is running
	21	TS_COMPLETED=3 #simulation is completed
	22
	23	class FileContent:
	24	'''
	25	Class is helpful for reading and writting the specific files.
	26	'''
	27	def __init__(self,filename):
	28	''' The instance is done by calling constructor FileContent(filename)
	29
	30	The object then reads filename if it exists, otherwise the data is empty string.
	31	User may force to reread file by calling the readline() method of the class.
	32
	33	Filename is stored in local variable for future file operations.
	34	'''
	35
	36	self.filename=filename
	37	self.readfile()
	38
	39	def readfile(self):
	40	'''Force reread of the file and setting the data'''
	41	self.data=""
	42	try:
	43	with open (self.filename, "r") as myfile:
	44	self.data=myfile.read().replace('\n', '') #read the file and remove newline from the text
	45	except:
	46	pass # does nothing if error occurs
	47
	48
	49	def writefile(self, data, mode='w'):
	50	'''File may be updated by completely rewritting the file contents or appending the data to the end of the file.
	51	this is achieved by calling writefile(data, mode) method, where data is the string data to be written and
	52	mode can be 'a' for append and 'w' for writting the file anew.
	53	'''
	54	with open (self.filename, mode) as myfile:
	55	myfile.write(data)
	56
	57	def getText(self):
	58	'''
	59	Method getText() or calling the object itself returns string of data
	60	'''
	61	return self.data
	62
	63	def __str__(self):
	64	'''
	65	Method getText() or calling the object itself returns string of data
	66	'''
	67	return self.getText()
	68
	69	class Tape:
	70	'''
	71	Special class that manages configuration of trisurf (commonly named tape). It can read and parse configuration from disk or parse it from string.
	72	'''
	73
	74	def __init__(self):
	75	'''The object is instatiated by calling Tape() constructor without parameters'''
	76	return
	77
	78	def readTape(self, tape='tape'):
	79	'''
	80	Tape is read and parsed by calling the readTape() method with optional tape parameter, which is full path to filename where the configuration is stored.
	81	If the tape cannot be read, it prints error and exits.
	82	'''
	83	try:
	84	self.config=configobj.ConfigObj(tape)
	85	with open (tape, "r") as myfile:
	86	self.rawText=myfile.read() #read the file
	87
	88	except:
	89	print("Error reading or parsing tape file!\n")
	90	exit(1)
	91
	92	def setTape(self, string):
	93	'''Method setTape(string) parses the string in memory that hold the tape contents.'''
	94	self.config=configobj.ConfigObj(io.StringIO(string))
	95	self.rawText=string
	96	return
	97
	98	def getValue(self,key):
	99	'''Method getValue(key) returns value of a single parsed setting named "key".'''
	100
	101	return self.config[key]
	102
	103	def __str__(self):
	104	'''Calling the object itself, it recreates the tape contents from parsed values in form of key=value.'''
	105	retval=""
	106	for key,val in self.config.iteritems():
	107	retval=retval+str(key)+" = "+str(val)+"\n"
	108	return retval
	109
	110
	111
	112	class Directory:
	113	'''
	114	Class deals with the paths where the simulation is run and data is stored.
	115	'''
	116	def __init__(self, maindir=".", simdir=""):
	117	'''Initialization Directory() takes two optional parameters, namely maindir and simdir. Defaults to current directory. It sets local variables maindir and simdir accordingly.'''
	118	self.maindir=maindir
	119	self.simdir=simdir
	120	return
	121
	122	def fullpath(self):
	123	'''
	124	Method returns string of path where the data is stored. It combines values of maindir and simdir as maindir/simdir on Unix.
	125	'''
	126	return os.path.join(self.maindir,self.simdir)
	127
	128	def exists(self):
	129	''' Method checks whether the directory specified by fullpath() exists. It return True/False on completion.'''
	130	path=self.fullpath()
	131	if(os.path.exists(path)):
	132	return True
	133	else:
	134	return False
	135
	136	def make(self):
	137	''' Method make() creates directory. If it fails it exits the program with error message.'''
	138	try:
	139	os.makedirs(self.fullpath())
	140	except:
	141	print("Cannot make directory "+self.fullpath()+"\n")
	142	exit(1)
	143	return
	144
	145	def makeifnotexist(self):
	146	'''Method makeifnotexist() creates directory if it does not exist.'''
	147	if(self.exists()==0):
	148	self.make()
	149	return True
	150	else:
	151	return False
	152
	153	def remove(self):
	154	'''Method remove() removes directory recursively. WARNING! No questions asked.'''
	155	if(self.exists()):
	156	try:
	157	os.rmdir(self.fullpath())
	158	except:
	159	print("Cannot remove directory "+self.fullpath()+ "\n")
	160	exit(1)
	161	return
	162
	163	def goto(self):
	164	'''
	165	Method goto() moves current directory to the one specified by fullpath(). WARNING: when using the relative paths, do not call this function multiple times.
	166	'''
	167	try:
	168	os.chdir(self.fullpath())
	169	except:
	170	print("Cannot go to directory "+self.fullpath()+"\n")
	171	return
	172
	173
	174	class Statistics:
	175	'''
	176	Class that deals with the statistics file from the simulations.
	177	File is generally large and not all data is needed, so it is dealt with in a specific way.
	178	'''
	179
	180	def __init__(self,path,filename="statistics.csv"):
	181	'''
	182	At the initialization call it receives optional filename parameter specifying the path and filename of the statistics file.
	183
	184	The local variables path, filename, fullname (joined path and filename) and private check if the file exists are stored.
	185	'''
	186	self.path=path
	187	self.filename=filename
	188	self.fullname=os.path.join(path,filename)
	189	return
	190
	191	def exists(self):
	192	'''Method check if the statistics file exists.'''
	193	if(os.path.isfile(self.fullname)):
	194	return True
	195	else:
	196	return False
	197
5795ac	198	def lineCount(self):
8ab985	199	'''
SP	200	Internal method for determining the number of the lines in the most efficient way. Is it really the most efficient?
	201	'''
	202	f = open(self.fullname, "r+")
	203	try:
	204	buf = mmap.mmap(f.fileno(), 0)
	205	lines = 0
	206	readline = buf.readline
	207	while readline():
	208	lines += 1
	209	f.close()
	210	except:
	211	lines=0
	212	f.close()
	213	return lines
	214
5795ac	215	def tail(self):
SP	216	with open(self.fullname,'r') as myfile:
8ab985	217	lines=myfile.readlines()
SP	218	return [lines[len(lines)-2].replace('\n',''),lines[len(lines)-1].replace('\n','')]
	219
5795ac	220	def checkFileValidity(self):
8ab985	221	try:
5795ac	222	lines=self.tail()
8ab985	223	except:
SP	224	return(False)
	225	if len(lines)<2:
	226	return(False)
5795ac	227	return(True)
SP	228
	229	def getSimulationDeltaTime(self):
	230	try:
	231	lines=self.tail()
	232	except:
	233	return 0
	234
8ab985	235	fields=shlex.split(lines[0])
SP	236	epoch1=fields[0]
	237	n1=fields[1]
	238
	239	fields=shlex.split(lines[1])
	240	epoch2=fields[0]
	241	n2=fields[1]
	242	try:
5795ac	243	dT=int(epoch2)-int(epoch1)
8ab985	244	except:
5795ac	245	return 0
SP	246	return dT
	247
	248	def getLastIterationInStatistics(self):
	249	try:
	250	lines=self.tail()
	251	except:
	252	return 0
	253
	254	fields=shlex.split(lines[0])
	255	epoch1=fields[0]
	256	n1=fields[1]
	257
	258	fields=shlex.split(lines[1])
	259	epoch2=fields[0]
	260	return (fields[1])
	261
8ab985	262
SP	263	def readText(self):
	264	with open(self.fullname, 'r+') as fin:
	265	cont=fin.read()
	266	return cont
	267
	268	def __str__(self):
	269	'''
	270	Prints the full path with filename of the statistics.csv file
	271	'''
	272	return(str(self.fullname))
	273
	274
	275
	276	class Runner:
	277	'''
	278	Class Runner consists of a single running or terminated instance of the trisurf. It manages starting, stopping, verifying the running process and printing the reports of the configured instances.
	279	'''
	280
	281	@property
	282	def Dir(self):
	283	return Directory(maindir=self.maindir,simdir=self.subdir)
	284
	285
	286	@property
	287	def Statistics(self):
	288	return Statistics(self.Dir.fullpath(), "statistics.csv")
	289
	290	def __init__(self, subdir='run0', tape=None, snapshot=None, runArgs=[]):
	291	self.subdir=subdir
	292	self.runArgs=runArgs
	293	self.isFromSnapshot=False
	294	if(tape!=None):
	295	self.initFromTape(tape)
	296	if(snapshot!=None):
	297	self.initFromSnapshot(snapshot)
	298	return
	299
	300
	301	def initFromTape(self, tape):
	302	self.Tape=Tape()
	303	self.Tape.readTape(tape)
	304	self.tapeFilename=tape
	305
	306	def initFromSnapshot(self, snapshotfile):
	307	try:
	308	tree = ET.parse(snapshotfile)
	309	except:
	310	print("Error reading snapshot file")
	311	exit(1)
	312	self.isFromSnapshot=True
	313	self.snapshotFile=snapshotfile
	314	root = tree.getroot()
	315	tapetxt=root.find('tape')
	316	version=root.find('trisurfversion')
	317	self.Tape=Tape()
	318	self.Tape.setTape(tapetxt.text)
	319
	320	def getPID(self):
	321	try:
	322	fp = open(os.path.join(self.Dir.fullpath(),'.lock'))
	323	except IOError as e:
	324	return 0 #file probably does not exist. e==2??
	325	pid=fp.readline()
	326	fp.close()
	327	return int(pid)
	328
	329	def getLastIteration(self):
	330	try:
	331	fp = open(os.path.join(self.Dir.fullpath(),'.status'))
	332	except IOError as e:
	333	return -1 #file probably does not exist. e==2??
	334	status=fp.readline()
	335	fp.close()
	336	return int(status)
	337
	338	def isCompleted(self):
	339	if int(self.Tape.getValue("iterations"))+int(self.Tape.getValue("inititer"))==self.getLastIteration()+1:
	340	return True
	341	else:
	342	return False
5795ac	343
SP	344
	345	def getStartTime(self):
	346	try:
	347	return os.path.getmtime(os.path.join(self.Dir.fullpath(),'.lock'))
	348	except:
	349	return -1
8ab985	350
SP	351	def getStatus(self):
	352	pid=self.getPID()
	353	if(self.isCompleted()):
	354	return TS_COMPLETED
	355	if(pid==0):
	356	return TS_NOLOCK
	357	if(psutil.pid_exists(int(pid))):
	358	proc= psutil.Process(int(pid))
	359	#psutil.__version__ == '3.4.2' requires name() and status(), some older versions reguire name, status
	360	if(psutil.__version__>='2.0.0'):
	361	procname=proc.name()
	362	procstat=proc.status()
	363	else:
	364	procname=proc.name
	365	procstat=proc.status
	366	if procname=="trisurf":
	367	if procstat=="stopped":
	368	return TS_STOPPED
	369	else:
	370	return TS_RUNNING
	371	else:
	372	return TS_NONEXISTANT
	373	else:
	374	return TS_NONEXISTANT
	375
	376	def start(self):
	377	if(self.getStatus()==0 or self.getStatus()==TS_COMPLETED):
	378	#check if executable exists
	379	if(shutil.which('trisurf')==None):
	380	print("Error. Trisurf executable not found in PATH. Please install trisurf prior to running trisurf manager.")
	381	exit(1)
	382	#Symlinks tape file to the directory or create tape file from snapshot in the direcory...
	383	if(self.Dir.makeifnotexist()):
	384	if(self.isFromSnapshot==False):
	385	try:
	386	os.symlink(os.path.abspath(self.tapeFilename), self.Dir.fullpath()+"/tape")
	387	except:
	388	print("Error while symlinking "+os.path.abspath(self.tapeFilename)+" to "+self.Dir.fullpath()+"/tape")
	389	exit(1)
	390	else:
	391	try:
	392	with open (os.path.join(self.Dir.fullpath(),"tape"), "w") as myfile:
	393	#myfile.write("#This is automatically generated tape file from snapshot")
	394	myfile.write(str(self.Tape.rawText))
	395	except:
	396	print("Error -- cannot make tapefile "+ os.path.join(self.Dir.fullpath(),"tape")+" from the snapshot in the running directory")
	397	exit(1)
	398	try:
	399	os.symlink(os.path.abspath(self.snapshotFile), os.path.join(self.Dir.fullpath(),"initial_snapshot.vtu"))
	400	except:
	401	print("Error while symlinking "+os.path.abspath(self.snapshotFile)+" to "+os.path.join(self.Dir.fullpath(),self.snapshotFile))
	402
	403	#check if the simulation has been completed. in this case notify user and stop executing.
	404	if(self.isCompleted() and ("--force-from-tape" not in self.runArgs) and ("--reset-iteration-count" not in self.runArgs)):
	405	print("The simulation was completed. Not starting executable in "+self.Dir.fullpath())
	406	return
	407
	408	cwd=Directory(maindir=os.getcwd())
	409	lastVTU=self.getLastVTU() #we get last VTU file in case we need to continue the simulation from last snapshot. Need to be done before the Dir.goto() call.
	410	self.Dir.goto()
	411	print("Starting trisurf-ng executable in "+self.Dir.fullpath())
	412	if(self.isFromSnapshot==True):
	413	#here we try to determine whether we should continue the simulation or start from last known VTU snapshot.
	414	if(lastVTU==None):
	415	initSnap="initial_snapshot.vtu"
	416	else:
	417	initSnap=lastVTU
	418	print("WARNING: Not using initial snapshot as starting point, but selecting "+initSnap+" as a starting vesicle")
	419	params=["trisurf", "--restore-from-vtk",initSnap]+self.runArgs
	420	print("InitSnap is: "+initSnap)
	421	else:
	422	#veify if dump exists. If not it is a first run and shoud be run with --force-from-tape
	423	if(os.path.isfile("dump.bin")==False):
	424	self.runArgs.append("--force-from-tape")
	425	params=["trisurf"]+self.runArgs
	426	subprocess.Popen (params, stdout=subprocess.DEVNULL)
	427	cwd.goto()
	428	else:
	429	print("Process in "+self.Dir.fullpath()+" already running. Not starting.")
	430	return
	431
	432
	433	def setMaindir(self,prefix,variables):
	434	maindir=""
	435	for p,v in zip(prefix,variables):
	436	if(v=="xk0"):
	437	tv=str(round(float(self.Tape.config[v])))
	438	if sys.version_info<(3,0):
	439	tv=str(int(float(self.Tape.config[v])))
	440	else:
	441	tv=self.Tape.config[v]
	442	maindir=maindir+p+tv
	443	self.maindir=maindir
	444	return
	445
	446	def setSubdir(self, subdir="run0"):
	447	self.subdir=subdir
	448	return
	449
	450	def getStatistics(self, statfile="statistics.csv"):
	451	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
	452	pid=self.getPID()
	453	status=self.getStatus()
5795ac	454	if(self.Statistics.checkFileValidity()):
SP	455	ETA=str(datetime.timedelta(microseconds=(int(self.Tape.config['iterations'])-int(self.Statistics.getLastIterationInStatistics()))self.Statistics.getSimulationDeltaTime())1000000)
8ab985	456	if(status==TS_NONEXISTANT or status==TS_NOLOCK):
SP	457	statustxt="Not running"
	458	pid=""
	459	ETA=""
	460	elif status==TS_STOPPED:
	461	statustxt="Stopped"
	462	ETA="N/A"
	463	elif status==TS_COMPLETED:
	464	statustxt="Completed"
	465	pid=""
	466	ETA=""
	467	else:
	468	statustxt="Running"
	469
5795ac	470	if(self.Statistics.checkFileValidity()):
SP	471	report=[time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(self.getStartTime()))),ETA, statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
8ab985	472	else:
SP	473	report=["N/A","N/A",statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
	474	return report
	475
	476
	477	def stop(self):
	478	try:
	479	p=psutil.Process(self.getPID())
	480	p.kill()
	481	except:
	482	print("Could not stop the process. Is the process running? Do you have sufficient privileges?")
	483
	484
	485	def writeComment(self, data, mode='w'):
	486	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
	487	self.Comment.writefile(data,mode=mode)
	488
	489
	490	def getLastVTU(self):
	491	vtuidx=self.getLastIteration()-int(self.Tape.getValue("inititer"))
	492	if vtuidx<0:
	493	return None
	494	else:
	495	return 'timestep_{:06d}.vtu'.format(vtuidx)
	496
	497	def __str__(self):
	498	if(self.getStatus()==0):
	499	str=" not running."
	500	else:
	501	str=" running."
	502	return(self.Dir.fullpath()+str)
	503
57b1a6	504	def __repr__(self):
SP	505	return("Instance of trisurf in "+self.Dir.fullpath())