trisurf-manager.git

Samo Penic

2017-01-05 57b1a6226483e8148c14a1a2a445a7b7291dac08

commit \| author \| age
8ab985	1	import configobj
SP	2	import xml.etree.ElementTree as ET
	3	import base64
	4	import zlib
	5	import sys,io
	6	import os
	7	from itertools import islice
	8	import mmap
	9	import shlex
	10	import psutil
	11	import time
	12	import datetime
	13	import subprocess
	14	import shutil
	15
	16	# Process status
	17	TS_NOLOCK=0 # lock file does not exist
	18	TS_NONEXISTANT=0 # process is not in the list of processes
	19	TS_STOPPED=1 # the process is listed, but is in stopped state
	20	TS_RUNNING=2 # process is running
	21	TS_COMPLETED=3 #simulation is completed
	22
	23	class FileContent:
	24	'''
	25	Class is helpful for reading and writting the specific files.
	26	'''
	27	def __init__(self,filename):
	28	''' The instance is done by calling constructor FileContent(filename)
	29
	30	The object then reads filename if it exists, otherwise the data is empty string.
	31	User may force to reread file by calling the readline() method of the class.
	32
	33	Filename is stored in local variable for future file operations.
	34	'''
	35
	36	self.filename=filename
	37	self.readfile()
	38
	39	def readfile(self):
	40	'''Force reread of the file and setting the data'''
	41	self.data=""
	42	try:
	43	with open (self.filename, "r") as myfile:
	44	self.data=myfile.read().replace('\n', '') #read the file and remove newline from the text
	45	except:
	46	pass # does nothing if error occurs
	47
	48
	49	def writefile(self, data, mode='w'):
	50	'''File may be updated by completely rewritting the file contents or appending the data to the end of the file.
	51	this is achieved by calling writefile(data, mode) method, where data is the string data to be written and
	52	mode can be 'a' for append and 'w' for writting the file anew.
	53	'''
	54	with open (self.filename, mode) as myfile:
	55	myfile.write(data)
	56
	57	def getText(self):
	58	'''
	59	Method getText() or calling the object itself returns string of data
	60	'''
	61	return self.data
	62
	63	def __str__(self):
	64	'''
	65	Method getText() or calling the object itself returns string of data
	66	'''
	67	return self.getText()
	68
	69	class Tape:
	70	'''
	71	Special class that manages configuration of trisurf (commonly named tape). It can read and parse configuration from disk or parse it from string.
	72	'''
	73
	74	def __init__(self):
	75	'''The object is instatiated by calling Tape() constructor without parameters'''
	76	return
	77
	78	def readTape(self, tape='tape'):
	79	'''
	80	Tape is read and parsed by calling the readTape() method with optional tape parameter, which is full path to filename where the configuration is stored.
	81	If the tape cannot be read, it prints error and exits.
	82	'''
	83	try:
	84	self.config=configobj.ConfigObj(tape)
	85	with open (tape, "r") as myfile:
	86	self.rawText=myfile.read() #read the file
	87
	88	except:
	89	print("Error reading or parsing tape file!\n")
	90	exit(1)
	91
	92	def setTape(self, string):
	93	'''Method setTape(string) parses the string in memory that hold the tape contents.'''
	94	self.config=configobj.ConfigObj(io.StringIO(string))
	95	self.rawText=string
	96	return
	97
	98	def getValue(self,key):
	99	'''Method getValue(key) returns value of a single parsed setting named "key".'''
	100
	101	return self.config[key]
	102
	103	def __str__(self):
	104	'''Calling the object itself, it recreates the tape contents from parsed values in form of key=value.'''
	105	retval=""
	106	for key,val in self.config.iteritems():
	107	retval=retval+str(key)+" = "+str(val)+"\n"
	108	return retval
	109
	110
	111
	112	class Directory:
	113	'''
	114	Class deals with the paths where the simulation is run and data is stored.
	115	'''
	116	def __init__(self, maindir=".", simdir=""):
	117	'''Initialization Directory() takes two optional parameters, namely maindir and simdir. Defaults to current directory. It sets local variables maindir and simdir accordingly.'''
	118	self.maindir=maindir
	119	self.simdir=simdir
	120	return
	121
	122	def fullpath(self):
	123	'''
	124	Method returns string of path where the data is stored. It combines values of maindir and simdir as maindir/simdir on Unix.
	125	'''
	126	return os.path.join(self.maindir,self.simdir)
	127
	128	def exists(self):
	129	''' Method checks whether the directory specified by fullpath() exists. It return True/False on completion.'''
	130	path=self.fullpath()
	131	if(os.path.exists(path)):
	132	return True
	133	else:
	134	return False
	135
	136	def make(self):
	137	''' Method make() creates directory. If it fails it exits the program with error message.'''
	138	try:
	139	os.makedirs(self.fullpath())
	140	except:
	141	print("Cannot make directory "+self.fullpath()+"\n")
	142	exit(1)
	143	return
	144
	145	def makeifnotexist(self):
	146	'''Method makeifnotexist() creates directory if it does not exist.'''
	147	if(self.exists()==0):
	148	self.make()
	149	return True
	150	else:
	151	return False
	152
	153	def remove(self):
	154	'''Method remove() removes directory recursively. WARNING! No questions asked.'''
	155	if(self.exists()):
	156	try:
	157	os.rmdir(self.fullpath())
	158	except:
	159	print("Cannot remove directory "+self.fullpath()+ "\n")
	160	exit(1)
	161	return
	162
	163	def goto(self):
	164	'''
	165	Method goto() moves current directory to the one specified by fullpath(). WARNING: when using the relative paths, do not call this function multiple times.
	166	'''
	167	try:
	168	os.chdir(self.fullpath())
	169	except:
	170	print("Cannot go to directory "+self.fullpath()+"\n")
	171	return
	172
	173
	174	class Statistics:
	175	'''
	176	Class that deals with the statistics file from the simulations.
	177	File is generally large and not all data is needed, so it is dealt with in a specific way.
	178	'''
	179
	180	def __init__(self,path,filename="statistics.csv"):
	181	'''
	182	At the initialization call it receives optional filename parameter specifying the path and filename of the statistics file.
	183
	184	The local variables path, filename, fullname (joined path and filename) and private check if the file exists are stored.
	185	'''
	186	self.path=path
	187	self.filename=filename
	188	self.fullname=os.path.join(path,filename)
	189	self.fileOK=self.read()
	190	return
	191
	192	def exists(self):
	193	'''Method check if the statistics file exists.'''
	194	if(os.path.isfile(self.fullname)):
	195	return True
	196	else:
	197	return False
	198
	199	def mapcount(self):
	200	'''
	201	Internal method for determining the number of the lines in the most efficient way. Is it really the most efficient?
	202	'''
	203	f = open(self.fullname, "r+")
	204	try:
	205	buf = mmap.mmap(f.fileno(), 0)
	206	lines = 0
	207	readline = buf.readline
	208	while readline():
	209	lines += 1
	210	f.close()
	211	except:
	212	lines=0
	213	f.close()
	214	return lines
	215
	216	def tail(self,filename,n=2):
	217	with open(filename,'r') as myfile:
	218	lines=myfile.readlines()
	219	return [lines[len(lines)-2].replace('\n',''),lines[len(lines)-1].replace('\n','')]
	220
	221	def read(self):
	222	try:
	223	lines=self.tail(self.fullname)
	224	except:
	225	return(False)
	226	if len(lines)<2:
	227	return(False)
	228	#print (line)
	229	fields=shlex.split(lines[0])
	230	epoch1=fields[0]
	231	n1=fields[1]
	232
	233	fields=shlex.split(lines[1])
	234	epoch2=fields[0]
	235	n2=fields[1]
	236	try:
	237	self.dT=int(epoch2)-int(epoch1)
	238	self.last=n2
	239	#print(epoch1)
	240	#print(epoch2)
	241	#print(self.dT)
	242	#print(self.last)
	243	self.startDate=os.path.getmtime(os.path.join(self.path,'.lock'))
	244	except:
	245	return(False)
	246	return(True)
	247
	248	def readText(self):
	249	with open(self.fullname, 'r+') as fin:
	250	cont=fin.read()
	251	return cont
	252
	253	def read_old(self):
	254	'''
	255	Method read() reads the statistics if it exists. It sets local variable dT storing the time differential between two intervals of simulation (outer loops). It also stores last simulation loop and the start of the run.
	256	'''
	257	if(self.exists()):
	258	# epoch1=0
	259	# epoch2=0
	260	# n1=0
	261	# n2=0
	262	nlines=self.mapcount()
	263	if nlines<2:
	264	return(False)
	265	try:
	266	with open(self.fullname, "r+") as fin:
	267	i=0;
	268	for line in fin:
	269	if(i==1):
	270	#print (line)
	271	fields=shlex.split(line)
	272	epoch1=fields[0]
	273	n1=fields[1]
	274	if(i==nlines-1):
	275	fields=shlex.split(line)
	276	epoch2=fields[0]
	277	n2=fields[1]
	278	i=i+1
	279	except:
	280	#print("Cannot read statistics file in "+self.fullname+"\n")
	281	return(False)
	282	else:
	283	#print("File "+self.fullname+" does not exists.\n")
	284	return(False)
	285	try:
	286	self.dT=(int(epoch2)-int(epoch1))/(int(n2)-int(n1))
	287	except:
	288	self.dT=0
	289	self.last=n2
	290	self.startDate=epoch1
	291	return(True)
	292
	293	def __str__(self):
	294	'''
	295	Prints the full path with filename of the statistics.csv file
	296	'''
	297	return(str(self.fullname))
	298
	299
	300
	301	class Runner:
	302	'''
	303	Class Runner consists of a single running or terminated instance of the trisurf. It manages starting, stopping, verifying the running process and printing the reports of the configured instances.
	304	'''
	305
	306	@property
	307	def Dir(self):
	308	return Directory(maindir=self.maindir,simdir=self.subdir)
	309
	310
	311	@property
	312	def Statistics(self):
	313	return Statistics(self.Dir.fullpath(), "statistics.csv")
	314
	315	def __init__(self, subdir='run0', tape=None, snapshot=None, runArgs=[]):
	316	self.subdir=subdir
	317	self.runArgs=runArgs
	318	self.isFromSnapshot=False
	319	if(tape!=None):
	320	self.initFromTape(tape)
	321	if(snapshot!=None):
	322	self.initFromSnapshot(snapshot)
	323	return
	324
	325
	326	def initFromTape(self, tape):
	327	self.Tape=Tape()
	328	self.Tape.readTape(tape)
	329	self.tapeFilename=tape
	330
	331	def initFromSnapshot(self, snapshotfile):
	332	try:
	333	tree = ET.parse(snapshotfile)
	334	except:
	335	print("Error reading snapshot file")
	336	exit(1)
	337	self.isFromSnapshot=True
	338	self.snapshotFile=snapshotfile
	339	root = tree.getroot()
	340	tapetxt=root.find('tape')
	341	version=root.find('trisurfversion')
	342	self.Tape=Tape()
	343	self.Tape.setTape(tapetxt.text)
	344
	345	def getPID(self):
	346	try:
	347	fp = open(os.path.join(self.Dir.fullpath(),'.lock'))
	348	except IOError as e:
	349	return 0 #file probably does not exist. e==2??
	350	pid=fp.readline()
	351	fp.close()
	352	return int(pid)
	353
	354	def getLastIteration(self):
	355	try:
	356	fp = open(os.path.join(self.Dir.fullpath(),'.status'))
	357	except IOError as e:
	358	return -1 #file probably does not exist. e==2??
	359	status=fp.readline()
	360	fp.close()
	361	return int(status)
	362
	363	def isCompleted(self):
	364	if int(self.Tape.getValue("iterations"))+int(self.Tape.getValue("inititer"))==self.getLastIteration()+1:
	365	return True
	366	else:
	367	return False
	368
	369	def getStatus(self):
	370	pid=self.getPID()
	371	if(self.isCompleted()):
	372	return TS_COMPLETED
	373	if(pid==0):
	374	return TS_NOLOCK
	375	if(psutil.pid_exists(int(pid))):
	376	proc= psutil.Process(int(pid))
	377	#psutil.__version__ == '3.4.2' requires name() and status(), some older versions reguire name, status
	378	if(psutil.__version__>='2.0.0'):
	379	procname=proc.name()
	380	procstat=proc.status()
	381	else:
	382	procname=proc.name
	383	procstat=proc.status
	384	if procname=="trisurf":
	385	if procstat=="stopped":
	386	return TS_STOPPED
	387	else:
	388	return TS_RUNNING
	389	else:
	390	return TS_NONEXISTANT
	391	else:
	392	return TS_NONEXISTANT
	393
	394	def start(self):
	395	if(self.getStatus()==0 or self.getStatus()==TS_COMPLETED):
	396	#check if executable exists
	397	if(shutil.which('trisurf')==None):
	398	print("Error. Trisurf executable not found in PATH. Please install trisurf prior to running trisurf manager.")
	399	exit(1)
	400	#Symlinks tape file to the directory or create tape file from snapshot in the direcory...
	401	if(self.Dir.makeifnotexist()):
	402	if(self.isFromSnapshot==False):
	403	try:
	404	os.symlink(os.path.abspath(self.tapeFilename), self.Dir.fullpath()+"/tape")
	405	except:
	406	print("Error while symlinking "+os.path.abspath(self.tapeFilename)+" to "+self.Dir.fullpath()+"/tape")
	407	exit(1)
	408	else:
	409	try:
	410	with open (os.path.join(self.Dir.fullpath(),"tape"), "w") as myfile:
	411	#myfile.write("#This is automatically generated tape file from snapshot")
	412	myfile.write(str(self.Tape.rawText))
	413	except:
	414	print("Error -- cannot make tapefile "+ os.path.join(self.Dir.fullpath(),"tape")+" from the snapshot in the running directory")
	415	exit(1)
	416	try:
	417	os.symlink(os.path.abspath(self.snapshotFile), os.path.join(self.Dir.fullpath(),"initial_snapshot.vtu"))
	418	except:
	419	print("Error while symlinking "+os.path.abspath(self.snapshotFile)+" to "+os.path.join(self.Dir.fullpath(),self.snapshotFile))
	420
	421	#check if the simulation has been completed. in this case notify user and stop executing.
	422	if(self.isCompleted() and ("--force-from-tape" not in self.runArgs) and ("--reset-iteration-count" not in self.runArgs)):
	423	print("The simulation was completed. Not starting executable in "+self.Dir.fullpath())
	424	return
	425
	426	cwd=Directory(maindir=os.getcwd())
	427	lastVTU=self.getLastVTU() #we get last VTU file in case we need to continue the simulation from last snapshot. Need to be done before the Dir.goto() call.
	428	self.Dir.goto()
	429	print("Starting trisurf-ng executable in "+self.Dir.fullpath())
	430	if(self.isFromSnapshot==True):
	431	#here we try to determine whether we should continue the simulation or start from last known VTU snapshot.
	432	if(lastVTU==None):
	433	initSnap="initial_snapshot.vtu"
	434	else:
	435	initSnap=lastVTU
	436	print("WARNING: Not using initial snapshot as starting point, but selecting "+initSnap+" as a starting vesicle")
	437	params=["trisurf", "--restore-from-vtk",initSnap]+self.runArgs
	438	print("InitSnap is: "+initSnap)
	439	else:
	440	#veify if dump exists. If not it is a first run and shoud be run with --force-from-tape
	441	if(os.path.isfile("dump.bin")==False):
	442	self.runArgs.append("--force-from-tape")
	443	params=["trisurf"]+self.runArgs
	444	subprocess.Popen (params, stdout=subprocess.DEVNULL)
	445	cwd.goto()
	446	else:
	447	print("Process in "+self.Dir.fullpath()+" already running. Not starting.")
	448	return
	449
	450
	451	def setMaindir(self,prefix,variables):
	452	maindir=""
	453	for p,v in zip(prefix,variables):
	454	if(v=="xk0"):
	455	tv=str(round(float(self.Tape.config[v])))
	456	if sys.version_info<(3,0):
	457	tv=str(int(float(self.Tape.config[v])))
	458	else:
	459	tv=self.Tape.config[v]
	460	maindir=maindir+p+tv
	461	self.maindir=maindir
	462	return
	463
	464	def setSubdir(self, subdir="run0"):
	465	self.subdir=subdir
	466	return
	467
	468	def getStatistics(self, statfile="statistics.csv"):
	469	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
	470	pid=self.getPID()
	471	status=self.getStatus()
	472	if(self.Statistics.fileOK):
	473	ETA=str(datetime.timedelta(microseconds=(int(self.Tape.config['iterations'])-int(self.Statistics.last))self.Statistics.dT)1000000)
	474	if(status==TS_NONEXISTANT or status==TS_NOLOCK):
	475	statustxt="Not running"
	476	pid=""
	477	ETA=""
	478	elif status==TS_STOPPED:
	479	statustxt="Stopped"
	480	ETA="N/A"
	481	elif status==TS_COMPLETED:
	482	statustxt="Completed"
	483	pid=""
	484	ETA=""
	485	else:
	486	statustxt="Running"
	487
	488	if(self.Statistics.fileOK):
	489	report=[time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(self.Statistics.startDate))),ETA, statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
	490	else:
	491	report=["N/A","N/A",statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
	492	return report
	493
	494
	495	def stop(self):
	496	try:
	497	p=psutil.Process(self.getPID())
	498	p.kill()
	499	except:
	500	print("Could not stop the process. Is the process running? Do you have sufficient privileges?")
	501
	502
	503	def writeComment(self, data, mode='w'):
	504	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
	505	self.Comment.writefile(data,mode=mode)
	506
	507
	508	def getLastVTU(self):
	509	vtuidx=self.getLastIteration()-int(self.Tape.getValue("inititer"))
	510	if vtuidx<0:
	511	return None
	512	else:
	513	return 'timestep_{:06d}.vtu'.format(vtuidx)
	514
	515	def __str__(self):
	516	if(self.getStatus()==0):
	517	str=" not running."
	518	else:
	519	str=" running."
	520	return(self.Dir.fullpath()+str)
	521
57b1a6	522	def __repr__(self):
SP	523	return("Instance of trisurf in "+self.Dir.fullpath())