trisurf-ng.git - Gitblit

Samo Penic

2017-01-04 96b0a0ac783d35fbf4e764fe8e610ddc2a79af27

commit \| author \| age
bc14fb	1	import configobj
bd6993	2	import xml.etree.ElementTree as ET
SP	3	import base64
	4	import zlib
56016c	5	import sys,io
c14bd6	6	import os
SP	7	from itertools import islice
	8	import mmap
7c45b1	9	import shlex
34600b	10	import psutil
f81585	11	import time
SP	12	import datetime
fca8d6	13	import subprocess
38cb4a	14	import shutil
bc14fb	15
055489	16	# Process status
SP	17	TS_NOLOCK=0 # lock file does not exist
	18	TS_NONEXISTANT=0 # process is not in the list of processes
	19	TS_STOPPED=1 # the process is listed, but is in stopped state
	20	TS_RUNNING=2 # process is running
3d0247	21	TS_COMPLETED=3 #simulation is completed
bc14fb	22
9154b3	23	class FileContent:
055489	24	'''
SP	25	Class is helpful for reading and writting the specific files.
	26	'''
9154b3	27	def __init__(self,filename):
055489	28	''' The instance is done by calling constructor FileContent(filename)
SP	29
	30	The object then reads filename if it exists, otherwise the data is empty string.
	31	User may force to reread file by calling the readline() method of the class.
	32
	33	Filename is stored in local variable for future file operations.
	34	'''
	35
9154b3	36	self.filename=filename
SP	37	self.readfile()
bc14fb	38
9154b3	39	def readfile(self):
055489	40	'''Force reread of the file and setting the data'''
SP	41	self.data=""
9154b3	42	try:
SP	43	with open (self.filename, "r") as myfile:
055489	44	self.data=myfile.read().replace('\n', '') #read the file and remove newline from the text
9154b3	45	except:
055489	46	pass # does nothing if error occurs
9154b3	47
SP	48
	49	def writefile(self, data, mode='w'):
055489	50	'''File may be updated by completely rewritting the file contents or appending the data to the end of the file.
SP	51	this is achieved by calling writefile(data, mode) method, where data is the string data to be written and
	52	mode can be 'a' for append and 'w' for writting the file anew.
	53	'''
9154b3	54	with open (self.filename, mode) as myfile:
SP	55	myfile.write(data)
	56
	57	def getText(self):
055489	58	'''
SP	59	Method getText() or calling the object itself returns string of data
	60	'''
9154b3	61	return self.data
SP	62
	63	def __str__(self):
055489	64	'''
SP	65	Method getText() or calling the object itself returns string of data
	66	'''
9154b3	67	return self.getText()
bc14fb	68
SP	69	class Tape:
055489	70	'''
SP	71	Special class that manages configuration of trisurf (commonly named tape). It can read and parse configuration from disk or parse it from string.
	72	'''
bc14fb	73
SP	74	def __init__(self):
055489	75	'''The object is instatiated by calling Tape() constructor without parameters'''
bc14fb	76	return
SP	77
	78	def readTape(self, tape='tape'):
055489	79	'''
SP	80	Tape is read and parsed by calling the readTape() method with optional tape parameter, which is full path to filename where the configuration is stored.
	81	If the tape cannot be read, it prints error and exits.
	82	'''
bc14fb	83	try:
SP	84	self.config=configobj.ConfigObj(tape)
cc95bd	85	with open (tape, "r") as myfile:
SP	86	self.rawText=myfile.read() #read the file
	87
bc14fb	88	except:
SP	89	print("Error reading or parsing tape file!\n")
	90	exit(1)
	91
	92	def setTape(self, string):
055489	93	'''Method setTape(string) parses the string in memory that hold the tape contents.'''
bd6993	94	self.config=configobj.ConfigObj(io.StringIO(string))
cc95bd	95	self.rawText=string
bc14fb	96	return
SP	97
	98	def getValue(self,key):
055489	99	'''Method getValue(key) returns value of a single parsed setting named "key".'''
SP	100
bc14fb	101	return self.config[key]
SP	102
c14bd6	103	def __str__(self):
055489	104	'''Calling the object itself, it recreates the tape contents from parsed values in form of key=value.'''
c14bd6	105	retval=""
SP	106	for key,val in self.config.iteritems():
	107	retval=retval+str(key)+" = "+str(val)+"\n"
	108	return retval
bc14fb	109
c14bd6	110
SP	111
	112	class Directory:
055489	113	'''
SP	114	Class deals with the paths where the simulation is run and data is stored.
	115	'''
c14bd6	116	def __init__(self, maindir=".", simdir=""):
055489	117	'''Initialization Directory() takes two optional parameters, namely maindir and simdir. Defaults to current directory. It sets local variables maindir and simdir accordingly.'''
c14bd6	118	self.maindir=maindir
SP	119	self.simdir=simdir
	120	return
	121
	122	def fullpath(self):
055489	123	'''
SP	124	Method returns string of path where the data is stored. It combines values of maindir and simdir as maindir/simdir on Unix.
	125	'''
c14bd6	126	return os.path.join(self.maindir,self.simdir)
SP	127
	128	def exists(self):
055489	129	''' Method checks whether the directory specified by fullpath() exists. It return True/False on completion.'''
c14bd6	130	path=self.fullpath()
SP	131	if(os.path.exists(path)):
055489	132	return True
c14bd6	133	else:
055489	134	return False
c14bd6	135
SP	136	def make(self):
055489	137	''' Method make() creates directory. If it fails it exits the program with error message.'''
c14bd6	138	try:
SP	139	os.makedirs(self.fullpath())
	140	except:
	141	print("Cannot make directory "+self.fullpath()+"\n")
	142	exit(1)
	143	return
	144
	145	def makeifnotexist(self):
055489	146	'''Method makeifnotexist() creates directory if it does not exist.'''
c14bd6	147	if(self.exists()==0):
SP	148	self.make()
bd826d	149	return True
SP	150	else:
	151	return False
c14bd6	152
SP	153	def remove(self):
055489	154	'''Method remove() removes directory recursively. WARNING! No questions asked.'''
c14bd6	155	if(self.exists()):
SP	156	try:
	157	os.rmdir(self.fullpath())
	158	except:
	159	print("Cannot remove directory "+self.fullpath()+ "\n")
	160	exit(1)
	161	return
	162
	163	def goto(self):
055489	164	'''
SP	165	Method goto() moves current directory to the one specified by fullpath(). WARNING: when using the relative paths, do not call this function multiple times.
	166	'''
c14bd6	167	try:
SP	168	os.chdir(self.fullpath())
	169	except:
	170	print("Cannot go to directory "+self.fullpath()+"\n")
	171	return
	172
	173
	174	class Statistics:
055489	175	'''
SP	176	Class that deals with the statistics file from the simulations.
	177	File is generally large and not all data is needed, so it is dealt with in a specific way.
	178	'''
	179
c14bd6	180	def __init__(self,path,filename="statistics.csv"):
055489	181	'''
SP	182	At the initialization call it receives optional filename parameter specifying the path and filename of the statistics file.
	183
	184	The local variables path, filename, fullname (joined path and filename) and private check if the file exists are stored.
	185	'''
c14bd6	186	self.path=path
SP	187	self.filename=filename
	188	self.fullname=os.path.join(path,filename)
f81585	189	self.fileOK=self.read()
c14bd6	190	return
SP	191
	192	def exists(self):
055489	193	'''Method check if the statistics file exists.'''
c14bd6	194	if(os.path.isfile(self.fullname)):
SP	195	return True
	196	else:
	197	return False
	198
	199	def mapcount(self):
055489	200	'''
SP	201	Internal method for determining the number of the lines in the most efficient way. Is it really the most efficient?
	202	'''
c14bd6	203	f = open(self.fullname, "r+")
d665c0	204	try:
SP	205	buf = mmap.mmap(f.fileno(), 0)
	206	lines = 0
	207	readline = buf.readline
	208	while readline():
	209	lines += 1
	210	f.close()
	211	except:
	212	lines=0
	213	f.close()
c14bd6	214	return lines
6568b3	215
SP	216	def tail(self,filename,n=2):
	217	with open(filename,'r') as myfile:
	218	lines=myfile.readlines()
	219	return [lines[len(lines)-2].replace('\n',''),lines[len(lines)-1].replace('\n','')]
	220
c14bd6	221	def read(self):
6568b3	222	try:
SP	223	lines=self.tail(self.fullname)
	224	except:
	225	return(False)
	226	if len(lines)<2:
	227	return(False)
	228	#print (line)
	229	fields=shlex.split(lines[0])
	230	epoch1=fields[0]
	231	n1=fields[1]
	232
	233	fields=shlex.split(lines[1])
	234	epoch2=fields[0]
	235	n2=fields[1]
	236	try:
	237	self.dT=int(epoch2)-int(epoch1)
	238	self.last=n2
ddb86f	239	#print(epoch1)
SP	240	#print(epoch2)
	241	#print(self.dT)
	242	#print(self.last)
6568b3	243	self.startDate=os.path.getmtime(os.path.join(self.path,'.lock'))
SP	244	except:
	245	return(False)
	246	return(True)
	247
61ad95	248	def readText(self):
SP	249	with open(self.fullname, 'r+') as fin:
	250	cont=fin.read()
	251	return cont
	252
6568b3	253	def read_old(self):
055489	254	'''
SP	255	Method read() reads the statistics if it exists. It sets local variable dT storing the time differential between two intervals of simulation (outer loops). It also stores last simulation loop and the start of the run.
	256	'''
c14bd6	257	if(self.exists()):
3d0247	258	# epoch1=0
SP	259	# epoch2=0
	260	# n1=0
	261	# n2=0
c14bd6	262	nlines=self.mapcount()
3d0247	263	if nlines<2:
SP	264	return(False)
c14bd6	265	try:
SP	266	with open(self.fullname, "r+") as fin:
	267	i=0;
	268	for line in fin:
	269	if(i==1):
7c45b1	270	#print (line)
SP	271	fields=shlex.split(line)
	272	epoch1=fields[0]
	273	n1=fields[1]
c14bd6	274	if(i==nlines-1):
7c45b1	275	fields=shlex.split(line)
SP	276	epoch2=fields[0]
	277	n2=fields[1]
c14bd6	278	i=i+1
SP	279	except:
f81585	280	#print("Cannot read statistics file in "+self.fullname+"\n")
7c45b1	281	return(False)
c14bd6	282	else:
f81585	283	#print("File "+self.fullname+" does not exists.\n")
7c45b1	284	return(False)
3d0247	285	try:
SP	286	self.dT=(int(epoch2)-int(epoch1))/(int(n2)-int(n1))
	287	except:
	288	self.dT=0
f81585	289	self.last=n2
SP	290	self.startDate=epoch1
7c45b1	291	return(True)
f81585	292
SP	293	def __str__(self):
055489	294	'''
SP	295	Prints the full path with filename of the statistics.csv file
	296	'''
f81585	297	return(str(self.fullname))
SP	298
7c45b1	299
bc14fb	300
SP	301	class Runner:
	302	'''
055489	303	Class Runner consists of a single running or terminated instance of the trisurf. It manages starting, stopping, verifying the running process and printing the reports of the configured instances.
bc14fb	304	'''
61ad95	305
SP	306	@property
	307	def Dir(self):
	308	return Directory(maindir=self.maindir,simdir=self.subdir)
	309
	310
	311	@property
	312	def Statistics(self):
	313	return Statistics(self.Dir.fullpath(), "statistics.csv")
	314
bd826d	315	def __init__(self, subdir='run0', tape=None, snapshot=None, runArgs=[]):
c14bd6	316	self.subdir=subdir
b122c4	317	self.runArgs=runArgs
96b0a0	318	self.isFromSnapshot=False
bd826d	319	if(tape!=None):
c14bd6	320	self.initFromTape(tape)
bd826d	321	if(snapshot!=None):
c14bd6	322	self.initFromSnapshot(snapshot)
SP	323	return
	324
	325
a99f2b	326	def initFromTape(self, tape):
3fc0ef	327	self.Tape=Tape()
SP	328	self.Tape.readTape(tape)
	329	self.tapeFilename=tape
bc14fb	330
bd6993	331	def initFromSnapshot(self, snapshotfile):
SP	332	try:
	333	tree = ET.parse(snapshotfile)
	334	except:
	335	print("Error reading snapshot file")
	336	exit(1)
96b0a0	337	self.isFromSnapshot=True
47d80d	338	self.snapshotFile=snapshotfile
bd6993	339	root = tree.getroot()
SP	340	tapetxt=root.find('tape')
	341	version=root.find('trisurfversion')
3fc0ef	342	self.Tape=Tape()
SP	343	self.Tape.setTape(tapetxt.text)
2ded03	344
9154b3	345	def getPID(self):
34600b	346	try:
SP	347	fp = open(os.path.join(self.Dir.fullpath(),'.lock'))
	348	except IOError as e:
	349	return 0 #file probably does not exist. e==2??
9154b3	350	pid=fp.readline()
SP	351	fp.close()
055489	352	return int(pid)
9154b3	353
3d0247	354	def getLastIteration(self):
SP	355	try:
	356	fp = open(os.path.join(self.Dir.fullpath(),'.status'))
	357	except IOError as e:
	358	return -1 #file probably does not exist. e==2??
	359	status=fp.readline()
	360	fp.close()
	361	return int(status)
	362
	363	def isCompleted(self):
3fc0ef	364	if int(self.Tape.getValue("iterations"))+int(self.Tape.getValue("inititer"))==self.getLastIteration()+1:
3d0247	365	return True
SP	366	else:
	367	return False
	368
9154b3	369	def getStatus(self):
SP	370	pid=self.getPID()
3d0247	371	if(self.isCompleted()):
SP	372	return TS_COMPLETED
9154b3	373	if(pid==0):
055489	374	return TS_NOLOCK
34600b	375	if(psutil.pid_exists(int(pid))):
055489	376	proc= psutil.Process(int(pid))
56ba2f	377	#psutil.__version__ == '3.4.2' requires name() and status(), some older versions reguire name, status
9aa037	378	if(psutil.__version__>='2.0.0'):
SP	379	procname=proc.name()
	380	procstat=proc.status()
	381	else:
	382	procname=proc.name
	383	procstat=proc.status
	384	if procname=="trisurf":
	385	if procstat=="stopped":
055489	386	return TS_STOPPED
SP	387	else:
	388	return TS_RUNNING
9154b3	389	else:
055489	390	return TS_NONEXISTANT
34600b	391	else:
055489	392	return TS_NONEXISTANT
bc14fb	393
SP	394	def start(self):
3d0247	395	if(self.getStatus()==0 or self.getStatus()==TS_COMPLETED):
38cb4a	396	#check if executable exists
SP	397	if(shutil.which('trisurf')==None):
	398	print("Error. Trisurf executable not found in PATH. Please install trisurf prior to running trisurf manager.")
	399	exit(1)
bd826d	400	#Symlinks tape file to the directory or create tape file from snapshot in the direcory...
SP	401	if(self.Dir.makeifnotexist()):
96b0a0	402	if(self.isFromSnapshot==False):
bd826d	403	try:
3fc0ef	404	os.symlink(os.path.abspath(self.tapeFilename), self.Dir.fullpath()+"/tape")
bd826d	405	except:
3fc0ef	406	print("Error while symlinking "+os.path.abspath(self.tapeFilename)+" to "+self.Dir.fullpath()+"/tape")
bd826d	407	exit(1)
SP	408	else:
	409	try:
	410	with open (os.path.join(self.Dir.fullpath(),"tape"), "w") as myfile:
cc95bd	411	#myfile.write("#This is automatically generated tape file from snapshot")
3fc0ef	412	myfile.write(str(self.Tape.rawText))
bd826d	413	except:
SP	414	print("Error -- cannot make tapefile "+ os.path.join(self.Dir.fullpath(),"tape")+" from the snapshot in the running directory")
	415	exit(1)
	416	try:
354145	417	os.symlink(os.path.abspath(self.snapshotFile), os.path.join(self.Dir.fullpath(),"initial_snapshot.vtu"))
bd826d	418	except:
SP	419	print("Error while symlinking "+os.path.abspath(self.snapshotFile)+" to "+os.path.join(self.Dir.fullpath(),self.snapshotFile))
3d0247	420
SP	421	#check if the simulation has been completed. in this case notify user and stop executing.
	422	if(self.isCompleted() and ("--force-from-tape" not in self.runArgs) and ("--reset-iteration-count" not in self.runArgs)):
3b83db	423	print("The simulation was completed. Not starting executable in "+self.Dir.fullpath())
3d0247	424	return
SP	425
47d80d	426	cwd=Directory(maindir=os.getcwd())
2f74e4	427	lastVTU=self.getLastVTU() #we get last VTU file in case we need to continue the simulation from last snapshot. Need to be done before the Dir.goto() call.
47d80d	428	self.Dir.goto()
3b83db	429	print("Starting trisurf-ng executable in "+self.Dir.fullpath())
96b0a0	430	if(self.isFromSnapshot==True):
2f74e4	431	#here we try to determine whether we should continue the simulation or start from last known VTU snapshot.
SP	432	if(lastVTU==None):
	433	initSnap="initial_snapshot.vtu"
	434	else:
	435	initSnap=lastVTU
	436	print("WARNING: Not using initial snapshot as starting point, but selecting "+initSnap+" as a starting vesicle")
	437	params=["trisurf", "--restore-from-vtk",initSnap]+self.runArgs
	438	print("InitSnap is: "+initSnap)
47d80d	439	else:
3d0247	440	#veify if dump exists. If not it is a first run and shoud be run with --force-from-tape
SP	441	if(os.path.isfile("dump.bin")==False):
	442	self.runArgs.append("--force-from-tape")
b122c4	443	params=["trisurf"]+self.runArgs
fca8d6	444	subprocess.Popen (params, stdout=subprocess.DEVNULL)
47d80d	445	cwd.goto()
c14bd6	446	else:
3b83db	447	print("Process in "+self.Dir.fullpath()+" already running. Not starting.")
c14bd6	448	return
bc14fb	449
SP	450
c14bd6	451	def setMaindir(self,prefix,variables):
055489	452	maindir=""
c14bd6	453	for p,v in zip(prefix,variables):
SP	454	if(v=="xk0"):
3fc0ef	455	tv=str(round(float(self.Tape.config[v])))
56016c	456	if sys.version_info<(3,0):
3fc0ef	457	tv=str(int(float(self.Tape.config[v])))
c14bd6	458	else:
3fc0ef	459	tv=self.Tape.config[v]
c14bd6	460	maindir=maindir+p+tv
SP	461	self.maindir=maindir
	462	return
	463
	464	def setSubdir(self, subdir="run0"):
	465	self.subdir=subdir
	466	return
	467
	468	def getStatistics(self, statfile="statistics.csv"):
9154b3	469	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
38cb4a	470	pid=self.getPID()
055489	471	status=self.getStatus()
61ad95	472	if(self.Statistics.fileOK):
3fc0ef	473	ETA=str(datetime.timedelta(microseconds=(int(self.Tape.config['iterations'])-int(self.Statistics.last))self.Statistics.dT)1000000)
055489	474	if(status==TS_NONEXISTANT or status==TS_NOLOCK):
SP	475	statustxt="Not running"
9154b3	476	pid=""
38cb4a	477	ETA=""
055489	478	elif status==TS_STOPPED:
SP	479	statustxt="Stopped"
38cb4a	480	ETA="N/A"
3d0247	481	elif status==TS_COMPLETED:
SP	482	statustxt="Completed"
38cb4a	483	pid=""
SP	484	ETA=""
055489	485	else:
SP	486	statustxt="Running"
9154b3	487
61ad95	488	if(self.Statistics.fileOK):
SP	489	report=[time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(self.Statistics.startDate))),ETA, statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
9154b3	490	else:
25296b	491	report=["N/A","N/A",statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
f81585	492	return report
c14bd6	493
f06659	494
SP	495	def stop(self):
61ad95	496	try:
SP	497	p=psutil.Process(self.getPID())
	498	p.kill()
	499	except:
	500	print("Could not stop the process. Is the process running? Do you have sufficient privileges?")
	501
f06659	502
055489	503	def writeComment(self, data, mode='w'):
9154b3	504	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
055489	505	self.Comment.writefile(data,mode=mode)
9154b3	506
2f74e4	507
SP	508	def getLastVTU(self):
3fc0ef	509	vtuidx=self.getLastIteration()-int(self.Tape.getValue("inititer"))
2f74e4	510	if vtuidx<0:
SP	511	return None
	512	else:
	513	return 'timestep_{:06d}.vtu'.format(vtuidx)
	514
bc14fb	515	def __str__(self):
c14bd6	516	if(self.getStatus()==0):
SP	517	str=" not running."
	518	else:
	519	str=" running."
	520	return(self.Dir.fullpath()+str)
	521
bc14fb	522