trisurf-ng.git - Gitblit

Samo Penic

2016-12-06 1d1f5861495fc6e07d8a683ca8dbbaa43936b212

commit \| author \| age
bc14fb	1	import configobj
bd6993	2	import xml.etree.ElementTree as ET
SP	3	import base64
	4	import zlib
56016c	5	import sys,io
c14bd6	6	import os
SP	7	from itertools import islice
	8	import mmap
7c45b1	9	import shlex
34600b	10	import psutil
f81585	11	import time
SP	12	import datetime
fca8d6	13	import subprocess
38cb4a	14	import shutil
bc14fb	15
055489	16	# Process status
SP	17	TS_NOLOCK=0 # lock file does not exist
	18	TS_NONEXISTANT=0 # process is not in the list of processes
	19	TS_STOPPED=1 # the process is listed, but is in stopped state
	20	TS_RUNNING=2 # process is running
3d0247	21	TS_COMPLETED=3 #simulation is completed
bc14fb	22
9154b3	23	class FileContent:
055489	24	'''
SP	25	Class is helpful for reading and writting the specific files.
	26	'''
9154b3	27	def __init__(self,filename):
055489	28	''' The instance is done by calling constructor FileContent(filename)
SP	29
	30	The object then reads filename if it exists, otherwise the data is empty string.
	31	User may force to reread file by calling the readline() method of the class.
	32
	33	Filename is stored in local variable for future file operations.
	34	'''
	35
9154b3	36	self.filename=filename
SP	37	self.readfile()
bc14fb	38
9154b3	39	def readfile(self):
055489	40	'''Force reread of the file and setting the data'''
SP	41	self.data=""
9154b3	42	try:
SP	43	with open (self.filename, "r") as myfile:
055489	44	self.data=myfile.read().replace('\n', '') #read the file and remove newline from the text
9154b3	45	except:
055489	46	pass # does nothing if error occurs
9154b3	47
SP	48
	49	def writefile(self, data, mode='w'):
055489	50	'''File may be updated by completely rewritting the file contents or appending the data to the end of the file.
SP	51	this is achieved by calling writefile(data, mode) method, where data is the string data to be written and
	52	mode can be 'a' for append and 'w' for writting the file anew.
	53	'''
9154b3	54	with open (self.filename, mode) as myfile:
SP	55	myfile.write(data)
	56
	57	def getText(self):
055489	58	'''
SP	59	Method getText() or calling the object itself returns string of data
	60	'''
9154b3	61	return self.data
SP	62
	63	def __str__(self):
055489	64	'''
SP	65	Method getText() or calling the object itself returns string of data
	66	'''
9154b3	67	return self.getText()
bc14fb	68
SP	69	class Tape:
055489	70	'''
SP	71	Special class that manages configuration of trisurf (commonly named tape). It can read and parse configuration from disk or parse it from string.
	72	'''
bc14fb	73
SP	74	def __init__(self):
055489	75	'''The object is instatiated by calling Tape() constructor without parameters'''
bc14fb	76	return
SP	77
	78	def readTape(self, tape='tape'):
055489	79	'''
SP	80	Tape is read and parsed by calling the readTape() method with optional tape parameter, which is full path to filename where the configuration is stored.
	81	If the tape cannot be read, it prints error and exits.
	82	'''
bc14fb	83	try:
SP	84	self.config=configobj.ConfigObj(tape)
cc95bd	85	with open (tape, "r") as myfile:
SP	86	self.rawText=myfile.read() #read the file
	87
bc14fb	88	except:
SP	89	print("Error reading or parsing tape file!\n")
	90	exit(1)
	91
	92	def setTape(self, string):
055489	93	'''Method setTape(string) parses the string in memory that hold the tape contents.'''
bd6993	94	self.config=configobj.ConfigObj(io.StringIO(string))
cc95bd	95	self.rawText=string
bc14fb	96	return
SP	97
	98	def getValue(self,key):
055489	99	'''Method getValue(key) returns value of a single parsed setting named "key".'''
SP	100
bc14fb	101	return self.config[key]
SP	102
c14bd6	103	def __str__(self):
055489	104	'''Calling the object itself, it recreates the tape contents from parsed values in form of key=value.'''
c14bd6	105	retval=""
SP	106	for key,val in self.config.iteritems():
	107	retval=retval+str(key)+" = "+str(val)+"\n"
	108	return retval
bc14fb	109
c14bd6	110
SP	111
	112	class Directory:
055489	113	'''
SP	114	Class deals with the paths where the simulation is run and data is stored.
	115	'''
c14bd6	116	def __init__(self, maindir=".", simdir=""):
055489	117	'''Initialization Directory() takes two optional parameters, namely maindir and simdir. Defaults to current directory. It sets local variables maindir and simdir accordingly.'''
c14bd6	118	self.maindir=maindir
SP	119	self.simdir=simdir
	120	return
	121
	122	def fullpath(self):
055489	123	'''
SP	124	Method returns string of path where the data is stored. It combines values of maindir and simdir as maindir/simdir on Unix.
	125	'''
c14bd6	126	return os.path.join(self.maindir,self.simdir)
SP	127
	128	def exists(self):
055489	129	''' Method checks whether the directory specified by fullpath() exists. It return True/False on completion.'''
c14bd6	130	path=self.fullpath()
SP	131	if(os.path.exists(path)):
055489	132	return True
c14bd6	133	else:
055489	134	return False
c14bd6	135
SP	136	def make(self):
055489	137	''' Method make() creates directory. If it fails it exits the program with error message.'''
c14bd6	138	try:
SP	139	os.makedirs(self.fullpath())
	140	except:
	141	print("Cannot make directory "+self.fullpath()+"\n")
	142	exit(1)
	143	return
	144
	145	def makeifnotexist(self):
055489	146	'''Method makeifnotexist() creates directory if it does not exist.'''
c14bd6	147	if(self.exists()==0):
SP	148	self.make()
bd826d	149	return True
SP	150	else:
	151	return False
c14bd6	152
SP	153	def remove(self):
055489	154	'''Method remove() removes directory recursively. WARNING! No questions asked.'''
c14bd6	155	if(self.exists()):
SP	156	try:
	157	os.rmdir(self.fullpath())
	158	except:
	159	print("Cannot remove directory "+self.fullpath()+ "\n")
	160	exit(1)
	161	return
	162
	163	def goto(self):
055489	164	'''
SP	165	Method goto() moves current directory to the one specified by fullpath(). WARNING: when using the relative paths, do not call this function multiple times.
	166	'''
c14bd6	167	try:
SP	168	os.chdir(self.fullpath())
	169	except:
	170	print("Cannot go to directory "+self.fullpath()+"\n")
	171	return
	172
	173
	174	class Statistics:
055489	175	'''
SP	176	Class that deals with the statistics file from the simulations.
	177	File is generally large and not all data is needed, so it is dealt with in a specific way.
	178	'''
	179
c14bd6	180	def __init__(self,path,filename="statistics.csv"):
055489	181	'''
SP	182	At the initialization call it receives optional filename parameter specifying the path and filename of the statistics file.
	183
	184	The local variables path, filename, fullname (joined path and filename) and private check if the file exists are stored.
	185	'''
c14bd6	186	self.path=path
SP	187	self.filename=filename
	188	self.fullname=os.path.join(path,filename)
f81585	189	self.fileOK=self.read()
c14bd6	190	return
SP	191
	192	def exists(self):
055489	193	'''Method check if the statistics file exists.'''
c14bd6	194	if(os.path.isfile(self.fullname)):
SP	195	return True
	196	else:
	197	return False
	198
	199	def mapcount(self):
055489	200	'''
SP	201	Internal method for determining the number of the lines in the most efficient way. Is it really the most efficient?
	202	'''
c14bd6	203	f = open(self.fullname, "r+")
d665c0	204	try:
SP	205	buf = mmap.mmap(f.fileno(), 0)
	206	lines = 0
	207	readline = buf.readline
	208	while readline():
	209	lines += 1
	210	f.close()
	211	except:
	212	lines=0
	213	f.close()
c14bd6	214	return lines
6568b3	215
SP	216	def tail(self,filename,n=2):
	217	with open(filename,'r') as myfile:
	218	lines=myfile.readlines()
	219	return [lines[len(lines)-2].replace('\n',''),lines[len(lines)-1].replace('\n','')]
	220
c14bd6	221	def read(self):
6568b3	222	try:
SP	223	lines=self.tail(self.fullname)
	224	except:
	225	return(False)
	226	if len(lines)<2:
	227	return(False)
	228	#print (line)
	229	fields=shlex.split(lines[0])
	230	epoch1=fields[0]
	231	n1=fields[1]
	232
	233	fields=shlex.split(lines[1])
	234	epoch2=fields[0]
	235	n2=fields[1]
	236	try:
	237	self.dT=int(epoch2)-int(epoch1)
	238	self.last=n2
ddb86f	239	#print(epoch1)
SP	240	#print(epoch2)
	241	#print(self.dT)
	242	#print(self.last)
6568b3	243	self.startDate=os.path.getmtime(os.path.join(self.path,'.lock'))
SP	244	except:
	245	return(False)
	246	return(True)
	247
	248	def read_old(self):
055489	249	'''
SP	250	Method read() reads the statistics if it exists. It sets local variable dT storing the time differential between two intervals of simulation (outer loops). It also stores last simulation loop and the start of the run.
	251	'''
c14bd6	252	if(self.exists()):
3d0247	253	# epoch1=0
SP	254	# epoch2=0
	255	# n1=0
	256	# n2=0
c14bd6	257	nlines=self.mapcount()
3d0247	258	if nlines<2:
SP	259	return(False)
c14bd6	260	try:
SP	261	with open(self.fullname, "r+") as fin:
	262	i=0;
	263	for line in fin:
	264	if(i==1):
7c45b1	265	#print (line)
SP	266	fields=shlex.split(line)
	267	epoch1=fields[0]
	268	n1=fields[1]
c14bd6	269	if(i==nlines-1):
7c45b1	270	fields=shlex.split(line)
SP	271	epoch2=fields[0]
	272	n2=fields[1]
c14bd6	273	i=i+1
SP	274	except:
f81585	275	#print("Cannot read statistics file in "+self.fullname+"\n")
7c45b1	276	return(False)
c14bd6	277	else:
f81585	278	#print("File "+self.fullname+" does not exists.\n")
7c45b1	279	return(False)
3d0247	280	try:
SP	281	self.dT=(int(epoch2)-int(epoch1))/(int(n2)-int(n1))
	282	except:
	283	self.dT=0
f81585	284	self.last=n2
SP	285	self.startDate=epoch1
7c45b1	286	return(True)
f81585	287
SP	288	def __str__(self):
055489	289	'''
SP	290	Prints the full path with filename of the statistics.csv file
	291	'''
f81585	292	return(str(self.fullname))
SP	293
7c45b1	294
bc14fb	295
SP	296	class Runner:
	297	'''
055489	298	Class Runner consists of a single running or terminated instance of the trisurf. It manages starting, stopping, verifying the running process and printing the reports of the configured instances.
bc14fb	299	'''
bd826d	300	def __init__(self, subdir='run0', tape=None, snapshot=None, runArgs=[]):
c14bd6	301	self.subdir=subdir
b122c4	302	self.runArgs=runArgs
47d80d	303	self.fromSnapshot=False
bd826d	304	if(tape!=None):
c14bd6	305	self.initFromTape(tape)
bd826d	306	if(snapshot!=None):
c14bd6	307	self.initFromSnapshot(snapshot)
SP	308	return
	309
	310
a99f2b	311	def initFromTape(self, tape):
bc14fb	312	self.tape=Tape()
SP	313	self.tape.readTape(tape)
bd826d	314	self.tapeFile=tape
bc14fb	315
bd6993	316	def initFromSnapshot(self, snapshotfile):
SP	317	try:
	318	tree = ET.parse(snapshotfile)
	319	except:
	320	print("Error reading snapshot file")
	321	exit(1)
47d80d	322	self.fromSnapshot=True
SP	323	self.snapshotFile=snapshotfile
bd6993	324	root = tree.getroot()
SP	325	tapetxt=root.find('tape')
	326	version=root.find('trisurfversion')
	327	self.tape=Tape()
	328	self.tape.setTape(tapetxt.text)
2ded03	329
9154b3	330	def getPID(self):
34600b	331	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
bd826d	332	#self.Dir.makeifnotexist()
34600b	333	try:
SP	334	fp = open(os.path.join(self.Dir.fullpath(),'.lock'))
	335	except IOError as e:
	336	return 0 #file probably does not exist. e==2??
9154b3	337	pid=fp.readline()
SP	338	fp.close()
055489	339	return int(pid)
9154b3	340
3d0247	341	def getLastIteration(self):
SP	342	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
	343	#self.Dir.makeifnotexist()
	344	try:
	345	fp = open(os.path.join(self.Dir.fullpath(),'.status'))
	346	except IOError as e:
	347	return -1 #file probably does not exist. e==2??
	348	status=fp.readline()
	349	fp.close()
	350	return int(status)
	351
	352	def isCompleted(self):
a17e91	353	if int(self.tape.getValue("iterations"))+int(self.tape.getValue("inititer"))==self.getLastIteration()+1:
3d0247	354	return True
SP	355	else:
	356	return False
	357
9154b3	358	def getStatus(self):
SP	359	pid=self.getPID()
3d0247	360	if(self.isCompleted()):
SP	361	return TS_COMPLETED
9154b3	362	if(pid==0):
055489	363	return TS_NOLOCK
34600b	364	if(psutil.pid_exists(int(pid))):
055489	365	proc= psutil.Process(int(pid))
56ba2f	366	#psutil.__version__ == '3.4.2' requires name() and status(), some older versions reguire name, status
9aa037	367	if(psutil.__version__>='2.0.0'):
SP	368	procname=proc.name()
	369	procstat=proc.status()
	370	else:
	371	procname=proc.name
	372	procstat=proc.status
	373	if procname=="trisurf":
	374	if procstat=="stopped":
055489	375	return TS_STOPPED
SP	376	else:
	377	return TS_RUNNING
9154b3	378	else:
055489	379	return TS_NONEXISTANT
34600b	380	else:
055489	381	return TS_NONEXISTANT
bc14fb	382
SP	383	def start(self):
3d0247	384	if(self.getStatus()==0 or self.getStatus()==TS_COMPLETED):
38cb4a	385	#check if executable exists
SP	386	if(shutil.which('trisurf')==None):
	387	print("Error. Trisurf executable not found in PATH. Please install trisurf prior to running trisurf manager.")
	388	exit(1)
c14bd6	389	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
bd826d	390	#Symlinks tape file to the directory or create tape file from snapshot in the direcory...
SP	391	if(self.Dir.makeifnotexist()):
	392	if(self.fromSnapshot==False):
	393	try:
	394	os.symlink(os.path.abspath(self.tapeFile), self.Dir.fullpath()+"/tape")
	395	except:
	396	print("Error while symlinking "+os.path.abspath(self.tapeFile)+" to "+self.Dir.fullpath()+"/tape")
	397	exit(1)
	398	else:
	399	try:
	400	with open (os.path.join(self.Dir.fullpath(),"tape"), "w") as myfile:
cc95bd	401	#myfile.write("#This is automatically generated tape file from snapshot")
SP	402	myfile.write(str(self.tape.rawText))
bd826d	403	except:
SP	404	print("Error -- cannot make tapefile "+ os.path.join(self.Dir.fullpath(),"tape")+" from the snapshot in the running directory")
	405	exit(1)
	406	try:
354145	407	os.symlink(os.path.abspath(self.snapshotFile), os.path.join(self.Dir.fullpath(),"initial_snapshot.vtu"))
bd826d	408	except:
SP	409	print("Error while symlinking "+os.path.abspath(self.snapshotFile)+" to "+os.path.join(self.Dir.fullpath(),self.snapshotFile))
3d0247	410
SP	411	#check if the simulation has been completed. in this case notify user and stop executing.
	412	if(self.isCompleted() and ("--force-from-tape" not in self.runArgs) and ("--reset-iteration-count" not in self.runArgs)):
3b83db	413	print("The simulation was completed. Not starting executable in "+self.Dir.fullpath())
3d0247	414	return
SP	415
47d80d	416	cwd=Directory(maindir=os.getcwd())
2f74e4	417	lastVTU=self.getLastVTU() #we get last VTU file in case we need to continue the simulation from last snapshot. Need to be done before the Dir.goto() call.
47d80d	418	self.Dir.goto()
3b83db	419	print("Starting trisurf-ng executable in "+self.Dir.fullpath())
47d80d	420	if(self.fromSnapshot==True):
2f74e4	421	#here we try to determine whether we should continue the simulation or start from last known VTU snapshot.
SP	422	if(lastVTU==None):
	423	initSnap="initial_snapshot.vtu"
	424	else:
	425	initSnap=lastVTU
	426	print("WARNING: Not using initial snapshot as starting point, but selecting "+initSnap+" as a starting vesicle")
	427	params=["trisurf", "--restore-from-vtk",initSnap]+self.runArgs
	428	print("InitSnap is: "+initSnap)
47d80d	429	else:
3d0247	430	#veify if dump exists. If not it is a first run and shoud be run with --force-from-tape
SP	431	if(os.path.isfile("dump.bin")==False):
	432	self.runArgs.append("--force-from-tape")
b122c4	433	params=["trisurf"]+self.runArgs
fca8d6	434	subprocess.Popen (params, stdout=subprocess.DEVNULL)
47d80d	435	cwd.goto()
c14bd6	436	else:
3b83db	437	print("Process in "+self.Dir.fullpath()+" already running. Not starting.")
c14bd6	438	return
bc14fb	439
SP	440
c14bd6	441	def setMaindir(self,prefix,variables):
055489	442	maindir=""
c14bd6	443	for p,v in zip(prefix,variables):
SP	444	if(v=="xk0"):
	445	tv=str(round(float(self.tape.config[v])))
56016c	446	if sys.version_info<(3,0):
SP	447	tv=str(int(float(self.tape.config[v])))
c14bd6	448	else:
SP	449	tv=self.tape.config[v]
	450	maindir=maindir+p+tv
	451	self.maindir=maindir
	452	return
	453
	454	def setSubdir(self, subdir="run0"):
	455	self.subdir=subdir
	456	return
	457
	458	def getStatistics(self, statfile="statistics.csv"):
7ca5ae	459	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
34600b	460	self.statistics=Statistics(self.Dir.fullpath(), statfile)
9154b3	461	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
38cb4a	462	pid=self.getPID()
055489	463	status=self.getStatus()
25296b	464	if(self.statistics.fileOK):
SP	465	ETA=str(datetime.timedelta(microseconds=(int(self.tape.config['iterations'])-int(self.statistics.last))self.statistics.dT)1000000)
055489	466	if(status==TS_NONEXISTANT or status==TS_NOLOCK):
SP	467	statustxt="Not running"
9154b3	468	pid=""
38cb4a	469	ETA=""
055489	470	elif status==TS_STOPPED:
SP	471	statustxt="Stopped"
38cb4a	472	ETA="N/A"
3d0247	473	elif status==TS_COMPLETED:
SP	474	statustxt="Completed"
38cb4a	475	pid=""
SP	476	ETA=""
055489	477	else:
SP	478	statustxt="Running"
9154b3	479
SP	480	if(self.statistics.fileOK):
38cb4a	481	report=[time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(self.statistics.startDate))),ETA, statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
9154b3	482	else:
25296b	483	report=["N/A","N/A",statustxt, pid, str(self.Dir.fullpath()), self.Comment.getText()]
f81585	484	return report
c14bd6	485
f06659	486
SP	487	def stop(self):
	488	p=psutil.Process(self.getPID())
	489	p.kill()
	490
055489	491	def writeComment(self, data, mode='w'):
9154b3	492	self.Dir=Directory(maindir=self.maindir,simdir=self.subdir)
SP	493	self.Comment=FileContent(os.path.join(self.Dir.fullpath(),".comment"))
055489	494	self.Comment.writefile(data,mode=mode)
9154b3	495
2f74e4	496
SP	497	def getLastVTU(self):
d58644	498	vtuidx=self.getLastIteration()-int(self.tape.getValue("inititer"))
2f74e4	499	if vtuidx<0:
SP	500	return None
	501	else:
	502	return 'timestep_{:06d}.vtu'.format(vtuidx)
	503
bc14fb	504	def __str__(self):
c14bd6	505	if(self.getStatus()==0):
SP	506	str=" not running."
	507	else:
	508	str=" running."
	509	return(self.Dir.fullpath()+str)
	510
bc14fb	511