| commit | author | age | ||
| 8ab985 | 1 | import argparse |
| SP | 2 | import paramiko |
| 3 | from . import Remote | |
| 4 | from . import trisurf | |
| 5 | import socket | |
| 6 | import os,sys | |
| 7 | import tabulate | |
| 8 | import subprocess,re | |
| 9 | import psutil | |
| 10 | #import http.server | |
| 11 | #import socketserver | |
| 12 | if sys.version_info>=(3,0): | |
| 13 | from urllib.parse import urlparse | |
| 14 | from . import WebTrisurf | |
| 15 | else: | |
| 16 | from urlparse import urlparse | |
| 17 | from vtk import * | |
| 8b7022 | 18 | |
| 8ab985 | 19 | #import io |
| SP | 20 | |
| 52e871 | 21 | #from IPython import embed |
| 8ab985 | 22 | |
| f0131d | 23 | import __main__ as main |
| SP | 24 | |
| 8ab985 | 25 | #Color definitions for terminal |
| SP | 26 | class bcolors: |
| 27 | HEADER = '\033[95m' | |
| 28 | OKBLUE = '\033[94m' | |
| 29 | OKGREEN = '\033[92m' | |
| 30 | WARNING = '\033[93m' | |
| 31 | FAIL = '\033[91m' | |
| 32 | ENDC = '\033[0m' | |
| 33 | BOLD = '\033[1m' | |
| 34 | UNDERLINE = '\033[4m' | |
| 35 | ||
| 36 | #parses Command Line Arguments and returns the list of parsed values | |
| 37 | def ParseCLIArguments(arguments): | |
| 38 | parser = argparse.ArgumentParser(description='Manages (start, stop, status) multiple simulation processes of trisurf according to the configuration file.') | |
| 39 | parser.add_argument('proc_no', metavar='PROC_NO', nargs='*', | |
| 40 | help='process number at host. If hostname is not specified, localhost is assumed. If no processes are specified all processes on all hosts are assumed.') | |
| 41 | action_group=parser.add_mutually_exclusive_group(required=True) | |
| 42 | action_group.add_argument('-c','--comment',nargs=1, help='append comment to current comment') | |
| 86c973 | 43 | action_group.add_argument('--analysis', nargs='+', help='runs analysis function defined in configuration file') |
| 8ab985 | 44 | action_group.add_argument('--delete-comment', help='delete comment',action='store_true') |
| SP | 45 | action_group.add_argument('-k','--kill','--stop','--suspend', help='stop/kill the process', action='store_true') |
| 46 | action_group.add_argument('-r','--run','--start','--continue', help='start/continue process', action='store_true') | |
| 47 | action_group.add_argument('-s','--status',help='print status of the processes',action='store_true') | |
| 48 | action_group.add_argument('-v','--version', help='print version information and exit', action='store_true') | |
| 49 | action_group.add_argument('--web-server', type=int,metavar="PORT", nargs=1, help='EXPERIMENTAL: starts web server and never exist.') | |
| 50 | action_group.add_argument('--jump-to-ipython', help='loads the variables and jumps to IPython shell', action="store_true") | |
| 51 | action_group.add_argument('-p','--preview',help='preview last VTU shape',action='store_true') | |
| 52 | parser.add_argument('--force', help='if dangerous operation (killing all the processes) is requested, this flag is required to execute the operation. Otherwise, the request will be ignored.', action="store_true") | |
| 1c8250 | 53 | parser.add_argument('-H', '--host', nargs='+', help='specifies which host is itended for the operation. Defauts to localhost for all operations except --status and --version, where all configured hosts are assumed.') |
| 8ab985 | 54 | parser.add_argument('--html', help='Generate HTML output', action="store_true") |
| SP | 55 | parser.add_argument('-n', nargs='+', metavar='PROC_NO', type=int, help='OBSOLETE. Specifies process numbers.') |
| 56 | parser.add_argument('-R','--raw',help='print status and the rest of the information in raw format', action="store_true") | |
| 57 | parser.add_argument('-x','--local-only',help='do not attempt to contact remote hosts. Run all operations only on local machine',action='store_true') | |
| 6f912f | 58 | parser.add_argument('--originating-host',nargs=1,help='specify which host started the remote connections. Useful mainly fo internal functionaly of tsmgr and analyses.') |
| 8ab985 | 59 | args = parser.parse_args(arguments) |
| SP | 60 | return args |
| 61 | ||
| 62 | ||
| 63 | #gets version of trisurf currently running | |
| 64 | def getTrisurfVersion(): | |
| 65 | p = subprocess.Popen('trisurf --version', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | |
| 66 | lines=p.stdout.readlines() | |
| 67 | version=re.findall(r'[0-9a-f]{7}(?:-dirty)?', lines[0].decode('ascii')) | |
| 68 | p.wait() | |
| 69 | if(len(version)): | |
| 70 | return version[0] | |
| 71 | else: | |
| 72 | return "unknown version" | |
| 73 | ||
| 74 | ||
| 75 | ||
| 76 | def copyConfigAndConnect(hosts): | |
| 77 | print("Connecting to remote hosts and copying config files, tapes and snapshots") | |
| d7e21a | 78 | #create a list of files to be copied across all the remote hosts |
| SP | 79 | file_list=[] |
| 80 | for h in hosts: | |
| 81 | for r in h['runs']: | |
| 82 | if(r.isFromSnapshot): | |
| a5f021 | 83 | file_list.append(r.snapshotFile) |
| d7e21a | 84 | else: |
| SP | 85 | file_list.append(r.tapeFilename) |
| 86 | file_list.append(main.__file__) | |
| 8ab985 | 87 | for host in hosts: |
| SP | 88 | if(host['name'] !=socket.gethostname()): #if I am not the computer named in host name |
| 89 | try: | |
| 90 | username=host['username'] | |
| 91 | except: | |
| 92 | username=os.getusername() #default username is current user user's name | |
| 93 | try: | |
| 94 | port=host['port'] | |
| 95 | except: | |
| 96 | port=22 #default ssh port | |
| 52e871 | 97 | try: |
| SP | 98 | rm=Remote.Connection(hostname=host['address'],username=username, port=port) |
| 99 | rm.connect() | |
| 100 | except: | |
| 101 | host['_conn']=None | |
| 102 | continue | |
| f0131d | 103 | # print ("Sendind file:"+main.__file__) |
| d7e21a | 104 | if('remotebasepath' in host): |
| SP | 105 | remote_dir=host['remotebasepath'] |
| 106 | else: | |
| 107 | remote_dir='trisurf_simulations' | |
| 108 | rm.send_multiple_files_in_directory(file_list,remote_dir) | |
| 109 | # rm.send_file(main.__file__,'remote_control.py') | |
| 110 | # for run in host['runs']: | |
| 111 | # try: | |
| 112 | # rm.send_file(run.tapeFile,run.tapeFile) | |
| 113 | # except: | |
| 114 | # pass | |
| 115 | # try: | |
| 116 | # rm.send_file(run.snapshotFile,run.snapshotFile) | |
| 117 | # except: | |
| 118 | # pass | |
| 8ab985 | 119 | host['_conn']= rm |
| SP | 120 | # we are connected to all hosts... |
| 121 | return hosts | |
| 122 | ||
| 123 | ||
| 124 | ||
| 125 | def getTargetRunIdxList(args): | |
| 126 | target_runs=list(map(int,args['proc_no'])) | |
| 127 | if len(target_runs)==0: | |
| 128 | #check if obsolete -n flags have numbers | |
| 129 | target_runs=args['n'] | |
| 130 | if target_runs==None: | |
| 131 | return None | |
| 132 | target_runs=list(set(target_runs)) | |
| 133 | return target_runs | |
| 134 | ||
| 135 | ||
| 136 | ||
| 137 | def status_processes(args,host): | |
| 52e871 | 138 | print("in status processes") |
| 8ab985 | 139 | target_runs=getTargetRunIdxList(args) |
| SP | 140 | if target_runs==None: |
| 141 | target_runs=list(range(1,len(host['runs'])+1)) | |
| 142 | report=[] | |
| f0131d | 143 | # print("was here") |
| 8ab985 | 144 | for i in target_runs: |
| SP | 145 | line=host['runs'][i-1].getStatistics() |
| 146 | line.insert(0,i) | |
| 147 | report.append(line) | |
| 148 | if(args['raw']): | |
| 149 | print(report) | |
| 150 | else: | |
| 151 | if(args['html']): | |
| 152 | tablefmt='html' | |
| 153 | else: | |
| 154 | tablefmt='fancy_grid' | |
| 155 | print(tabulate.tabulate(report,headers=["Run no.", "Run start time", "ETA", "Status", "PID", "Path", "Comment"], tablefmt=tablefmt)) | |
| 156 | return | |
| 157 | ||
| 158 | def run_processes(args,host): | |
| 159 | target_runs=getTargetRunIdxList(args) | |
| 160 | if target_runs==None: | |
| 161 | target_runs=list(range(1,len(host['runs'])+1)) | |
| 162 | for i in target_runs: | |
| 163 | host['runs'][i-1].start() | |
| 164 | return | |
| 165 | ||
| 166 | def kill_processes(args,host): | |
| 167 | target_runs=getTargetRunIdxList(args) | |
| 168 | if target_runs==None: | |
| 169 | if args['force']==True: | |
| 170 | target_runs=list(range(1,len(host['runs'])+1)) | |
| 171 | else: | |
| 172 | print("Not stopping all processes on the host. Run with --force flag if you are really sure to stop all simulations") | |
| 173 | return | |
| 174 | for i in target_runs: | |
| 175 | host['runs'][i-1].stop() | |
| 176 | return | |
| 177 | ||
| 178 | def comment_processes(args,host): | |
| 179 | target_runs=getTargetRunIdxList(args) | |
| 180 | if target_runs==None: | |
| 181 | target_runs=list(range(1,len(host['runs'])+1)) | |
| 182 | for i in target_runs: | |
| 183 | host['runs'][i-1].writeComment(args['comment'][0],'a') | |
| 184 | print("Comment added") | |
| 185 | return | |
| 186 | ||
| 187 | def delete_comments(args,host): | |
| 188 | target_runs=getTargetRunIdxList(args) | |
| 189 | if target_runs==None: | |
| 190 | if args['force']==True: | |
| 191 | target_runs=list(range(1,len(host['runs'])+1)) | |
| 192 | else: | |
| 193 | print("Not deleting comments on all posts on the host. Run with --force flag if you are really sure to delete all comments") | |
| 194 | return | |
| 195 | for i in target_runs: | |
| 196 | host['runs'][i-1].writeComment("") | |
| 197 | print("Comment deleted") | |
| 198 | return | |
| 199 | ||
| 200 | ||
| 201 | def start_web_server(args,host): | |
| 202 | print('Server listening on port {}'.format(args['web_server'][0])) | |
| 203 | if sys.version_info>=(3,0): | |
| 204 | WebTrisurf.WebServer(port=args['web_server'][0]) | |
| 205 | else: | |
| 206 | print("Cannot start WebServer in python 2.7") | |
| 207 | exit(0) | |
| 208 | ||
| f0131d | 209 | |
| 1c8250 | 210 | def analyze(args,host,a_dict, analysis,hosts): |
| 86c973 | 211 | if len(a_dict)==0: |
| SP | 212 | print ('Error: no analyses are specified in the tsmgr.start()!') |
| 213 | exit(1) | |
| f0131d | 214 | target_runs=getTargetRunIdxList(args) |
| SP | 215 | if target_runs==None: |
| 216 | target_runs=list(range(1,len(host['runs'])+1)) | |
| 217 | for i in target_runs: | |
| 218 | ||
| 219 | for anal in analysis: | |
| 220 | if(anal not in a_dict): | |
| 221 | print("Analysis '"+anal+"' is not known. Available analyses: "+", ".join(a_dict.keys())+".") | |
| 222 | exit(0) | |
| 223 | for anal in analysis: | |
| 1c8250 | 224 | retval=a_dict[anal](host['runs'][i-1],host=host, args=args, hosts=hosts) |
| 8c3c29 | 225 | #try: |
| SP | 226 | if(retval): |
| 227 | exit(0) | |
| 228 | #except: | |
| 229 | # pass | |
| f0131d | 230 | |
| 1c8250 | 231 | |
| SP | 232 | |
| f0131d | 233 | |
| SP | 234 | def perform_action(args,host,**kwargs): |
| 8ab985 | 235 | #find which flags have been used and act upon them. -r -s -k -v -c --delete-comment are mutually exclusive, so only one of them is active |
| SP | 236 | if args['run']: |
| 237 | run_processes(args,host) | |
| 238 | elif args['kill']: | |
| 239 | kill_processes(args,host) | |
| 240 | elif args['status']: | |
| 241 | status_processes(args,host) | |
| 242 | elif args['comment']!= None: | |
| 243 | comment_processes(args,host) | |
| 244 | elif args['delete_comment']: | |
| 245 | delete_comments(args,host) | |
| 246 | elif args['web_server']!=None: | |
| 247 | start_web_server(args,host) | |
| 248 | elif args['preview']: | |
| 249 | preview_vtu(args,host) | |
| 250 | elif args['jump_to_ipython']: | |
| 251 | print('Jumping to shell...') | |
| 252 | embed() | |
| f0131d | 253 | exit(0) |
| SP | 254 | elif args['analysis']!= None: |
| 1c8250 | 255 | analyze(args,host,kwargs.get('analyses', {}), args['analysis'],kwargs.get('hosts',None)) |
| 8ab985 | 256 | exit(0) |
| SP | 257 | else: #version requested |
| 258 | print(getTrisurfVersion()) | |
| 259 | return | |
| 260 | ||
| 261 | ||
| 262 | ||
| 263 | def preview_vtu(args,host): | |
| d00e25 | 264 | from . import VTKRendering |
| f49271 | 265 | target_runs=getTargetRunIdxList(args) |
| SP | 266 | if target_runs==None: |
| 267 | target_runs=list(range(1,len(host['runs'])+1)) | |
| 8ab985 | 268 | if host['name'] == socket.gethostname(): |
| f49271 | 269 | for i in target_runs: |
| SP | 270 | VTKRendering.Renderer(args,host,host['runs'][i-1]) |
| 271 | else: | |
| 272 | print("VTK rendering currently works on localhost only!") | |
| 273 | ||
| 8ab985 | 274 | |
| SP | 275 | def getListOfHostConfigurationByHostname(hosts,host): |
| 276 | rhost=[] | |
| 277 | for chost in hosts: | |
| 278 | if chost['name'] in host: | |
| 279 | rhost.append(chost) | |
| 280 | return rhost | |
| 281 | ||
| 282 | ||
| 283 | ||
| f0131d | 284 | def start(hosts,argv=sys.argv[1:], analyses={}): |
| 8ab985 | 285 | args=vars(ParseCLIArguments(argv)) |
| SP | 286 | #Backward compatibility... If running just on localmode, the host specification is unnecessary. Check if only Runs are specified |
| 287 | try: | |
| 288 | test_host=hosts[0]['name'] | |
| 289 | except: | |
| f0131d | 290 | print("Old syntax detected.") |
| SP | 291 | hosts=({'name':socket.gethostname(),'address':'127.0.0.1', 'runs':hosts},) |
| 8ab985 | 292 | |
| SP | 293 | #find the host at which the action is attended |
| 20db82 | 294 | # print(args) |
| 8ab985 | 295 | if args['host']==None: |
| 1c8250 | 296 | #Only status and version commands are automatically executed on all the hosts. stopping or starting or other actions is not! |
| 20db82 | 297 | # if(args['status']==False and args['version']==False and): |
| SP | 298 | # hosts=getListOfHostConfigurationByHostname(hosts,socket.gethostname()) |
| 299 | pass | |
| 8ab985 | 300 | else: |
| SP | 301 | hosts=getListOfHostConfigurationByHostname(hosts,args['host']) |
| 302 | if len(hosts)==0: | |
| 303 | print ('Hostname "{}" does not exist in configuration file. Please check the spelling'.format(args['host'][0])) | |
| 304 | exit(1) | |
| 305 | if not args['local_only']: | |
| 20db82 | 306 | # print("copying to remotes, DEBUG") |
| SP | 307 | # print(hosts) |
| 8ab985 | 308 | hosts=copyConfigAndConnect(hosts) |
| SP | 309 | #do local stuff: |
| 310 | for host in hosts: | |
| 311 | if host['name'] == socket.gethostname(): | |
| 312 | if(args['html']): | |
| 8c3c29 | 313 | print("Host <font color='orange'>"+host['name']+"</font> reports:") |
| 8ab985 | 314 | else: |
| 8c3c29 | 315 | print("Host "+bcolors.WARNING+host['name']+bcolors.ENDC+" reports:") |
| 1c8250 | 316 | perform_action(args,host, analyses=analyses, hosts=hosts) |
| 8ab985 | 317 | elif not args['local_only']: |
| d7e21a | 318 | if('remotebasepath' in host): |
| SP | 319 | remote_dir=host['remotebasepath'] |
| 320 | else: | |
| 321 | remote_dir='trisurf_simulations' | |
| 4e6b88 | 322 | #output=host['_conn'].execute('cd '+remote_dir) |
| SP | 323 | #print(remote_dir) |
| 324 | #print(main.__file__) | |
| 325 | #print('python3 '+main.__file__+' -x '+" ".join(argv)) | |
| 52e871 | 326 | if(host['_conn']!=None): |
| SP | 327 | # print("was here, "+'cd '+remote_dir+ '; python3 '+main.__file__+' -x --originating-host ' +socket.gethostname()+" "+" ".join(argv)) |
| 328 | output=host['_conn'].execute('cd '+remote_dir+ '; python3 '+main.__file__+' -x --originating-host ' +socket.gethostname()+" "+" ".join(argv)) | |
| 329 | for line in output: | |
| 330 | print(line.replace('\n','')) | |
| 8ab985 | 331 | |
| SP | 332 | |
| 333 | if not args['local_only']: | |
| 334 | print("Closing connections to remote hosts") | |
| 335 | for host in hosts: | |
| 336 | if(host['name'] !=socket.gethostname()): | |
| 52e871 | 337 | if(host['_conn']): |
| SP | 338 | host['_conn'].disconnect() |
| 8ab985 | 339 | |
| SP | 340 | |
| 341 | ||
