From d7e21ad16473dbc2b12f20c6ef34a09f73863912 Mon Sep 17 00:00:00 2001
From: Samo Penic <samo.penic@gmail.com>
Date: Sat, 07 Jan 2017 20:55:10 +0000
Subject: [PATCH] Copying all tapes to all remotes. Copy is possible to configured directory. Must fix runs
---
networkedExample.py | 4 +-
trisurf/Remote.py | 31 +++++++++++++++
trisurf/tsmgr.py | 40 ++++++++++++++-----
3 files changed, 61 insertions(+), 14 deletions(-)
diff --git a/networkedExample.py b/networkedExample.py
index 5552276..a38856f 100755
--- a/networkedExample.py
+++ b/networkedExample.py
@@ -37,8 +37,8 @@
-hosts=({'name':'natalie','address':'kabinet.penic.eu', 'runs':Runs, 'username':'samo'},
+hosts=({'name':'natalie','address':'kabinet.penic.eu', 'runs':Runs, 'username':'samo', 'remotebasepath':'simulations-test/subdir/subdir'},
{'name':'Hestia','address':'127.0.0.1', 'runs':Runs, 'username':'samo'})
-
analyses={'analysis1':analyze,}
+
tsmgr.start(hosts, analyses=analyses)
diff --git a/trisurf/Remote.py b/trisurf/Remote.py
index 89a00bd..977b96a 100644
--- a/trisurf/Remote.py
+++ b/trisurf/Remote.py
@@ -1,5 +1,5 @@
import paramiko
-
+import os.path
class Connection:
def __init__(self, hostname, port=22, username=None, password=None):
@@ -69,4 +69,33 @@
sftp.mkdir(directory)
sftp.close()
+ def mkdir_p(self,sftp, remote_directory):
+ """Change to this directory, recursively making new folders if needed.
+ Returns True if any folders were created. Recursive algorithm."""
+ if remote_directory == '/':
+ # absolute path so change directory to root
+ sftp.chdir('/')
+ return
+ if remote_directory == '':
+ # top-level relative directory must exist
+ return
+ try:
+ sftp.chdir(remote_directory) # sub-directory exists
+ except IOError:
+ dirname, basename = os.path.split(remote_directory.rstrip('/'))
+ self.mkdir_p(sftp, dirname) # make parent directories
+ sftp.mkdir(basename) # sub-directory missing, so created it
+ sftp.chdir(basename)
+ return True
+ def send_multiple_files_in_directory(self,local_files,directory):
+ sftp=self.ssh.open_sftp()
+# try:
+# sftp.chdir(directory) # Test if remote_path exists
+# except (IOError,FileNotFoundError):
+# sftp.mkdir(directory) # Create remote_path
+# sftp.chdir(directory)
+ self.mkdir_p(sftp, directory)
+ for f in set(local_files):
+ sftp.put(f, f)
+ sftp.close()
diff --git a/trisurf/tsmgr.py b/trisurf/tsmgr.py
index ca590c4..419c26b 100644
--- a/trisurf/tsmgr.py
+++ b/trisurf/tsmgr.py
@@ -74,6 +74,15 @@
def copyConfigAndConnect(hosts):
print("Connecting to remote hosts and copying config files, tapes and snapshots")
+ #create a list of files to be copied across all the remote hosts
+ file_list=[]
+ for h in hosts:
+ for r in h['runs']:
+ if(r.isFromSnapshot):
+ file_list.append(r.snapshotFilename)
+ else:
+ file_list.append(r.tapeFilename)
+ file_list.append(main.__file__)
for host in hosts:
if(host['name'] !=socket.gethostname()): #if I am not the computer named in host name
try:
@@ -87,16 +96,21 @@
rm=Remote.Connection(hostname=host['address'],username=username, port=port)
rm.connect()
# print ("Sendind file:"+main.__file__)
- rm.send_file(main.__file__,'remote_control.py')
- for run in host['runs']:
- try:
- rm.send_file(run.tapeFile,run.tapeFile)
- except:
- pass
- try:
- rm.send_file(run.snapshotFile,run.snapshotFile)
- except:
- pass
+ if('remotebasepath' in host):
+ remote_dir=host['remotebasepath']
+ else:
+ remote_dir='trisurf_simulations'
+ rm.send_multiple_files_in_directory(file_list,remote_dir)
+# rm.send_file(main.__file__,'remote_control.py')
+# for run in host['runs']:
+# try:
+# rm.send_file(run.tapeFile,run.tapeFile)
+# except:
+# pass
+# try:
+# rm.send_file(run.snapshotFile,run.snapshotFile)
+# except:
+# pass
host['_conn']= rm
# we are connected to all hosts...
return hosts
@@ -287,7 +301,11 @@
print("Host "+bcolors.WARNING+host['name']+bcolors.ENDC+" reports:")
perform_action(args,host, analyses=analyses, hosts=hosts)
elif not args['local_only']:
- output=host['_conn'].execute('python3 ./remote_control.py -x '+" ".join(argv))
+ if('remotebasepath' in host):
+ remote_dir=host['remotebasepath']
+ else:
+ remote_dir='trisurf_simulations'
+ output=host['_conn'].execute('python3 ./'+remote_dir+'/'+main.__file__+' -x '+" ".join(argv))
for line in output:
print(line.replace('\n',''))
--
Gitblit v1.9.3