Copying all tapes to all remotes. Copy is possible to configured directory....

Samo Penic

2017-01-07 d7e21ad16473dbc2b12f20c6ef34a09f73863912

Copying all tapes to all remotes. Copy is possible to configured directory. Must fix runs

 networkedExample.py

@@ -37,8 +37,8 @@



hosts=({'name':'natalie','address':'kabinet.penic.eu', 'runs':Runs, 'username':'samo'},
hosts=({'name':'natalie','address':'kabinet.penic.eu', 'runs':Runs, 'username':'samo', 'remotebasepath':'simulations-test/subdir/subdir'},
    {'name':'Hestia','address':'127.0.0.1', 'runs':Runs, 'username':'samo'})

analyses={'analysis1':analyze,}

tsmgr.start(hosts, analyses=analyses)

 trisurf/Remote.py

@@ -1,5 +1,5 @@
import paramiko

import os.path

class Connection:
    def __init__(self, hostname, port=22, username=None, password=None):
@@ -69,4 +69,33 @@
        sftp.mkdir(directory)
        sftp.close()

    def mkdir_p(self,sftp, remote_directory):
        """Change to this directory, recursively making new folders if needed.
        Returns True if any folders were created. Recursive algorithm."""
        if remote_directory == '/':
            # absolute path so change directory to root
            sftp.chdir('/')
            return
        if remote_directory == '':
            # top-level relative directory must exist
            return
        try:
            sftp.chdir(remote_directory) # sub-directory exists
        except IOError:
            dirname, basename = os.path.split(remote_directory.rstrip('/'))
            self.mkdir_p(sftp, dirname) # make parent directories
            sftp.mkdir(basename) # sub-directory missing, so created it
            sftp.chdir(basename)
        return True
    
    def send_multiple_files_in_directory(self,local_files,directory):
        sftp=self.ssh.open_sftp()
#        try:
#            sftp.chdir(directory)  # Test if remote_path exists
#        except (IOError,FileNotFoundError):
#            sftp.mkdir(directory)  # Create remote_path
#            sftp.chdir(directory)
        self.mkdir_p(sftp, directory)
        for f in set(local_files):	
            sftp.put(f, f)
        sftp.close()	

 trisurf/tsmgr.py

@@ -74,6 +74,15 @@

def copyConfigAndConnect(hosts):
    print("Connecting to remote hosts and copying config files, tapes and snapshots")
    #create a list of files to be copied across all the remote hosts
    file_list=[]
    for h in hosts:
        for r in h['runs']:
            if(r.isFromSnapshot):
                file_list.append(r.snapshotFilename)
            else:
                file_list.append(r.tapeFilename)
    file_list.append(main.__file__)
    for host in hosts:
        if(host['name'] !=socket.gethostname()): #if I am not the computer named in host name
            try:
@@ -87,16 +96,21 @@
            rm=Remote.Connection(hostname=host['address'],username=username, port=port)
            rm.connect()
#            print ("Sendind file:"+main.__file__)
            rm.send_file(main.__file__,'remote_control.py')
            for run in host['runs']:
                try:
                    rm.send_file(run.tapeFile,run.tapeFile)
                except:
                    pass
                try:
                    rm.send_file(run.snapshotFile,run.snapshotFile)
                except:
                    pass
            if('remotebasepath' in host):
                remote_dir=host['remotebasepath']
            else:
                remote_dir='trisurf_simulations'
            rm.send_multiple_files_in_directory(file_list,remote_dir)
#            rm.send_file(main.__file__,'remote_control.py')
#            for run in host['runs']:
#                try:
#                    rm.send_file(run.tapeFile,run.tapeFile)
#                except:
#                    pass
#                try:
#                    rm.send_file(run.snapshotFile,run.snapshotFile)
#                except:
#                    pass
            host['_conn']= rm
    # we are connected to all hosts...
    return hosts
@@ -287,7 +301,11 @@
                print("Host "+bcolors.WARNING+host['name']+bcolors.ENDC+" reports:")
            perform_action(args,host, analyses=analyses, hosts=hosts)
        elif not args['local_only']:
            output=host['_conn'].execute('python3 ./remote_control.py -x '+" ".join(argv))
            if('remotebasepath' in host):
                remote_dir=host['remotebasepath']
            else:
                remote_dir='trisurf_simulations'
            output=host['_conn'].execute('python3 ./'+remote_dir+'/'+main.__file__+' -x '+" ".join(argv))
            for line in output:
                print(line.replace('\n',''))