Transfer output files with SCP

Related to my previous post on monitoring output, I also needed to provide a script to transfer the output files from the scripts to a remote server. For the same reasons, so the researcher can maintain his own scripts, this was written in Python rather than Bash.

I used the Paramiko library to do this with Python, which is available by default on our HPC cluster.

#!/usr/bin/env python

# Core libraries
import os
from pathlib import Path, PurePath

# 3rd party libraries
from paramiko.client import SSHClient


def copy_file(source_path, destination_host, destination_path,
              destination_username, private_key_file,
              private_key_passphrase):
    """
    Transfer a file with SFTP to remote system.

    Args:
        source_path: local file path to copy
        destination_host: remote host to copy to
        destination_path: remote path to copy to
        destination_username: remote username to login as
        private_key_file: local path to private key
        private_key_passphrase: passphrase to unlock the private key
    """

    client = SSHClient()
    client.load_system_host_keys()
    client.connect(destination_host, username=destination_username,
                   key_filename=private_key_file,
                   passphrase=private_key_passphrase)
    sftp = client.open_sftp()
    sftp.put(source_path, destination_path)
    sftp.close()
    client.close()


def copy_dir(source_path, destination_host, destination_path,
             destination_username, private_key_file,
             private_key_passphrase):
    """
    Transfer a directory recursively with SFTP to remote system.

    Args:
        source_path: local directory to copy
        destination_host: remote host to copy to
        destination_path: remote directory to copy into
        destination_username: remote username to login as
        private_key_file: local path to private key
        private_key_passphrase: passphrase to unlock the private key
    """

    def __copy_dir(sftp_client, dir_path, destination_path):
        for file in [x for x in dir_path.iterdir()]:
            if file.is_dir():
                sftp_client.mkdir(os.fspath(destination_path / file.name))
                # Recursively copy sub-directory
                __copy_dir(sftp_client, file, destination_path / file.name)
            else:
                sftp_client.put(os.fspath(file),
                                os.fspath(destination_path / file.name))

    client = SSHClient()
    client.load_system_host_keys()
    client.connect(destination_host, username=destination_username,
                   key_filename=private_key_file,
                   passphrase=private_key_passphrase)
    sftp = client.open_sftp()
    __copy_dir(sftp, Path(source_path), PurePath(destination_path))
    sftp.close()
    client.close()


if __name__ == '__main__':
    # copy_file('/path/to/python-sftp/test_scp.py',
    #           'remote-host.tld', '/remote/folder/test_scp.py',
    #           'remote-user', '/path/to/ssh/key', 'ssh-key-passphrase')
    copy_dir('/path/to/python-sftp', 'remote-host.tld',
             '/remote/folder', 'remote-user', '/path/to/ssh/key',
             'ssh-key-passphrase')