Modification of text files over ssh with Python - performance
Question: I need to replace some lines in text files via ssh, if they contain a certain
key string. I wrote the following simple Python function for that:

    
    
    def ssh_edit_file(h, u, file_in, file_out, key, new): 
        import paramiko, string, os
    
        c_out = []
    
        ssh = paramiko.SSHClient()
        ssh.load_system_host_keys(os.environ["HOME"] + "/.ssh/known_hosts")
        ssh.connect(h,username=u)
        ftp = ssh.open_sftp()
    
        f_in = ftp.file(file_in, "r")
        c_in = f_in.readlines()
        for line in c_in:
            if string.find(line, key) > 0:
                c_out.append(new + '\n')
            else:
                s = line
                c_out.append(s)
    
       f_out = ftp.file(file_out, "w")
       f_out.writelines(c_out)
    
       f_in.close()
       f_out.close()
       ftp.close()
       ssh.close()
    

This is a bit slow, which is ok for most of my files. If I use it for larger
files (~2k lines), speed becomes a problem. What could be done to speed this
up easily?


Answer: The idea of using [sed](/questions/tagged/sed "show questions tagged 'sed'")
worked well for my case. In return for the helpful suggestion I like the share
solution:

    
    
    def ssh_edit_file(h, u, file_in, file_out, key, new):
        """
        SSH_EDIT_FILE - replace a line in a remote file
        """ 
    
        import warnings
    
        try:
            s = new.replace("'","\'")
            if file_in == file_out:
                ssh_exe('sed -i "s/.*' + key + '.*/' + s + '/g" ' + file_out,
                        h = h, u = u )
            else:
                ssh_exe('sed -e "s/.*' + key + '.*/' + s + '/g" ' + 
                        file_in + '>|' +  file_out,
                        h = h, u = u )
        except:
            warnings.warn('key not found')
    
    def ssh_exe(cmd, client=None, h=None,u=None):
        """
        SSH_EXE - Run command via ssh
        """
    
        import paramiko, os
    
        if h:
            if client:
                  raise RuntimeError('Please specify either client of h and u')
            client = paramiko.SSHClient()
            client.load_system_host_keys(os.environ["HOME"] + "/.ssh/known_hosts")
            client.connect(h,username=u)
        else:
            if h or u:
                  raise RuntimeError('Please specify either client of h and u')
    
        stdin, stdout, stderr = client.exec_command(cmd)
    
        if stdout.channel.recv_exit_status():
            e = stderr.readlines()
            if not e == []:
                raise RuntimeError('Error in executing ' + cmd + '\n'.join(e))
    
        if h:
            client.close()
    

