import subprocess
from processing import Pool

def do_cp(destination, local, remote):
    print "Copying", local, "to", remote, "on", destination
    subprocess.call('cat %s | ssh %s "hadoop dfs -put - %s"' % (local, destination, remote), shell=True)

class RemoteDFS:
    concurrency = 4
    
    def __init__(self, destination):
        self.destination = destination
        self.pool = Pool(processes=self.concurrency)

    def cp(self, local, remote):
        self.pool.apply_async(do_cp, args=(self.destination, local, remote))