                     c?()
             async.mapLimit(projects, opts.limit, f, cb)
     ], (err) -> opts.cb?(err))
-exports.migrate_bup_all = (opts) ->
+
+exports.xxxmigrate_bup_all = (opts) ->
     opts = defaults opts,
         limit : 20           # no more than this many projects will be migrated simultaneously
         start : undefined    # if given, only takes projects.slice(start, stop) -- useful for debugging



# This NOW ASSUMES that prepare_bup completed.
exports.migrate_bup = (opts) ->
    opts = defaults opts,
        project_id : required
        server     : undefined   # rsync here...




exports.migrate2_bup_all = (opts) ->
    opts = defaults opts,
        limit : 1           # no more than this many projects will be migrated simultaneously
        start : undefined    # if given, only takes projects.slice(start, stop) -- useful for debugging
        stop  : undefined
        status: undefined      # if given, should be a list, which will get status for projects push'd as they are running.
        timeout : 7200         # timeout on any given migration -- actually leaves them running, but moves on...
        exclude : []
        loop  : 0              # if >=1; call it again with same inputs once it finishes
        cb    : undefined      # cb(err, {project_id:errors when migrating that project})

    projects = undefined
    errors   = {}
    done = 0
    fail = 0
    todo = undefined
    dbg = (m) -> winston.debug("#{new Date()} -- migrate2_bup_all: #{m}")
    t = misc.walltime()
    limit = 100000
    servers = {by_id:{}, by_dc:{}}

    async.series([
        (cb) ->
            connect_to_database(cb)
        (cb) ->
            dbg("get storage server information")
            database.select
                table     : "storage_servers"
                columns   : ['server_id', 'host', 'port', 'dc']
                where     : {dummy:true}
                objectify : true
                cb        : (err, results) =>
                    if err
                        cb(err)
                    else
                        for x in results
                            x.host = cassandra.inet_to_str(x.host)
                            v = servers.by_dc[x.dc]
                            if not v?
                                servers.by_dc[x.dc] = [x]
                            else
                                v.push(x)
                            servers.by_id[x.server_id] = x
                        cb()
        (cb) ->
            dbg("querying database...")
            database.select
                table   : 'projects'
                columns : ['project_id', 'last_edited', 'last_bup_save', 'last_migrate_bup_error', 'abuser', 'last_snapshot']
                jsonify : true
                limit   : limit
                cb      : (err, result) ->
                    if result?
                        dbg("got #{result.length} results in #{misc.walltime(t)} seconds")
                        result.sort (a,b) ->
                            if a.project_id < b.project_id
                                return -1
                            else if a.project_id > b.project_id
                                return 1
                            else
                                return 0

                        if opts.start? and opts.stop?
                            result = result.slice(opts.start, opts.stop)
                        else if opts.start?
                            result = result.slice(opts.start)
                        else if opts.stop?
                            result = result.slice(0, opts.stop)

                        dbg("filter out known abusers: before #{result.length}")
                        result = (x for x in result when not x.abuser?)
                        dbg("filter out known abusers: after #{result.length}")

                        dbg("filter out those that haven't ever had a snapshot: before #{result.length}")
                        result = (x for x in result when x.last_snapshot?)
                        dbg("filter out those that haven't ever had a snapshot: after #{result.length}")

                        dbg("filter out those that are already done: before #{result.length}")
                        v = []
                        for x in result
                            if not x.last_bup_save? or misc.len(x.last_bup_save) < 3
                                v.push(x)
                            else
                                for k, tm of x.last_bup_save
                                    if tm < x.last_edited
                                        v.push(x)
                                        break
                        projects = result
                        todo = projects.length
                        dbg("of these -- #{todo} in the range remain to be migrated")
                    cb(err)
        (cb) ->
            i = 1
            times = []
            start0 = misc.walltime()
            g = (i, cb) ->
                project = projects[i]
                project_id = project.project_id
                dbg("*******************************************")
                dbg("Starting to migrate #{project_id}: #{i+1}/#{todo}")
                dbg("*******************************************")
                start = misc.walltime()
                if opts.status?
                    stat = {status:'migrating...', project_id:project_id}
                    opts.status.push(stat)
                exports.migrate2_bup
                    project_id    : project_id
                    bup_last_save : project.bup_last_save
                    status        : stat
                    servers       : servers
                    cb            : (err) ->

                        tm = misc.walltime(start)
                        times.push(tm)
                        avg_time = times.reduce((t,s)->t+s)/times.length
                        eta_time = ((todo - times.length) * avg_time)/opts.limit

                        total_time = misc.walltime(start0)
                        avg_time2 = total_time / times.length
                        eta_time2 = (todo - times.length) * avg_time2

                        if err
                            if stat?
                                stat.status='failed'
                                stat.error = err
                                stat.walltime = tm
                            fail += 1
                        else
                            if stat?
                                stat.status='done'
                            done += 1
                        dbg("******************************************* ")
                        dbg("finished #{project_id} in #{tm} seconds     ")
                        dbg("MIGRATE_ALL (loop=#{opts.loop+1}, #{opts.limit} at once) STATUS: (success=#{done} + fail=#{fail} = #{done+fail})/#{todo}; #{todo-done-fail} left")
                        dbg("    total time     : #{total_time}")
                        dbg("    avg time per   : #{avg_time}s/each")
                        dbg("    eta if per     : #{eta_time/3600}h or #{eta_time/60}m")
                        dbg("    effective avg  : #{avg_time2}s/each")
                        dbg("    effective eta  : #{eta_time2/3600}h or #{eta_time2/60}m")
                        dbg("*******************************************")
                        if err
                            errors[project_id] = err
                        cb()
            f = (i, cb) ->
                h = () ->
                    dbg("timed out #{i}=#{projects[i]} after #{opts.timeout} seconds")
                    cb()
                timer = setTimeout(h, opts.timeout*1000)
                g i, () ->
                    clearTimeout(timer)
                    cb()

            async.mapLimit([0...projects.length], opts.limit, f, cb)
    ], (err) ->
        if opts.loop
            f = () =>
                opts.loop += 1
                exports.migrate_bup_all(opts)
            winston.debug("WAITING 90 seconds to space things out... before doing loop #{opts.loop+1}")
            setTimeout(f, 1000*90)
            return
        opts.cb?(err, errors)
    )

exports.migrate2_bup = (opts) ->
    opts = defaults opts,
        project_id    : required
        bup_last_save : {}
        status        : undefined
        servers       : required
        cb            : required
    dbg = (m) -> winston.debug("migrate2_bup(#{opts.project_id}): #{m}")
    dbg()

    needs_update = undefined
    last_migrate_bup = cassandra.now()
    host = undefined
    hosts = undefined
    client = undefined
    last_migrate_bup_error = undefined   # could be useful to know below...
    abuser = undefined
    lastmod = undefined
    hashrings = undefined
    targets = undefined

    dbg("determine bup_last_save")
    if misc.len(opts.bup_last_save) < 3
        # ensure have at least one from each dc
        dcs = ("#{servers.by_id[x].dc}" for x in misc.keys(opts.bup_last_save))
        for dc in ['0','1','2']
            if dc not in dcs
                opts.bup_last_save[misc.random_choice(servers.by_dc[dc])] = 0
    dbg("bup_last_save=#{misc.to_json(opts.bup_last_save)}")



    async.series([
        (cb) ->
            dbg("setting last_migrate_bup_error to start...")
            database.update
                table : 'projects'
                set   : {last_migrate_bup_error : "start"}
                where : {project_id : opts.project_id}
                cb    : cb
        (cb) ->
            if host?
                cb(); return
            dbg("get current location of project from database")
            get_current_location
                project_id : opts.project_id
                cb         : (err, x) ->
                    host = x
                    cb(err)
        (cb) ->
            dbg("get ordered list of hosts, based on newest snapshots")
            get_snapshots
                project_id : opts.project_id
                cb         : (err, snapshots) ->
                    # randomize so not all in same DC0...
                    v = ([snaps[0], Math.random(), host] for host, snaps of snapshots when snaps?.length >=1)
                    v.sort()
                    v.reverse()
                    v = (x[2] for x in v when x[2] != host)
                    if host?
                        v = [host].concat(v)
                    dbg("v = #{misc.to_json(v)}")
                    if v.length == 0
                        # nothing to do -- project never opened
                        hosts = undefined
                        cb()
                    else
                        hosts = v
                        cb()
        (cb) ->
            if not hosts?
                cb(); return
            dbg("project is available on #{hosts}")
            if opts.status?
                opts.status.hosts = hosts
            cb()


        (cb) ->
            if not hosts?
                cb(); return
            done = false
            targets = (servers.by_id[id].host for id in misc.keys(opts.bup_save_time))
            errors = {}
            f = (host, c) ->
                if done
                    c(); return
                dbg("run python script to migrate over from #{host} to #{targets.join(',')}...")
                execute_on
                    user        : 'root'
                    host        : targets[0]
                    command     : "/home/salvus/salvus/salvus/scripts/bup_storage.py migrate_remote #{host} --targets=#{targets.slice(1).join(',')} #{opts.project_id}"
                    timeout     : 3*60*60
                    err_on_exit : false
                    err_on_stderr : false
                    cb          : (err, output) ->
                        if err
                            errors[host] = err
                            c()
                        else
                            out = output.stdout + output.stderr
                            if out.indexOf('ABUSE') != -1
                                done = true
                                # mark as an abusive project
                                abuser = true
                            if out.indexOf('SUCCESS') != -1
                                done = true
                                c()
                            else
                                errors[host] = output.stderr
                                c()

            async.mapSeries hosts, f, (err) ->
                if not done
                    cb("unable to migrate from any host! -- #{misc.to_json(errors)}")
                else
                    cb()
        (cb) ->
            dbg("success -- record bup_last_save, etc.  in database")
            for k, v of opts.bup_last_save
                opts.bup_last_save[k] = last_migrate_bup
            database.update
                table : 'projects'
                set   : {bup_last_save:opts.bup_last_save, last_migrate_bup : last_migrate_bup,  last_migrate_bup_error:undefined, abuser:abuser}
                where : {project_id : opts.project_id}
                cb    : cb
    ], (err) =>
        if err
            database.update
                table : 'projects'
                set   : {last_migrate_bup_error : misc.to_json(err), last_migrate_bup : last_migrate_bup}
                where : {project_id : opts.project_id}
        opts.cb(err)
    )





