mergin/client_pull.py
def pull_project_async(mc, directory):
    """
    Starts project pull in background and returns handle to the pending job.
    Using that object it is possible to watch progress or cancel the ongoing work.
    """

    mp = MerginProject(directory)
    if mp.has_unfinished_pull():
        try:
            mp.resolve_unfinished_pull(mc.username())
        except ClientError as err:
            mp.log.error("Error resolving unfinished pull: " + str(err))
            mp.log.info("--- pull aborted")
            raise

    project_path = mp.metadata["name"]
    local_version = mp.metadata["version"]

    mp.log.info("--- version: " + mc.user_agent_info())
    mp.log.info(f"--- start pull {project_path}")

    try:
        server_info = mc.project_info(project_path, since=local_version)
    except ClientError as err:
        mp.log.error("Error getting project info: " + str(err))
        mp.log.info("--- pull aborted")
        raise
    server_version = server_info["version"]

    mp.log.info(f"got project info: local version {local_version} / server version {server_version}")

    if local_version == server_version:
        mp.log.info("--- pull - nothing to do (already at server version)")
        return  # Project is up to date

    # we either download a versioned file using diffs (strongly preferred),
    # but if we don't have history with diffs (e.g. uploaded without diffs)
    # then we just download the whole file
    _pulling_file_with_diffs = lambda f: 'diffs' in f and len(f['diffs']) != 0

    temp_dir = mp.fpath_meta(f'fetch_{local_version}-{server_version}')
    os.makedirs(temp_dir, exist_ok=True)
    pull_changes = mp.get_pull_changes(server_info["files"])
    mp.log.debug("pull changes:\n" + pprint.pformat(pull_changes))
    fetch_files = []
    for f in pull_changes["added"]:
        f['version'] = server_version
        fetch_files.append(f)
    # extend fetch files download list with various version of diff files (if needed)
    for f in pull_changes["updated"]:
        if _pulling_file_with_diffs(f):
            for diff in f['diffs']:
                diff_file = copy.deepcopy(f)
                for k, v in f['history'].items():
                    if 'diff' not in v:
                        continue
                    if diff == v['diff']['path']:
                        diff_file['version'] = k
                        diff_file['diff'] = v['diff']
                fetch_files.append(diff_file)
        else:
            f['version'] = server_version
            fetch_files.append(f)

    files_to_merge = []   # list of FileToMerge instances

    for file in fetch_files:
        diff_only = _pulling_file_with_diffs(file)
        items = _download_items(file, temp_dir, diff_only)

        # figure out destination path for the file
        file_dir = os.path.dirname(os.path.normpath(os.path.join(temp_dir, file['path'])))
        basename = os.path.basename(file['diff']['path']) if diff_only else os.path.basename(file['path'])
        dest_file_path = os.path.join(file_dir, basename)
        os.makedirs(file_dir, exist_ok=True)
        files_to_merge.append( FileToMerge(dest_file_path, items) )


    # make sure we can update geodiff reference files (aka. basefiles) with diffs or
    # download their full versions so we have them up-to-date for applying changes
    basefiles_to_patch = []  # list of tuples (relative path within project, list of diff files in temp dir to apply)
    for file in pull_changes['updated']:
        if not _pulling_file_with_diffs(file):
            continue  # this is only for diffable files (e.g. geopackages)

        basefile = mp.fpath_meta(file['path'])
        if not os.path.exists(basefile):
            # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file
            # or we removed it within previous pull because we failed to apply patch the older version for some reason).
            # But it's not a problem - we will download the newest version and we're sorted.
            file_path = file['path']
            mp.log.info(f"missing base file for {file_path} -> going to download it (version {server_version})")
            file['version'] = server_version
            items = _download_items(file, temp_dir, diff_only=False)
            dest_file_path = mp.fpath(file["path"], temp_dir)
            #dest_file_path = os.path.join(os.path.dirname(os.path.normpath(os.path.join(temp_dir, file['path']))), os.path.basename(file['path']))
            files_to_merge.append( FileToMerge(dest_file_path, items) )
            continue

        basefiles_to_patch.append( (file['path'], file['diffs']) )

    # make a single list of items to download
    total_size = 0
    download_list = []
    for file_to_merge in files_to_merge:
        download_list.extend(file_to_merge.downloaded_items)
        for item in file_to_merge.downloaded_items:
            total_size += item.size

    mp.log.info(f"will download {len(download_list)} chunks, total size {total_size}")

    job = PullJob(project_path, pull_changes, total_size, server_version, files_to_merge, download_list, temp_dir, mp, server_info, basefiles_to_patch, mc.username())

    # start download
    job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
    job.futures = []
    for item in download_list:
        future = job.executor.submit(_do_download, item, mc, mp, project_path, job)
        job.futures.append(future)

    return job

    Starts project pull in background and returns handle to the pending job.
    Using that object it is possible to watch progress or cancel the ongoing work.
    
Starts project pull in background and returns handle to the pending job.
Using that object it is possible to watch progress or cancel the ongoing work.
Starts project pull in background and returns handle to the pending job . Using that object it is possible to watch progress or cancel the ongoing work .
def pull_project_async(mc, directory):
    mp = MerginProject(directory)
    if mp.has_unfinished_pull():
        try:
            mp.resolve_unfinished_pull(mc.username())
        except ClientError as err:
            mp.log.error("Error resolving unfinished pull: " + str(err))
            mp.log.info("--- pull aborted")
            raise
    project_path = mp.metadata["name"]
    local_version = mp.metadata["version"]
    mp.log.info("--- version: " + mc.user_agent_info())
    mp.log.info(f"--- start pull {project_path}")
    try:
        server_info = mc.project_info(project_path, since=local_version)
    except ClientError as err:
        mp.log.error("Error getting project info: " + str(err))
        mp.log.info("--- pull aborted")
        raise
    server_version = server_info["version"]
    mp.log.info(f"got project info: local version {local_version} / server version {server_version}")
    if local_version == server_version:
        mp.log.info("--- pull - nothing to do (already at server version)")
        return  
    _pulling_file_with_diffs = lambda f: 'diffs' in f and len(f['diffs']) != 0
    temp_dir = mp.fpath_meta(f'fetch_{local_version}-{server_version}')
    os.makedirs(temp_dir, exist_ok=True)
    pull_changes = mp.get_pull_changes(server_info["files"])
    mp.log.debug("pull changes:\n" + pprint.pformat(pull_changes))
    fetch_files = []
    for f in pull_changes["added"]:
        f['version'] = server_version
        fetch_files.append(f)
    for f in pull_changes["updated"]:
        if _pulling_file_with_diffs(f):
            for diff in f['diffs']:
                diff_file = copy.deepcopy(f)
                for k, v in f['history'].items():
                    if 'diff' not in v:
                        continue
                    if diff == v['diff']['path']:
                        diff_file['version'] = k
                        diff_file['diff'] = v['diff']
                fetch_files.append(diff_file)
        else:
            f['version'] = server_version
            fetch_files.append(f)
    files_to_merge = []   
    for file in fetch_files:
        diff_only = _pulling_file_with_diffs(file)
        items = _download_items(file, temp_dir, diff_only)
        file_dir = os.path.dirname(os.path.normpath(os.path.join(temp_dir, file['path'])))
        basename = os.path.basename(file['diff']['path']) if diff_only else os.path.basename(file['path'])
        dest_file_path = os.path.join(file_dir, basename)
        os.makedirs(file_dir, exist_ok=True)
        files_to_merge.append( FileToMerge(dest_file_path, items) )
    basefiles_to_patch = []  
    for file in pull_changes['updated']:
        if not _pulling_file_with_diffs(file):
            continue  
        basefile = mp.fpath_meta(file['path'])
        if not os.path.exists(basefile):
            file_path = file['path']
            mp.log.info(f"missing base file for {file_path} -> going to download it (version {server_version})")
            file['version'] = server_version
            items = _download_items(file, temp_dir, diff_only=False)
            dest_file_path = mp.fpath(file["path"], temp_dir)
            files_to_merge.append( FileToMerge(dest_file_path, items) )
            continue
        basefiles_to_patch.append( (file['path'], file['diffs']) )
    total_size = 0
    download_list = []
    for file_to_merge in files_to_merge:
        download_list.extend(file_to_merge.downloaded_items)
        for item in file_to_merge.downloaded_items:
            total_size += item.size
    mp.log.info(f"will download {len(download_list)} chunks, total size {total_size}")
    job = PullJob(project_path, pull_changes, total_size, server_version, files_to_merge, download_list, temp_dir, mp, server_info, basefiles_to_patch, mc.username())
    job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
    job.futures = []
    for item in download_list:
        future = job.executor.submit(_do_download, item, mc, mp, project_path, job)
        job.futures.append(future)
    return job
def pull_project_async ( mc , directory ) : mp = MerginProject ( directory ) if mp . has_unfinished_pull ( ) : try : mp . resolve_unfinished_pull ( mc . username ( ) ) except ClientError as err : mp . log . error ( "Error resolving unfinished pull: " + str ( err ) ) mp . log . info ( "--- pull aborted" ) raise project_path = mp . metadata [ "name" ] local_version = mp . metadata [ "version" ] mp . log . info ( "--- version: " + mc . user_agent_info ( ) ) mp . log . info ( f"--- start pull {project_path}" ) try : server_info = mc . project_info ( project_path , since = local_version ) except ClientError as err : mp . log . error ( "Error getting project info: " + str ( err ) ) mp . log . info ( "--- pull aborted" ) raise server_version = server_info [ "version" ] mp . log . info ( f"got project info: local version {local_version} / server version {server_version}" ) if local_version == server_version : mp . log . info ( "--- pull - nothing to do (already at server version)" ) return _pulling_file_with_diffs = lambda f : 'diffs' in f and len ( f [ 'diffs' ] ) != 0 temp_dir = mp . fpath_meta ( f'fetch_{local_version}-{server_version}' ) os . makedirs ( temp_dir , exist_ok = True ) pull_changes = mp . get_pull_changes ( server_info [ "files" ] ) mp . log . debug ( "pull changes:\n" + pprint . pformat ( pull_changes ) ) fetch_files = [ ] for f in pull_changes [ "added" ] : f [ 'version' ] = server_version fetch_files . append ( f ) for f in pull_changes [ "updated" ] : if _pulling_file_with_diffs ( f ) : for diff in f [ 'diffs' ] : diff_file = copy . deepcopy ( f ) for k , v in f [ 'history' ] . items ( ) : if 'diff' not in v : continue if diff == v [ 'diff' ] [ 'path' ] : diff_file [ 'version' ] = k diff_file [ 'diff' ] = v [ 'diff' ] fetch_files . append ( diff_file ) else : f [ 'version' ] = server_version fetch_files . append ( f ) files_to_merge = [ ] for file in fetch_files : diff_only = _pulling_file_with_diffs ( file ) items = _download_items ( file , temp_dir , diff_only ) file_dir = os . path . dirname ( os . path . normpath ( os . path . join ( temp_dir , file [ 'path' ] ) ) ) basename = os . path . basename ( file [ 'diff' ] [ 'path' ] ) if diff_only else os . path . basename ( file [ 'path' ] ) dest_file_path = os . path . join ( file_dir , basename ) os . makedirs ( file_dir , exist_ok = True ) files_to_merge . append ( FileToMerge ( dest_file_path , items ) ) basefiles_to_patch = [ ] for file in pull_changes [ 'updated' ] : if not _pulling_file_with_diffs ( file ) : continue basefile = mp . fpath_meta ( file [ 'path' ] ) if not os . path . exists ( basefile ) : file_path = file [ 'path' ] mp . log . info ( f"missing base file for {file_path} -> going to download it (version {server_version})" ) file [ 'version' ] = server_version items = _download_items ( file , temp_dir , diff_only = False ) dest_file_path = mp . fpath ( file [ "path" ] , temp_dir ) files_to_merge . append ( FileToMerge ( dest_file_path , items ) ) continue basefiles_to_patch . append ( ( file [ 'path' ] , file [ 'diffs' ] ) ) total_size = 0 download_list = [ ] for file_to_merge in files_to_merge : download_list . extend ( file_to_merge . downloaded_items ) for item in file_to_merge . downloaded_items : total_size += item . size mp . log . info ( f"will download {len(download_list)} chunks, total size {total_size}" ) job = PullJob ( project_path , pull_changes , total_size , server_version , files_to_merge , download_list , temp_dir , mp , server_info , basefiles_to_patch , mc . username ( ) ) job . executor = concurrent . futures . ThreadPoolExecutor ( max_workers = 4 ) job . futures = [ ] for item in download_list : future = job . executor . submit ( _do_download , item , mc , mp , project_path , job ) job . futures . append ( future ) return job
Starts project pull in background and returns handle to the pending job.
Starts project pull in background and returns handle to the pending job .
"""
    Starts project pull in background and returns handle to the pending job.
    Using that object it is possible to watch progress or cancel the ongoing work.
    """ # Project is up to date # we either download a versioned file using diffs (strongly preferred), # but if we don't have history with diffs (e.g. uploaded without diffs) # then we just download the whole file # extend fetch files download list with various version of diff files (if needed) # list of FileToMerge instances # figure out destination path for the file # make sure we can update geodiff reference files (aka. basefiles) with diffs or # download their full versions so we have them up-to-date for applying changes # list of tuples (relative path within project, list of diff files in temp dir to apply) # this is only for diffable files (e.g. geopackages) # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file # or we removed it within previous pull because we failed to apply patch the older version for some reason). # But it's not a problem - we will download the newest version and we're sorted. #dest_file_path = os.path.join(os.path.dirname(os.path.normpath(os.path.join(temp_dir, file['path']))), os.path.basename(file['path'])) # make a single list of items to download # start download
mc
directory