def _getChangeSet(self, chgSetList, recurse = True, withFiles = True,
                      withFileContents = True, target = None,
                      excludeAutoSource = False, primaryTroveList = None,
                      callback = None, forceLocalGeneration = False,
                      changesetVersion = None, mirrorMode = False):
        # This is a bit complicated due to servers not wanting to talk
        # to other servers. To make this work, we do this:
        #
        #   1. Split the list of change set requests into ones for
        #   remote servers (by server) and ones we need to generate
        #   locally
        #
        #   2. Get the changesets from the remote servers. This also
        #   gives us lists of other changesets we need (which need
        #   to be locally generated, or the repository server would
        #   have created them for us).
        #
        #   3. Create the local changesets. Doing this could well
        #   result in our needing changesets which we're better off
        #   generating on a server.
        #
        #   4. If more changesets are needed (from step 3) go to
        #   step 2.
        #
        #   5. Download any extra files (and create any extra diffs)
        #   which step 2 couldn't do for us.

        def _separateJobList(jobList, removedList, forceLocalGeneration,
                             mirrorMode):
            if forceLocalGeneration:
                return {}, jobList

            serverJobs = {}
            ourJobList = []
            for (troveName, (old, oldFlavor), (new, newFlavor), absolute) in \
                    jobList:
                if not new:
                    # XXX does doing this on the client get recursion right?
                    ourJobList.append((troveName, (old, oldFlavor),
                                       (new, newFlavor), absolute))
                    continue

                serverName = new.getHost()
                if old and mirrorMode and \
                            self.c[serverName].getProtocolVersion() < 49:
                    # old clients don't support mirrorMode argument; force
                    # local changeset generation (but only for relative
                    # change sets)
                    ourJobList.append((troveName, (old, oldFlavor),
                                       (new, newFlavor), absolute))
                elif old:
                    if self.c.singleServer(old, new):
                        l = serverJobs.setdefault(serverName, [])
                        l.append((troveName,
                                  (self.fromVersion(old),
                                   self.fromFlavor(oldFlavor)),
                                  (self.fromVersion(new),
                                   self.fromFlavor(newFlavor)),
                                  absolute))
                    else:
                        ourJobList.append((troveName, (old, oldFlavor),
                                           (new, newFlavor), absolute))
                else:
                    l = serverJobs.setdefault(serverName, [])
                    l.append((troveName,
                              (0, 0),
                              (self.fromVersion(new),
                               self.fromFlavor(newFlavor)),
                              absolute))

            ourJobList += removedList

            return (serverJobs, ourJobList)

        def _getLocalTroves(troveList):
            if not self.localRep or not troveList:
                return [ None ] * len(troveList)

            return self.localRep.getTroves(troveList, pristine=True)

        def _getCsFromRepos(target, cs, server, job, recurse,
                            withFiles, withFileContents,
                            excludeAutoSource, filesNeeded,
                            chgSetList, removedList, changesetVersion,
                            mirrorMode):
            if callback:
                callback.requestingChangeSet()
            server.setAbortCheck(None)
            args = (job, recurse, withFiles, withFileContents,
                    excludeAutoSource)
            kwargs = {}
            serverVersion = server.getProtocolVersion()

            if mirrorMode and serverVersion >= 49:
                if not changesetVersion:
                    changesetVersion = \
                        filecontainer.FILE_CONTAINER_VERSION_LATEST

                args += (changesetVersion, mirrorMode, )
            elif changesetVersion and serverVersion > 47:
                args += (changesetVersion, )

            # seek to the end of the file
            outFile.seek(0, 2)
            start = resume = outFile.tell()
            attempts = max(1, self.cfg.downloadAttempts)
            while attempts > 0:
                if resume - start:
                    assert serverVersion >= 73
                    outFile.seek(resume)
                    kwargs['resumeOffset'] = resume - start
                    if callback:
                        callback.warning("Changeset download was interrupted. "
                                "Attempting to resume where it left off.")
                try:
                    (sizes, extraTroveList, extraFileList, removedTroveList,
                            extra,) = _getCsOnce(serverVersion, args, kwargs)
                    break
                except errors.TruncatedResponseError:
                    attempts -= 1
                    if not attempts or serverVersion < 73:
                        raise
                    # Figure out how many bytes were downloaded, then trim off
                    # a bit to ensure any garbage (e.g. a proxy error page) is
                    # discarded.
                    keep = max(resume, outFile.tell() -
                            self.cfg.downloadRetryTrim)
                    if self.cfg.downloadRetryTrim and (
                            keep - resume > self.cfg.downloadRetryThreshold):
                        attempts = max(1, self.cfg.downloadAttempts)
                    resume = keep

            chgSetList += self.toJobList(extraTroveList)
            filesNeeded.update(self.toFilesNeeded(extraFileList))
            removedList += self.toJobList(removedTroveList)

            for size in sizes:
                f = util.SeekableNestedFile(outFile, size, start)
                try:
                    newCs = changeset.ChangeSetFromFile(f)
                except IOError, err:
                    assert False, 'IOError in changeset (%s); args = %r' % (
                            str(err), args,)
                if not cs:
                    cs = newCs
                else:
                    cs.merge(newCs)
                start += size

            return (cs, self.toJobList(extraTroveList),
                    self.toFilesNeeded(extraFileList))

        def _getCsOnce(serverVersion, args, kwargs):
            l = server.getChangeSet(*args, **kwargs)
            extra = {}
            if serverVersion >= 50:
                url = l[0]
                sizes = [ x[0] for x in l[1] ]
                extraTroveList = [ x for x in itertools.chain(
                                    *[ x[1] for x in l[1] ] ) ]
                extraFileList = [ x for x in itertools.chain(
                                    *[ x[2] for x in l[1] ] ) ]
                removedTroveList = [ x for x in itertools.chain(
                                    *[ x[3] for x in l[1] ] ) ]
                if serverVersion >= 73:
                    extra = l[2]
            elif serverVersion < 38:
                (url, sizes, extraTroveList, extraFileList) = l
                removedTroveList = []
            else:
                (url, sizes, extraTroveList,
                 extraFileList, removedTroveList) = l
            # ensure that sizes are integers.  protocol version 44 and
            # later sends them as strings instead of ints due to the 2
            # GiB limitation
            sizes = [ int(x) for x in sizes ]

            if hasattr(url, 'read'):
                # Nested changeset file in a multi-part response
                inF = url
            elif os.path.exists(url):
                # attempt to remove temporary local files
                # possibly created by a shim client
                inF = open(url, 'rb')
                try:
                    os.unlink(url)
                except OSError, err:
                    if err.args[0] != errno.EPERM:
                        raise
            else:
                # "forceProxy" here makes sure that multi-part requests go back
                # through the same proxy on subsequent requests.
                forceProxy = server.usedProxy()
                headers = [('X-Conary-Servername', server._serverName)]
                try:
                    inF = transport.ConaryURLOpener(proxyMap=self.c.proxyMap
                            ).open(url, forceProxy=forceProxy, headers=headers)
                except transport.TransportError, e:
                    raise errors.RepositoryError(str(e))

            if callback:
                wrapper = callbacks.CallbackRateWrapper(
                    callback, callback.downloadingChangeSet,
                    sum(sizes))
                copyCallback = wrapper.callback
                abortCheck = callback.checkAbort
            else:
                copyCallback = None
                abortCheck = None

            resumeOffset = kwargs.get('resumeOffset') or 0
            # Start the total at resumeOffset so that progress callbacks
            # continue where they left off.
            copied = util.copyfileobj(inF, outFile, callback=copyCallback,
                    abortCheck=abortCheck, rateLimit=self.downloadRateLimit,
                    total=resumeOffset)
            if copied is None:
                raise errors.RepositoryError("Unknown error downloading changeset")
            totalSize = copied + resumeOffset
            if hasattr(inF, 'headers') and 'content-length' in inF.headers:
                expectSize = resumeOffset + long(inF.headers['content-length'])
                if totalSize != expectSize:
                    raise errors.TruncatedResponseError(expectSize, totalSize)
                assert sum(sizes) == expectSize
            elif totalSize != sum(sizes):
                raise errors.TruncatedResponseError(sum(sizes), totalSize)
            inF.close()

            return (sizes, extraTroveList, extraFileList, removedTroveList,
                    extra)

        def _getCsFromShim(target, cs, server, job, recurse, withFiles,
                           withFileContents, excludeAutoSource,
                           filesNeeded, chgSetList, removedList):
            (newCs, extraTroveList, extraFileList, removedList) = \
                  server.getChangeSetObj(job, recurse,
                                         withFiles, withFileContents,
                                         excludeAutoSource)
            if not cs:
                cs = newCs
            else:
                cs.merge(newCs)
            return cs, extraTroveList, extraFileList

        if not chgSetList:
            # no need to work hard to find this out
            return changeset.ReadOnlyChangeSet()

        # make sure the absolute flag isn't set for any differential change
        # sets
        assert(not [ x for x in chgSetList if (x[1][0] and x[-1]) ])

        cs = None
        internalCs = None
        filesNeeded = set()
        removedList = []

        if target:
            try:
                outFile = util.ExtendedFile(target, "w+", buffering = False)
            except __HOLE__, e:
                strerr = "Error writing to file %s: %s" % (e.filename,
                    e.strerror)
                raise errors.FilesystemError(e.errno, e.filename, e.strerror,
                    strerr)
        else:
            (outFd, tmpName) = util.mkstemp(suffix = '.ccs')
            outFile = util.ExtendedFile(tmpName, "w+", buffering = False)
            os.close(outFd)
            os.unlink(tmpName)

        if primaryTroveList is None:
            # (name, version, release) list. removed troves aren't primary
            primaryTroveList = [ (x[0], x[2][0], x[2][1]) for x in chgSetList
                                        if x[2][0] is not None ]

        while chgSetList or removedList:
            (serverJobs, ourJobList) = _separateJobList(chgSetList,
                                                        removedList,
                                                        forceLocalGeneration,
                                                        mirrorMode)

            chgSetList = []
            removedList = []

            for serverName, job in serverJobs.iteritems():
                server = self.c[serverName]
                args = (target, cs, server, job, recurse, withFiles,
                        withFileContents, excludeAutoSource,
                        filesNeeded, chgSetList, removedList)

                try:
                    if server.__class__ == ServerProxy:
                        # this is a XML-RPC proxy for a remote repository
                        rc = _getCsFromRepos(*(args + (changesetVersion,
                                                       mirrorMode)))
                    else:
                        # assume we are a shim repository
                        rc = _getCsFromShim(*args)
                    cs, extraTroveList, extraFileList = rc
                except Exception:
                    if target and os.path.exists(target):
                        os.unlink(target)
                    elif os.path.exists(tmpName):
                        os.unlink(tmpName)
                    raise

                chgSetList += extraTroveList
                filesNeeded.update(extraFileList)

            if (ourJobList or filesNeeded) and not internalCs:
                internalCs = changeset.ChangeSet()

            # Handle everything in ourJobList which is just a deletion. We
            # need timestamped versions for this; only go the repository
            # to get those if the ones we have are not versioned.
            delList = []
            timesNeeded = [ ]
            for i, (troveName, (oldVersion, oldFlavor),
                  (newVersion, newFlavor), absolute) in enumerate(ourJobList):
                if not newVersion:
                    delList.append(((troveName, oldVersion, oldFlavor), i))
                    if not sum(oldVersion.timeStamps()):
                        timesNeeded.append(delList[-1])

            # XXX this is an expensive way to get a version w/ timestamps, but
            # it's easier than other approaches :-(
            trvs = self.getTroves([ x[0] for x in timesNeeded ],
                                  withFiles = False)
            timeDict = dict(zip([ x[0] for x in timesNeeded ],
                                [ x.getVersion() for x in trvs ]))

            # this lets us remove from ourJobList from back to front, keeping
            # our indices valid
            delList.reverse()

            for trvInfo, i in delList:
                ver = timeDict.get(trvInfo, trvInfo[1])
                internalCs.oldTrove(trvInfo[0], ver, trvInfo[2])
                del ourJobList[i]
            del delList

            # generate this change set, and put any recursive generation
            # which is needed onto the chgSetList for the next pass
            allTrovesNeeded = []
            for (troveName, (oldVersion, oldFlavor),
                            (newVersion, newFlavor), absolute) in ourJobList:
                if oldVersion is not None:
                    allTrovesNeeded.append((troveName, oldVersion, oldFlavor))
                allTrovesNeeded.append((troveName, newVersion, newFlavor))

            troves = _getLocalTroves(allTrovesNeeded)
            remoteTrovesNeeded = []
            indices = []
            for i, (trove, req) in enumerate(zip(troves, allTrovesNeeded)):
                # don't ask for local troves from a remote server
                if trove is None and not req[1].isOnLocalHost():
                    remoteTrovesNeeded.append(req)
                    indices.append(i)

            remoteTroves = self.getTroves(remoteTrovesNeeded)
            for i, trove in zip(indices, remoteTroves):
                troves[i] = trove

            del allTrovesNeeded, remoteTrovesNeeded, indices, remoteTroves

            i = 0
            for (troveName, (oldVersion, oldFlavor),
                            (newVersion, newFlavor), absolute) in ourJobList:
                if oldVersion is not None:
                    old = troves[i]
                    i += 1
                else:
                    old = None

                new = troves[i]
                i += 1

                # if the old version is marked removed, pretend as though
                # it doesn't exist.
                if old and old.isRemoved():
                    old = None
                (troveChgSet, newFilesNeeded, pkgsNeeded) = \
                                new.diff(old, absolute = absolute)
                # newFilesNeeded = [ (pathId, oldFileVersion, newFileVersion) ]
                filesNeeded.update( ( (x[0], troveName,
                        (oldVersion, oldFlavor, x[1], x[2]),
                        (newVersion, newFlavor, x[3], x[4]))
                            for x in newFilesNeeded ) )

                if recurse:
                    for (otherTroveName, (otherOldVersion, otherOldFlavor),
                                         (otherNewVersion, otherNewFlavor),
                         otherIsAbsolute) in pkgsNeeded:
                        chgSetList.append((otherTroveName,
                                           (otherOldVersion, otherOldFlavor),
                                           (otherNewVersion, otherNewFlavor),
                                           absolute))

                internalCs.newTrove(troveChgSet)

        # Files that are missing from upstream
        missingFiles = []

        if withFiles and filesNeeded:
            need = []
            for (pathId, troveName,
                (oldTroveVersion, oldTroveFlavor, oldFileId, oldFileVersion),
                (newTroveVersion, newTroveFlavor, newFileId, newFileVersion)) \
                                in filesNeeded:
                if oldFileVersion:
                    need.append((pathId, oldFileId, oldFileVersion))
                need.append((pathId, newFileId, newFileVersion))

            # If a callback was passed in, then allow for missing files
            fileObjs = self.getFileVersions(need, lookInLocal = True,
                                            allowMissingFiles = bool(callback))
            fileDict = {}
            for ((pathId, fileId, fileVersion), fileObj, error) in zip(
                    need, fileObjs, fileObjs.errors):
                fileDict[(pathId, fileId)] = fileObj, error
            del fileObj, fileObjs, need, fileId

            contentsNeeded = []
            fileJob = []

            for (pathId, troveName,
                    (oldTroveVersion, oldTroveF, oldFileId, oldFileVersion),
                    (newTroveVersion, newTroveF, newFileId, newFileVersion)) \
                                in filesNeeded:
                if oldFileVersion:
                    oldFileObj, _ = fileDict[(pathId, oldFileId)]
                else:
                    oldFileObj = None

                newFileObj, newFileError = fileDict[(pathId, newFileId)]
                if newFileObj is None:
                    # File missing from server
                    missingFiles.append((troveName, newTroveVersion, newTroveF,
                        newFileError, pathId, newFileId, newFileVersion))
                    continue

                forceAbsolute = mirrorMode and oldFileObj and (
                        oldFileId != newFileId
                        or oldFileVersion.getHost() != newFileVersion.getHost()
                        )
                if forceAbsolute:
                    (filecs, hash) = changeset.fileChangeSet(pathId,
                                                             None,
                                                             newFileObj)
                else:
                    (filecs, hash) = changeset.fileChangeSet(pathId,
                                                             oldFileObj,
                                                             newFileObj)

                internalCs.addFile(oldFileId, newFileId, filecs)

                if not withFileContents:
                    continue
                if excludeAutoSource and newFileObj.flags.isAutoSource():
                    continue

                if hash or (forceAbsolute and newFileObj.hasContents):
                    # pull contents from the trove it was originally
                    # built in
                    fetchItems = []
                    needItems = []

                    if (not mirrorMode and
                                    changeset.fileContentsUseDiff(oldFileObj,
                                                                  newFileObj)):
                        fetchItems.append( (oldFileId, oldFileVersion,
                                            oldFileObj) )
                        needItems.append( (pathId, None, oldFileObj) )

                    if not newFileObj.flags.isEncapsulatedContent():
                        fetchItems.append( (newFileId, newFileVersion, newFileObj) )
                        contentsNeeded += fetchItems

                        needItems.append( (pathId, newFileId, newFileObj) )
                        fileJob.extend([ needItems ])

            contentList = self.getFileContents(contentsNeeded,
                                               tmpFile = outFile,
                                               lookInLocal = True,
                                               callback = callback,
                                               compressed = True)

            i = 0
            for item in fileJob:
                pathId, fileId, fileObj = item[0]
                contents = contentList[i]
                i += 1

                if len(item) == 1:
                    internalCs.addFileContents(pathId, fileId,
                                   changeset.ChangedFileTypes.file,
                                   contents,
                                   fileObj.flags.isConfig(),
                                   compressed = True)
                else:
                    # Don't bother with diffs. Clients can reconstruct them for
                    # installs and they're just a pain to assemble here anyway.
                    fileId = item[1][1]
                    newFileObj = item[1][2]
                    newContents = contentList[i]
                    i += 1

                    (contType, cont) = changeset.fileContentsDiff(None,
                                            None, newFileObj, newContents,
                                            mirrorMode = mirrorMode)
                    internalCs.addFileContents(pathId, fileId, contType,
                                               cont, True,
                                               compressed = True)

        if not cs and internalCs:
            cs = internalCs
            internalCs = None
        elif cs and internalCs:
            cs.merge(internalCs)

        # convert the versions in here to ones w/ timestamps
        cs.setPrimaryTroveList([])
        oldTroveSet = dict([ (x,x) for x in cs.getOldTroveList() ] )
        for (name, version, flavor) in primaryTroveList:
            if cs.hasNewTrove(name, version, flavor):
                trove = cs.getNewTroveVersion(name, version, flavor)
                cs.addPrimaryTrove(name, trove.getNewVersion(), flavor)
            else:
                cs.addPrimaryTrove(*oldTroveSet[(name, version,flavor)])

        if missingFiles:
            mfs = []
            for mf in missingFiles:
                trvName, trvVersion, trvFlavor, error = mf[:4]
                trv = cs.getNewTroveVersion(trvName, trvVersion, trvFlavor)
                # Find the file path associated with this missing file
                for pathId, path, fileId, version in trv.getNewFileList():
                    if (pathId, fileId, version) == mf[4:]:
                        break
                else: # for
                    # Unable to find this file
                    raise Exception("Cannot find file in changeset")
                mfs.append((trvName, trvVersion, trvFlavor,
                            pathId, path, fileId, version, error))

            # The test for the presence of the callback is redundant, if we
            # have missing files we should have a callback, otherwise
            # getFileVersions would have raised an exception because of the
            # allowMissingFiles flag.
            ret = False
            if callback:
                assert(hasattr(callback, 'missingFiles'))
                ret = callback.missingFiles(mfs)

            # If the callback returns False, or no callback is present,
            # keep the old behavior of raising the exception
            # Note that the callback can choose to raise an exception itself,
            # in which case this code will not get executed.
            if not ret:
                # Grab just the first file
                mf = mfs[0]
                fileId, error = mf[5], mf[7]
                if error:
                    error.throw()
                else:
                    raise errors.FileStreamMissing(fileId)

        if target and cs:
            if cs.oldTroves or cs.newTroves:
                os.unlink(target)
                cs.writeToFile(target, versionOverride = changesetVersion)

            cs = None
        elif target:
            os.unlink(target)



        return cs
dataset/ETHPy150Open sassoftware/conary/conary/repository/netclient.py/NetworkRepositoryClient._getChangeSet