perceval/backends/core/mediawiki.py
def __fetch_pre1_27(self, from_date=None):
        """Fetch the pages from the backend url.

        The method retrieves, from a MediaWiki url, the
        wiki pages.

        :returns: a generator of pages
        """

        def fetch_incremental_changes(namespaces_contents):
            # Use recent changes API to get the pages from date
            npages = 0  # number of pages processed
            tpages = 0  # number of total pages
            pages_done = []  # pages already retrieved in reviews API

            rccontinue = ''
            hole_created = True  # To detect that incremental is not complete
            while rccontinue is not None:
                raw_pages = self.client.get_recent_pages(namespaces_contents, rccontinue)
                data_json = json.loads(raw_pages)

                if 'query-continue' in data_json:
                    # < 1.27
                    rccontinue = data_json['query-continue']['recentchanges']['rccontinue']
                elif 'continue' in data_json:
                    # >= 1.27
                    rccontinue = data_json['continue']['rccontinue']
                else:
                    rccontinue = None

                pages_json = data_json['query']['recentchanges']
                for page in pages_json:

                    page_ts = dateutil.parser.parse(page['timestamp'])
                    if from_date >= page_ts:
                        # The rest of recent changes are older than from_date
                        logger.debug("All recent changes newer than %s processed.", from_date)
                        rccontinue = None
                        hole_created = False
                        break

                    if page['pageid'] in pages_done:
                        logger.debug("Page %s already processed; skipped", page['pageid'])
                        continue

                    tpages += 1
                    pages_done.append(page['pageid'])
                    page_reviews = self.__get_page_reviews(page)

                    if not page_reviews:
                        logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                       page['title'], page['pageid'])
                        continue

                    yield page_reviews
                    npages += 1
            if hole_created:
                logger.error("Incremental update NOT completed. Hole in history created.")
            logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)

        def fetch_all_pages(namespaces_contents):
            # Use get all pages API to get pages
            npages = 0  # number of pages processed
            tpages = 0  # number of total pages
            pages_done = []  # pages already retrieved in reviews API

            for ns in namespaces_contents:
                apcontinue = ''  # pagination for getting pages
                logger.debug("Getting pages for namespace: %s", ns)
                while apcontinue is not None:
                    raw_pages = self.client.get_pages(ns, apcontinue)
                    data_json = json.loads(raw_pages)
                    if 'query-continue' in data_json:
                        # < 1.27
                        apcontinue = data_json['query-continue']['allpages']['apcontinue']
                    elif 'continue' in data_json:
                        # >= 1.27
                        apcontinue = data_json['continue']['apcontinue']
                    else:
                        apcontinue = None
                    pages_json = data_json['query']['allpages']
                    for page in pages_json:

                        if page['pageid'] in pages_done:
                            logger.debug("Page %s already processed; skipped", page['pageid'])
                            continue

                        tpages += 1
                        pages_done.append(page['pageid'])
                        page_reviews = self.__get_page_reviews(page)

                        if not page_reviews:
                            logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                           page['title'], page['pageid'])
                            continue

                        yield page_reviews
                        npages += 1
            logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)

        logger.info("Looking for pages at url '%s'", self.url)

        # from_date can not be older than MAX_RECENT_DAYS days ago
        if from_date:
            if (datetime_utcnow() - from_date).days >= MAX_RECENT_DAYS:
                cause = "Can't get incremental pages older than %i days." % MAX_RECENT_DAYS
                cause += " Do a complete analysis without from_date for older changes."
                raise BackendError(cause=cause)

        namespaces_contents = self.__get_namespaces_contents()

        if not from_date:
            return fetch_all_pages(namespaces_contents)
        else:
            return fetch_incremental_changes(namespaces_contents)
def __fetch_pre1_27(self, from_date=None):
        """Fetch the pages from the backend url.

        The method retrieves, from a MediaWiki url, the
        wiki pages.

        :returns: a generator of pages
        """

        def fetch_incremental_changes(namespaces_contents):
            # Use recent changes API to get the pages from date
            npages = 0  # number of pages processed
            tpages = 0  # number of total pages
            pages_done = []  # pages already retrieved in reviews API

            rccontinue = ''
            hole_created = True  # To detect that incremental is not complete
            while rccontinue is not None:
                raw_pages = self.client.get_recent_pages(namespaces_contents, rccontinue)
                data_json = json.loads(raw_pages)

                if 'query-continue' in data_json:
                    # < 1.27
                    rccontinue = data_json['query-continue']['recentchanges']['rccontinue']
                elif 'continue' in data_json:
                    # >= 1.27
                    rccontinue = data_json['continue']['rccontinue']
                else:
                    rccontinue = None

                pages_json = data_json['query']['recentchanges']
                for page in pages_json:

                    page_ts = dateutil.parser.parse(page['timestamp'])
                    if from_date >= page_ts:
                        # The rest of recent changes are older than from_date
                        logger.debug("All recent changes newer than %s processed.", from_date)
                        rccontinue = None
                        hole_created = False
                        break

                    if page['pageid'] in pages_done:
                        logger.debug("Page %s already processed; skipped", page['pageid'])
                        continue

                    tpages += 1
                    pages_done.append(page['pageid'])
                    page_reviews = self.__get_page_reviews(page)

                    if not page_reviews:
                        logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                       page['title'], page['pageid'])
                        continue

                    yield page_reviews
                    npages += 1
            if hole_created:
                logger.error("Incremental update NOT completed. Hole in history created.")
            logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)

        def fetch_all_pages(namespaces_contents):
            # Use get all pages API to get pages
            npages = 0  # number of pages processed
            tpages = 0  # number of total pages
            pages_done = []  # pages already retrieved in reviews API

            for ns in namespaces_contents:
                apcontinue = ''  # pagination for getting pages
                logger.debug("Getting pages for namespace: %s", ns)
                while apcontinue is not None:
                    raw_pages = self.client.get_pages(ns, apcontinue)
                    data_json = json.loads(raw_pages)
                    if 'query-continue' in data_json:
                        # < 1.27
                        apcontinue = data_json['query-continue']['allpages']['apcontinue']
                    elif 'continue' in data_json:
                        # >= 1.27
                        apcontinue = data_json['continue']['apcontinue']
                    else:
                        apcontinue = None
                    pages_json = data_json['query']['allpages']
                    for page in pages_json:

                        if page['pageid'] in pages_done:
                            logger.debug("Page %s already processed; skipped", page['pageid'])
                            continue

                        tpages += 1
                        pages_done.append(page['pageid'])
                        page_reviews = self.__get_page_reviews(page)

                        if not page_reviews:
                            logger.warning("Revisions not found in %s [page id: %s], page skipped",
                                           page['title'], page['pageid'])
                            continue

                        yield page_reviews
                        npages += 1
            logger.info("Total number of pages: %i, skipped %i", tpages, tpages - npages)

        logger.info("Looking for pages at url '%s'", self.url)

        # from_date can not be older than MAX_RECENT_DAYS days ago
        if from_date:
            if (datetime_utcnow() - from_date).days >= MAX_RECENT_DAYS:
                cause = "Can't get incremental pages older than %i days." % MAX_RECENT_DAYS
                cause += " Do a complete analysis without from_date for older changes."
                raise BackendError(cause=cause)

        namespaces_contents = self.__get_namespaces_contents()

        if not from_date:
            return fetch_all_pages(namespaces_contents)
        else:
            return fetch_incremental_changes(namespaces_contents)
def __fetch_pre1_27 ( self , from_date = None ) : def fetch_incremental_changes ( namespaces_contents ) : # Use recent changes API to get the pages from date npages = 0 # number of pages processed tpages = 0 # number of total pages pages_done = [ ] # pages already retrieved in reviews API rccontinue = '' hole_created = True # To detect that incremental is not complete while rccontinue is not None : raw_pages = self . client . get_recent_pages ( namespaces_contents , rccontinue ) data_json = json . loads ( raw_pages ) if 'query-continue' in data_json : # < 1.27 rccontinue = data_json [ 'query-continue' ] [ 'recentchanges' ] [ 'rccontinue' ] elif 'continue' in data_json : # >= 1.27 rccontinue = data_json [ 'continue' ] [ 'rccontinue' ] else : rccontinue = None pages_json = data_json [ 'query' ] [ 'recentchanges' ] for page in pages_json : page_ts = dateutil . parser . parse ( page [ 'timestamp' ] ) if from_date >= page_ts : # The rest of recent changes are older than from_date logger . debug ( "All recent changes newer than %s processed." , from_date ) rccontinue = None hole_created = False break if page [ 'pageid' ] in pages_done : logger . debug ( "Page %s already processed; skipped" , page [ 'pageid' ] ) continue tpages += 1 pages_done . append ( page [ 'pageid' ] ) page_reviews = self . __get_page_reviews ( page ) if not page_reviews : logger . warning ( "Revisions not found in %s [page id: %s], page skipped" , page [ 'title' ] , page [ 'pageid' ] ) continue yield page_reviews npages += 1 if hole_created : logger . error ( "Incremental update NOT completed. Hole in history created." ) logger . info ( "Total number of pages: %i, skipped %i" , tpages , tpages - npages ) def fetch_all_pages ( namespaces_contents ) : # Use get all pages API to get pages npages = 0 # number of pages processed tpages = 0 # number of total pages pages_done = [ ] # pages already retrieved in reviews API for ns in namespaces_contents : apcontinue = '' # pagination for getting pages logger . debug ( "Getting pages for namespace: %s" , ns ) while apcontinue is not None : raw_pages = self . client . get_pages ( ns , apcontinue ) data_json = json . loads ( raw_pages ) if 'query-continue' in data_json : # < 1.27 apcontinue = data_json [ 'query-continue' ] [ 'allpages' ] [ 'apcontinue' ] elif 'continue' in data_json : # >= 1.27 apcontinue = data_json [ 'continue' ] [ 'apcontinue' ] else : apcontinue = None pages_json = data_json [ 'query' ] [ 'allpages' ] for page in pages_json : if page [ 'pageid' ] in pages_done : logger . debug ( "Page %s already processed; skipped" , page [ 'pageid' ] ) continue tpages += 1 pages_done . append ( page [ 'pageid' ] ) page_reviews = self . __get_page_reviews ( page ) if not page_reviews : logger . warning ( "Revisions not found in %s [page id: %s], page skipped" , page [ 'title' ] , page [ 'pageid' ] ) continue yield page_reviews npages += 1 logger . info ( "Total number of pages: %i, skipped %i" , tpages , tpages - npages ) logger . info ( "Looking for pages at url '%s'" , self . url ) # from_date can not be older than MAX_RECENT_DAYS days ago if from_date : if ( datetime_utcnow ( ) - from_date ) . days >= MAX_RECENT_DAYS : cause = "Can't get incremental pages older than %i days." % MAX_RECENT_DAYS cause += " Do a complete analysis without from_date for older changes." raise BackendError ( cause = cause ) namespaces_contents = self . __get_namespaces_contents ( ) if not from_date : return fetch_all_pages ( namespaces_contents ) else : return fetch_incremental_changes ( namespaces_contents )
Fetch the pages from the backend url.

        The method retrieves, from a MediaWiki url, the
        wiki pages.

        :returns: a generator of pages
Fetch the pages from the backend url .