ci_mapping/run_pipeline.py
def collect_mag(self):
        """Collect papers from MAG and store the response locally as a pickle."""
        # Convert strings to datetime objects
        mag_start_date = str2datetime(self.mag_start_date)
        mag_end_date = str2datetime(self.mag_end_date)

        # Number of time intervals for the data collection
        total_intervals = (
            abs(mag_start_date.year - mag_end_date.year) + 1
        ) * self.intervals_in_a_year

        i = 0
        query_count = 1000
        for date in toolz.sliding_window(
            2, list(date_range(mag_start_date, mag_end_date, total_intervals))
        ):
            logger.info(f"Date interval: {date}")
            expression = build_composite_expr(self.query_values, self.entity_name, date)
            logger.info(f"{expression}")

            has_content = True
            # i = 1
            offset = 0
            # Request the API as long as we receive non-empty responses
            while has_content:
                logger.info(f"Query {i} - Offset {offset}...")

                data = query_mag_api(
                    expression,
                    self.metadata,
                    self.subscription_key,
                    query_count=query_count,
                    offset=offset,
                )

                if self.with_doi:
                    # Keep only papers with a DOI
                    results = [
                        ents for ents in data["entities"] if "DOI" in ents.keys()
                    ]
                else:
                    results = [ents for ents in data["entities"]]

                # Store results
                with open(
                    f"{ci_mapping.project_dir}/{self.store_path}_{i}.pickle", "wb"
                ) as h:
                    pickle.dump(results, h)
                logger.info(f"Number of stored results from query {i}: {len(results)}")

                i += 1
                offset += query_count

                if len(results) == 0:
                    has_content = False

        self.next(self.parse_mag)
Collect papers from MAG and store the response locally as a pickle.
Collect papers from MAG and store the response locally as a pickle.
Collect papers from MAG and store the response locally as a pickle .
def collect_mag(self):
        mag_start_date = str2datetime(self.mag_start_date)
        mag_end_date = str2datetime(self.mag_end_date)
        total_intervals = (
            abs(mag_start_date.year - mag_end_date.year) + 1
        ) * self.intervals_in_a_year
        i = 0
        query_count = 1000
        for date in toolz.sliding_window(
            2, list(date_range(mag_start_date, mag_end_date, total_intervals))
        ):
            logger.info(f"Date interval: {date}")
            expression = build_composite_expr(self.query_values, self.entity_name, date)
            logger.info(f"{expression}")
            has_content = True
            offset = 0
            while has_content:
                logger.info(f"Query {i} - Offset {offset}...")
                data = query_mag_api(
                    expression,
                    self.metadata,
                    self.subscription_key,
                    query_count=query_count,
                    offset=offset,
                )
                if self.with_doi:
                    results = [
                        ents for ents in data["entities"] if "DOI" in ents.keys()
                    ]
                else:
                    results = [ents for ents in data["entities"]]
                with open(
                    f"{ci_mapping.project_dir}/{self.store_path}_{i}.pickle", "wb"
                ) as h:
                    pickle.dump(results, h)
                logger.info(f"Number of stored results from query {i}: {len(results)}")
                i += 1
                offset += query_count
                if len(results) == 0:
                    has_content = False
        self.next(self.parse_mag)
def collect_mag ( self ) : mag_start_date = str2datetime ( self . mag_start_date ) mag_end_date = str2datetime ( self . mag_end_date ) total_intervals = ( abs ( mag_start_date . year - mag_end_date . year ) + 1 ) * self . intervals_in_a_year i = 0 query_count = 1000 for date in toolz . sliding_window ( 2 , list ( date_range ( mag_start_date , mag_end_date , total_intervals ) ) ) : logger . info ( f"Date interval: {date}" ) expression = build_composite_expr ( self . query_values , self . entity_name , date ) logger . info ( f"{expression}" ) has_content = True offset = 0 while has_content : logger . info ( f"Query {i} - Offset {offset}..." ) data = query_mag_api ( expression , self . metadata , self . subscription_key , query_count = query_count , offset = offset , ) if self . with_doi : results = [ ents for ents in data [ "entities" ] if "DOI" in ents . keys ( ) ] else : results = [ ents for ents in data [ "entities" ] ] with open ( f"{ci_mapping.project_dir}/{self.store_path}_{i}.pickle" , "wb" ) as h : pickle . dump ( results , h ) logger . info ( f"Number of stored results from query {i}: {len(results)}" ) i += 1 offset += query_count if len ( results ) == 0 : has_content = False self . next ( self . parse_mag )
Collect papers from MAG and store the response locally as a pickle.
Collect papers from MAG and store the response locally as a pickle .
"""Collect papers from MAG and store the response locally as a pickle.""" # Convert strings to datetime objects # Number of time intervals for the data collection # i = 1 # Request the API as long as we receive non-empty responses # Keep only papers with a DOI # Store results
self