ci_mapping/run_pipeline.py
def data_wrangling(self):
        """Cleaning data for exploratory data analysis."""
        # Connect to postgresql
        s = self._create_session()

        # Read geocoded affiliations
        self.aff_location = pd.read_sql(s.query(AffiliationLocation).statement, s.bind)
        self.aff_location = self.aff_location.dropna(subset=["country"])
        # Read journals, open access flag and conferences
        self.journals = pd.read_sql(s.query(Journal).statement, s.bind)
        self.open_access = pd.read_sql(s.query(OpenAccess).statement, s.bind)
        self.conferences = pd.read_sql(s.query(Conference).statement, s.bind)
        # Read Fields of Study and their metadata (level in hierarchy)
        pfos = pd.read_sql(s.query(PaperFieldsOfStudy).statement, s.bind)
        fos = pd.read_sql(s.query(FieldOfStudy).statement, s.bind)
        self.pfos = pfos.merge(fos, left_on="field_of_study_id", right_on="id")[
            ["paper_id", "field_of_study_id", "name"]
        ]
        # That's very hacky, sorry :(
        self.pfos["name"] = [
            self.fos_mapping[n] if n in self.fos_mapping.keys() else n
            for n in self.pfos.name
        ]
        self.fos_metadata = pd.read_sql(s.query(FosMetadata).statement, s.bind)

        # Data wrangling
        self.data = clean_data(s)
        self.aff_papers, self.paper_author_aff = clean_author_affiliations(s, self.data)

        self.next(self.eda)
Cleaning data for exploratory data analysis.
Cleaning data for exploratory data analysis.
Cleaning data for exploratory data analysis .
def data_wrangling(self):
        s = self._create_session()
        self.aff_location = pd.read_sql(s.query(AffiliationLocation).statement, s.bind)
        self.aff_location = self.aff_location.dropna(subset=["country"])
        self.journals = pd.read_sql(s.query(Journal).statement, s.bind)
        self.open_access = pd.read_sql(s.query(OpenAccess).statement, s.bind)
        self.conferences = pd.read_sql(s.query(Conference).statement, s.bind)
        pfos = pd.read_sql(s.query(PaperFieldsOfStudy).statement, s.bind)
        fos = pd.read_sql(s.query(FieldOfStudy).statement, s.bind)
        self.pfos = pfos.merge(fos, left_on="field_of_study_id", right_on="id")[
            ["paper_id", "field_of_study_id", "name"]
        ]
        self.pfos["name"] = [
            self.fos_mapping[n] if n in self.fos_mapping.keys() else n
            for n in self.pfos.name
        ]
        self.fos_metadata = pd.read_sql(s.query(FosMetadata).statement, s.bind)
        self.data = clean_data(s)
        self.aff_papers, self.paper_author_aff = clean_author_affiliations(s, self.data)
        self.next(self.eda)
def data_wrangling ( self ) : s = self . _create_session ( ) self . aff_location = pd . read_sql ( s . query ( AffiliationLocation ) . statement , s . bind ) self . aff_location = self . aff_location . dropna ( subset = [ "country" ] ) self . journals = pd . read_sql ( s . query ( Journal ) . statement , s . bind ) self . open_access = pd . read_sql ( s . query ( OpenAccess ) . statement , s . bind ) self . conferences = pd . read_sql ( s . query ( Conference ) . statement , s . bind ) pfos = pd . read_sql ( s . query ( PaperFieldsOfStudy ) . statement , s . bind ) fos = pd . read_sql ( s . query ( FieldOfStudy ) . statement , s . bind ) self . pfos = pfos . merge ( fos , left_on = "field_of_study_id" , right_on = "id" ) [ [ "paper_id" , "field_of_study_id" , "name" ] ] self . pfos [ "name" ] = [ self . fos_mapping [ n ] if n in self . fos_mapping . keys ( ) else n for n in self . pfos . name ] self . fos_metadata = pd . read_sql ( s . query ( FosMetadata ) . statement , s . bind ) self . data = clean_data ( s ) self . aff_papers , self . paper_author_aff = clean_author_affiliations ( s , self . data ) self . next ( self . eda )
Cleaning data for exploratory data analysis.
Cleaning data for exploratory data analysis .
"""Cleaning data for exploratory data analysis.""" # Connect to postgresql # Read geocoded affiliations # Read journals, open access flag and conferences # Read Fields of Study and their metadata (level in hierarchy) # That's very hacky, sorry :( # Data wrangling
self