python_code/consolidate_data.py
def consolidate(court_name):
    """
    Given court_name, a string representing a CourtListener court, consolidate will iterate through every file in the
    'clusters' and 'opinions' subdirectories, beginning with 'clusters'. For every case, the following information is
    grabbed:
        -Case number: INT -- note: this is not the same as the citation id
        -Cluster file: BOOLEAN True if we have a cluster file for this case
        -Opinion file: BOOLEAN True if we have an opinion file for this case
        -Date: DATETIME object based on CourtListener's 'file_date'
        -Judges: LIST of judges found in the cluster file
        -citation_id: INT representing the citation id found in the cluster file

    A summary of all of this information is then exported to a CSV in the court parent directory.
    """

    court_data_dir = data_dir + r'/%s' % court_name
    cluster_cases = {case: None for case in os.listdir(court_data_dir + r'/clusters')}
    num_clust = len(cluster_cases.keys())
    opinion_cases = {case: None for case in os.listdir(court_data_dir + r'/opinions')}
    num_op = len(opinion_cases.keys())
    print '%s cluster files, %s opinion files detected.' % (num_clust, num_op)

    data = {}
    print 'Consolidating cluster files...'
    clust_checked = 0
    for case in cluster_cases.keys():
        case_number = case.rsplit('.')[0]  # Drop '.json'
        has_cluster_file = True

        if case in opinion_cases.keys():
            has_opinion_file = True
            del opinion_cases[case]
        else:
            has_opinion_file = False

        cluster_file_data = helper_functions.json_to_dict(court_data_dir + r'/clusters/%s' % case)

        year, month, day = [int(element) for element in cluster_file_data['date_filed'].rsplit('-')]
        file_date = datetime.date(year=year, month=month, day=day)

        judges = cluster_file_data['judges']
        citation_id = "" if cluster_file_data['citation_id'] is None \
            else int(cluster_file_data['citation_id'])

        data[case_number] = [str(case_number), has_cluster_file, has_opinion_file,
                             '%s/%s/%s' % (file_date.month, file_date.day, file_date.year),
                             judges, citation_id]

        clust_checked += 1
        if clust_checked % 100 == 0:
            print '...%s of %s clusters consolidated...' % (clust_checked, num_clust)

    num_op = len(opinion_cases.keys())
    op_checked = 0
    for case in opinion_cases.keys()[:5]:
        case_number = case.rsplit('.')[0]  # Drop '.json'
        data[case_number] = [case_number, False, True, "", "", ""]

        op_checked += 1
        if op_checked % 100 == 0:
            print '...%s of %s opinions consolidated...' % (op_checked, num_op)

    consolidated_data = [['case_no', 'cluster_file', 'opinion_file', 'date', 'judges', 'citation_id']]
    for case in data.keys():
        consolidated_data.append(data[case])
    helper_functions.list_to_csv(data_dir + '/%s/node_metadata.csv'
                                 % court_name,
                                 consolidated_data)

    Given court_name, a string representing a CourtListener court, consolidate will iterate through every file in the
    'clusters' and 'opinions' subdirectories, beginning with 'clusters'. For every case, the following information is
    grabbed:
        -Case number: INT -- note: this is not the same as the citation id
        -Cluster file: BOOLEAN True if we have a cluster file for this case
        -Opinion file: BOOLEAN True if we have an opinion file for this case
        -Date: DATETIME object based on CourtListener's 'file_date'
        -Judges: LIST of judges found in the cluster file
        -citation_id: INT representing the citation id found in the cluster file

    A summary of all of this information is then exported to a CSV in the court parent directory.
    


A summary of all of this information is then exported to a CSV in the court parent directory.
A summary of all of this information is then exported to a CSV in the court parent directory .
def consolidate(court_name):
    court_data_dir = data_dir + r'/%s' % court_name
    cluster_cases = {case: None for case in os.listdir(court_data_dir + r'/clusters')}
    num_clust = len(cluster_cases.keys())
    opinion_cases = {case: None for case in os.listdir(court_data_dir + r'/opinions')}
    num_op = len(opinion_cases.keys())
    print '%s cluster files, %s opinion files detected.' % (num_clust, num_op)
    data = {}
    print 'Consolidating cluster files...'
    clust_checked = 0
    for case in cluster_cases.keys():
        case_number = case.rsplit('.')[0]  
        has_cluster_file = True
        if case in opinion_cases.keys():
            has_opinion_file = True
            del opinion_cases[case]
        else:
            has_opinion_file = False
        cluster_file_data = helper_functions.json_to_dict(court_data_dir + r'/clusters/%s' % case)
        year, month, day = [int(element) for element in cluster_file_data['date_filed'].rsplit('-')]
        file_date = datetime.date(year=year, month=month, day=day)
        judges = cluster_file_data['judges']
        citation_id = "" if cluster_file_data['citation_id'] is None \
            else int(cluster_file_data['citation_id'])
        data[case_number] = [str(case_number), has_cluster_file, has_opinion_file,
                             '%s/%s/%s' % (file_date.month, file_date.day, file_date.year),
                             judges, citation_id]
        clust_checked += 1
        if clust_checked % 100 == 0:
            print '...%s of %s clusters consolidated...' % (clust_checked, num_clust)
    num_op = len(opinion_cases.keys())
    op_checked = 0
    for case in opinion_cases.keys()[:5]:
        case_number = case.rsplit('.')[0]  
        data[case_number] = [case_number, False, True, "", "", ""]
        op_checked += 1
        if op_checked % 100 == 0:
            print '...%s of %s opinions consolidated...' % (op_checked, num_op)
    consolidated_data = [['case_no', 'cluster_file', 'opinion_file', 'date', 'judges', 'citation_id']]
    for case in data.keys():
        consolidated_data.append(data[case])
    helper_functions.list_to_csv(data_dir + '/%s/node_metadata.csv'
                                 % court_name,
                                 consolidated_data)
def consolidate ( court_name ) : court_data_dir = data_dir + r'/%s' % court_name cluster_cases = { case : None for case in os . listdir ( court_data_dir + r'/clusters' ) } num_clust = len ( cluster_cases . keys ( ) ) opinion_cases = { case : None for case in os . listdir ( court_data_dir + r'/opinions' ) } num_op = len ( opinion_cases . keys ( ) ) print '%s cluster files, %s opinion files detected.' % ( num_clust , num_op ) data = { } print 'Consolidating cluster files...' clust_checked = 0 for case in cluster_cases . keys ( ) : case_number = case . rsplit ( '.' ) [ 0 ] has_cluster_file = True if case in opinion_cases . keys ( ) : has_opinion_file = True del opinion_cases [ case ] else : has_opinion_file = False cluster_file_data = helper_functions . json_to_dict ( court_data_dir + r'/clusters/%s' % case ) year , month , day = [ int ( element ) for element in cluster_file_data [ 'date_filed' ] . rsplit ( '-' ) ] file_date = datetime . date ( year = year , month = month , day = day ) judges = cluster_file_data [ 'judges' ] citation_id = "" if cluster_file_data [ 'citation_id' ] is None else int ( cluster_file_data [ 'citation_id' ] ) data [ case_number ] = [ str ( case_number ) , has_cluster_file , has_opinion_file , '%s/%s/%s' % ( file_date . month , file_date . day , file_date . year ) , judges , citation_id ] clust_checked += 1 if clust_checked % 100 == 0 : print '...%s of %s clusters consolidated...' % ( clust_checked , num_clust ) num_op = len ( opinion_cases . keys ( ) ) op_checked = 0 for case in opinion_cases . keys ( ) [ : 5 ] : case_number = case . rsplit ( '.' ) [ 0 ] data [ case_number ] = [ case_number , False , True , "" , "" , "" ] op_checked += 1 if op_checked % 100 == 0 : print '...%s of %s opinions consolidated...' % ( op_checked , num_op ) consolidated_data = [ [ 'case_no' , 'cluster_file' , 'opinion_file' , 'date' , 'judges' , 'citation_id' ] ] for case in data . keys ( ) : consolidated_data . append ( data [ case ] ) helper_functions . list_to_csv ( data_dir + '/%s/node_metadata.csv' % court_name , consolidated_data )
Given court_name, a string representing a CourtListener court, consolidate will iterate through every file in the
'clusters' and 'opinions' subdirectories, beginning with 'clusters'.
Given court_name a string representing a CourtListener court consolidate will iterate through every file in the ' clusters ' and ' opinions ' subdirectories beginning with ' clusters ' .
"""
    Given court_name, a string representing a CourtListener court, consolidate will iterate through every file in the
    'clusters' and 'opinions' subdirectories, beginning with 'clusters'. For every case, the following information is
    grabbed:
        -Case number: INT -- note: this is not the same as the citation id
        -Cluster file: BOOLEAN True if we have a cluster file for this case
        -Opinion file: BOOLEAN True if we have an opinion file for this case
        -Date: DATETIME object based on CourtListener's 'file_date'
        -Judges: LIST of judges found in the cluster file
        -citation_id: INT representing the citation id found in the cluster file

    A summary of all of this information is then exported to a CSV in the court parent directory.
    """ # Drop '.json' # Drop '.json'
court_name