scripts/update_filter_lists.py
def update_derived_lists():
    """Update lists created from base lists
    """
    # Update KNOWN_FISHING list
    infopath = "GFW/VESSEL_INFO/CONSOLIDATED_LISTS.csv"
    path = "GFW/FISHING_MMSI/KNOWN"
    print("Updating", path)
    with open(os.path.join(asset_dir, infopath)) as fin:
        with open(os.path.join(asset_dir, path, "ALL_YEARS.txt"), 'w') as fout:
            reader = csv.DictReader(fin)
            for row in reader:
                if row['label'] in fishing_classes:
                    mmsi = row['mmsi'].strip()
                    fout.write(mmsi + '\n')


    # Update KNOWN_NONFISHING list
    infopath = "GFW/VESSEL_INFO/CONSOLIDATED_LISTS.csv"
    path = "GFW/NONFISHING_MMSI/KNOWN"
    print("Updating", path)
    with open(os.path.join(asset_dir, infopath)) as fin:
        with open(os.path.join(asset_dir, path, "ALL_YEARS.txt"), 'w') as fout:
            reader = csv.DictReader(fin)
            for row in reader:
                if row['label'] in non_fishing_classes:
                    mmsi = row['mmsi'].strip()
                    fout.write(mmsi + '\n')


    # Update combined fishing list
    path = "GFW/FISHING_MMSI/KNOWN_AND_LIKELY"
    print("Updating", path)
    any_year = set()
    for date_range in default_date_ranges:
            start_date, end_date = date_range
            year = end_date[:4]
            print(year)
            combined = build_combined_fishing_list(asset_dir, year)
            any_year |= set(combined)
            dest_path = os.path.join(asset_dir, path, "{}.txt".format(year))
            with open(dest_path, "w") as dest:
                for mmsi in combined:
                    dest.write(mmsi + '\n')
    dest_path = os.path.join(asset_dir, path, "ANY_YEAR.txt")
    with open(dest_path, "w") as dest:
        for mmsi in sorted(any_year):
            dest.write(mmsi + '\n')

    # Update suspected fishing list
    path = "GFW/FISHING_MMSI/SUSPECTED/ANY_YEAR.txt"
    print("Updating", path)
    joint = set(any_year)
    with open(os.path.join(asset_dir, "GFW/VESSEL_INFO/VESSEL_LISTS/ALL_YEARS.csv")) as fin:
        with open(os.path.join(asset_dir, path), 'w') as fout:
            reader = csv.DictReader(fin)
            for row in reader:
                label = row['inferred']
                if label in fishing_classes:
                    mmsi = row['mmsi'].strip()
                    joint.add(mmsi)
                    fout.write(mmsi + '\n')
                else:
                    assert label in non_fishing_classes, "{} not in non-fishing".format(label)

    # Update joint fishing list
    path = "GFW/FISHING_MMSI/KNOWN_LIKELY_AND_SUSPECTED/ANY_YEAR.txt"
    print("Updating", path)
    any_year = set()
    with open(os.path.join(asset_dir, path), 'w') as fout:
        for mmsi in sorted(joint):
            fout.write(str(mmsi) + '\n')
Update lists created from base lists
    
Update lists created from base lists
Update lists created from base lists
def update_derived_lists():
    infopath = "GFW/VESSEL_INFO/CONSOLIDATED_LISTS.csv"
    path = "GFW/FISHING_MMSI/KNOWN"
    print("Updating", path)
    with open(os.path.join(asset_dir, infopath)) as fin:
        with open(os.path.join(asset_dir, path, "ALL_YEARS.txt"), 'w') as fout:
            reader = csv.DictReader(fin)
            for row in reader:
                if row['label'] in fishing_classes:
                    mmsi = row['mmsi'].strip()
                    fout.write(mmsi + '\n')
    infopath = "GFW/VESSEL_INFO/CONSOLIDATED_LISTS.csv"
    path = "GFW/NONFISHING_MMSI/KNOWN"
    print("Updating", path)
    with open(os.path.join(asset_dir, infopath)) as fin:
        with open(os.path.join(asset_dir, path, "ALL_YEARS.txt"), 'w') as fout:
            reader = csv.DictReader(fin)
            for row in reader:
                if row['label'] in non_fishing_classes:
                    mmsi = row['mmsi'].strip()
                    fout.write(mmsi + '\n')
    path = "GFW/FISHING_MMSI/KNOWN_AND_LIKELY"
    print("Updating", path)
    any_year = set()
    for date_range in default_date_ranges:
            start_date, end_date = date_range
            year = end_date[:4]
            print(year)
            combined = build_combined_fishing_list(asset_dir, year)
            any_year |= set(combined)
            dest_path = os.path.join(asset_dir, path, "{}.txt".format(year))
            with open(dest_path, "w") as dest:
                for mmsi in combined:
                    dest.write(mmsi + '\n')
    dest_path = os.path.join(asset_dir, path, "ANY_YEAR.txt")
    with open(dest_path, "w") as dest:
        for mmsi in sorted(any_year):
            dest.write(mmsi + '\n')
    path = "GFW/FISHING_MMSI/SUSPECTED/ANY_YEAR.txt"
    print("Updating", path)
    joint = set(any_year)
    with open(os.path.join(asset_dir, "GFW/VESSEL_INFO/VESSEL_LISTS/ALL_YEARS.csv")) as fin:
        with open(os.path.join(asset_dir, path), 'w') as fout:
            reader = csv.DictReader(fin)
            for row in reader:
                label = row['inferred']
                if label in fishing_classes:
                    mmsi = row['mmsi'].strip()
                    joint.add(mmsi)
                    fout.write(mmsi + '\n')
                else:
                    assert label in non_fishing_classes, "{} not in non-fishing".format(label)
    path = "GFW/FISHING_MMSI/KNOWN_LIKELY_AND_SUSPECTED/ANY_YEAR.txt"
    print("Updating", path)
    any_year = set()
    with open(os.path.join(asset_dir, path), 'w') as fout:
        for mmsi in sorted(joint):
            fout.write(str(mmsi) + '\n')
def update_derived_lists ( ) : infopath = "GFW/VESSEL_INFO/CONSOLIDATED_LISTS.csv" path = "GFW/FISHING_MMSI/KNOWN" print ( "Updating" , path ) with open ( os . path . join ( asset_dir , infopath ) ) as fin : with open ( os . path . join ( asset_dir , path , "ALL_YEARS.txt" ) , 'w' ) as fout : reader = csv . DictReader ( fin ) for row in reader : if row [ 'label' ] in fishing_classes : mmsi = row [ 'mmsi' ] . strip ( ) fout . write ( mmsi + '\n' ) infopath = "GFW/VESSEL_INFO/CONSOLIDATED_LISTS.csv" path = "GFW/NONFISHING_MMSI/KNOWN" print ( "Updating" , path ) with open ( os . path . join ( asset_dir , infopath ) ) as fin : with open ( os . path . join ( asset_dir , path , "ALL_YEARS.txt" ) , 'w' ) as fout : reader = csv . DictReader ( fin ) for row in reader : if row [ 'label' ] in non_fishing_classes : mmsi = row [ 'mmsi' ] . strip ( ) fout . write ( mmsi + '\n' ) path = "GFW/FISHING_MMSI/KNOWN_AND_LIKELY" print ( "Updating" , path ) any_year = set ( ) for date_range in default_date_ranges : start_date , end_date = date_range year = end_date [ : 4 ] print ( year ) combined = build_combined_fishing_list ( asset_dir , year ) any_year |= set ( combined ) dest_path = os . path . join ( asset_dir , path , "{}.txt" . format ( year ) ) with open ( dest_path , "w" ) as dest : for mmsi in combined : dest . write ( mmsi + '\n' ) dest_path = os . path . join ( asset_dir , path , "ANY_YEAR.txt" ) with open ( dest_path , "w" ) as dest : for mmsi in sorted ( any_year ) : dest . write ( mmsi + '\n' ) path = "GFW/FISHING_MMSI/SUSPECTED/ANY_YEAR.txt" print ( "Updating" , path ) joint = set ( any_year ) with open ( os . path . join ( asset_dir , "GFW/VESSEL_INFO/VESSEL_LISTS/ALL_YEARS.csv" ) ) as fin : with open ( os . path . join ( asset_dir , path ) , 'w' ) as fout : reader = csv . DictReader ( fin ) for row in reader : label = row [ 'inferred' ] if label in fishing_classes : mmsi = row [ 'mmsi' ] . strip ( ) joint . add ( mmsi ) fout . write ( mmsi + '\n' ) else : assert label in non_fishing_classes , "{} not in non-fishing" . format ( label ) path = "GFW/FISHING_MMSI/KNOWN_LIKELY_AND_SUSPECTED/ANY_YEAR.txt" print ( "Updating" , path ) any_year = set ( ) with open ( os . path . join ( asset_dir , path ) , 'w' ) as fout : for mmsi in sorted ( joint ) : fout . write ( str ( mmsi ) + '\n' )
Update lists created from base lists
Update lists created from base lists
"""Update lists created from base lists
    """ # Update KNOWN_FISHING list # Update KNOWN_NONFISHING list # Update combined fishing list # Update suspected fishing list # Update joint fishing list