scripts/download_sen12.py
def process_datasource(
        self, loop_start=0, limit=None, polygon_id_list=None
    ):
        '''
        Applies Cloudfree Merging to the polygons in the GEE FeatureCollection and exports them one at a time.

        IMPORTANT NOTE: GEE only allows 3,000 exports in your export queue at any one time. Please make sure
        your feature collection does not exceed 3,000.

        "loop_start": This method loops through your entire FeatureCollection by default. If you wish to instead
        start the loop at a specific value, input that value here. Default is 0.

        "limit": Again, this method loops through the entire FeatureCollection by default.
        If you want to only export the first X polygons instead, input that X here.

        "minutes_to_wait": If GEE times out, this method waits 60 minutes by default before starting again where it left off.
        You may change the number of minutes it waits here.

        "polygon_id_list": If you only want to export certain polygons, enter their IDs into a list and input that list here.
        '''
        # source = self.source
        # sensor = self.sensor
        # export_folder = self.export_folder
        # polygons = self.polygons
        # date_range_list = self.date_range_list
        # debug = self.debug

        feature_list = self.polygons.sort(self.source['sort_by']).toList(self.polygons.size())
        n_polygons = feature_list.size().getInfo()
        print(f"{n_polygons} features have been loaded")

        #task_list = []

        exports = []
        exceptions = []

        filename_parts = self._create_filename_parts()

        if not limit:
            limit = n_polygons

        ### ERROR? ###
        ## Originally this was range(1, n_features), but we're pretty sure
        ## that should be 0 so we changed it.


        if polygon_id_list:
            if type(polygon_id_list) is list or type(polygon_id_list) is tuple:
                if all(isinstance(x, int) for x in polygon_id_list) is True:
                    loop_values = [poly_id - 1 for poly_id in polygon_id_list]
                else:
                    raise ValueError(f'all values within polygon_id_list should be of type {int}')    
            else:
                raise ValueError('polygon_id_list has to be either a list or a tuple')
                        
            
        else:
            loop_values = range(loop_start, limit)

        for i in loop_values:
            polygon_id = i + 1

            print(f'Processing polygon {polygon_id} of {limit}', end='\r', flush=True)

            feature = ee.Feature( feature_list.get(i) )
            if self._get_feature_area(feature) > 1000:
                print(f'Polygon {polygon_id} has area greater than 1000; skipping')
            else:
                roi = feature.geometry()
                roi = roi.coordinates().getInfo()[0]
                # tile = None

                time_stamp = "_".join(time.ctime().split(" ")[1:])
                filename = "_".join([str(polygon_id)] + self.source['name'] + [time_stamp])
                dest_path = "/".join(filename_parts + [filename])

                self.export_params = {
                    'bucket': self.export_folder,
                    'resolution': self.source['resolution'],
                    'filename': filename,
                    'dest_path': dest_path
                }

                date_range = self.date_range_list[i]

                self.current_poly = {
                    'date_range': date_range,
                    'polygon_id': polygon_id,
                    'roi': roi
                }


                if self.dynamic_date_range is False:
                    self.skip_test_check = True
                else:
                    self.skip_test_check = False
                        
                self._export_try_except_loop(attempts=0)

                if self.debug:
                    logging_timestamp = "_".join(time.ctime().split(" ")[1:])
                    logging.info(f'{logging_timestamp}: Polygon {polygon_id} successfully processed')

        return self.exports, self.exceptions

        Applies Cloudfree Merging to the polygons in the GEE FeatureCollection and exports them one at a time.

        IMPORTANT NOTE: GEE only allows 3,000 exports in your export queue at any one time. Please make sure
        your feature collection does not exceed 3,000.

        "loop_start": This method loops through your entire FeatureCollection by default. If you wish to instead
        start the loop at a specific value, input that value here. Default is 0.

        "limit": Again, this method loops through the entire FeatureCollection by default.
        If you want to only export the first X polygons instead, input that X here.

        "minutes_to_wait": If GEE times out, this method waits 60 minutes by default before starting again where it left off.
        You may change the number of minutes it waits here.

        "polygon_id_list": If you only want to export certain polygons, enter their IDs into a list and input that list here.
        
Applies Cloudfree Merging to the polygons in the GEE FeatureCollection and exports them one at a time.
IMPORTANT NOTE: GEE only allows 3,000 exports in your export queue at any one time. Please make sure
your feature collection does not exceed 3,000.

"loop_start": This method loops through your entire FeatureCollection by default. If you wish to instead
start the loop at a specific value, input that value here. Default is 0.

"limit": Again, this method loops through the entire FeatureCollection by default.
If you want to only export the first X polygons instead, input that X here.

"minutes_to_wait": If GEE times out, this method waits 60 minutes by default before starting again where it left off.
You may change the number of minutes it waits here.

"polygon_id_list": If you only want to export certain polygons, enter their IDs into a list and input that list here.
Applies Cloudfree Merging to the polygons in the GEE FeatureCollection and exports them one at a time . IMPORTANT NOTE : GEE only allows 3 000 exports in your export queue at any one time . Please make sure your feature collection does not exceed 3 000 . " loop_start " : This method loops through your entire FeatureCollection by default . If you wish to instead start the loop at a specific value input that value here . Default is 0 . " limit " : Again this method loops through the entire FeatureCollection by default . If you want to only export the first X polygons instead input that X here . " minutes_to_wait " : If GEE times out this method waits 60 minutes by default before starting again where it left off . You may change the number of minutes it waits here . " polygon_id_list " : If you only want to export certain polygons enter their IDs into a list and input that list here .
def process_datasource(
        self, loop_start=0, limit=None, polygon_id_list=None
    ):
        feature_list = self.polygons.sort(self.source['sort_by']).toList(self.polygons.size())
        n_polygons = feature_list.size().getInfo()
        print(f"{n_polygons} features have been loaded")
        exports = []
        exceptions = []
        filename_parts = self._create_filename_parts()
        if not limit:
            limit = n_polygons
        if polygon_id_list:
            if type(polygon_id_list) is list or type(polygon_id_list) is tuple:
                if all(isinstance(x, int) for x in polygon_id_list) is True:
                    loop_values = [poly_id - 1 for poly_id in polygon_id_list]
                else:
                    raise ValueError(f'all values within polygon_id_list should be of type {int}')    
            else:
                raise ValueError('polygon_id_list has to be either a list or a tuple')
        else:
            loop_values = range(loop_start, limit)
        for i in loop_values:
            polygon_id = i + 1
            print(f'Processing polygon {polygon_id} of {limit}', end='\r', flush=True)
            feature = ee.Feature( feature_list.get(i) )
            if self._get_feature_area(feature) > 1000:
                print(f'Polygon {polygon_id} has area greater than 1000; skipping')
            else:
                roi = feature.geometry()
                roi = roi.coordinates().getInfo()[0]
                time_stamp = "_".join(time.ctime().split(" ")[1:])
                filename = "_".join([str(polygon_id)] + self.source['name'] + [time_stamp])
                dest_path = "/".join(filename_parts + [filename])
                self.export_params = {
                    'bucket': self.export_folder,
                    'resolution': self.source['resolution'],
                    'filename': filename,
                    'dest_path': dest_path
                }
                date_range = self.date_range_list[i]
                self.current_poly = {
                    'date_range': date_range,
                    'polygon_id': polygon_id,
                    'roi': roi
                }
                if self.dynamic_date_range is False:
                    self.skip_test_check = True
                else:
                    self.skip_test_check = False
                self._export_try_except_loop(attempts=0)
                if self.debug:
                    logging_timestamp = "_".join(time.ctime().split(" ")[1:])
                    logging.info(f'{logging_timestamp}: Polygon {polygon_id} successfully processed')
        return self.exports, self.exceptions
def process_datasource ( self , loop_start = 0 , limit = None , polygon_id_list = None ) : feature_list = self . polygons . sort ( self . source [ 'sort_by' ] ) . toList ( self . polygons . size ( ) ) n_polygons = feature_list . size ( ) . getInfo ( ) print ( f"{n_polygons} features have been loaded" ) exports = [ ] exceptions = [ ] filename_parts = self . _create_filename_parts ( ) if not limit : limit = n_polygons if polygon_id_list : if type ( polygon_id_list ) is list or type ( polygon_id_list ) is tuple : if all ( isinstance ( x , int ) for x in polygon_id_list ) is True : loop_values = [ poly_id - 1 for poly_id in polygon_id_list ] else : raise ValueError ( f'all values within polygon_id_list should be of type {int}' ) else : raise ValueError ( 'polygon_id_list has to be either a list or a tuple' ) else : loop_values = range ( loop_start , limit ) for i in loop_values : polygon_id = i + 1 print ( f'Processing polygon {polygon_id} of {limit}' , end = '\r' , flush = True ) feature = ee . Feature ( feature_list . get ( i ) ) if self . _get_feature_area ( feature ) > 1000 : print ( f'Polygon {polygon_id} has area greater than 1000; skipping' ) else : roi = feature . geometry ( ) roi = roi . coordinates ( ) . getInfo ( ) [ 0 ] time_stamp = "_" . join ( time . ctime ( ) . split ( " " ) [ 1 : ] ) filename = "_" . join ( [ str ( polygon_id ) ] + self . source [ 'name' ] + [ time_stamp ] ) dest_path = "/" . join ( filename_parts + [ filename ] ) self . export_params = { 'bucket' : self . export_folder , 'resolution' : self . source [ 'resolution' ] , 'filename' : filename , 'dest_path' : dest_path } date_range = self . date_range_list [ i ] self . current_poly = { 'date_range' : date_range , 'polygon_id' : polygon_id , 'roi' : roi } if self . dynamic_date_range is False : self . skip_test_check = True else : self . skip_test_check = False self . _export_try_except_loop ( attempts = 0 ) if self . debug : logging_timestamp = "_" . join ( time . ctime ( ) . split ( " " ) [ 1 : ] ) logging . info ( f'{logging_timestamp}: Polygon {polygon_id} successfully processed' ) return self . exports , self . exceptions
Applies Cloudfree Merging to the polygons in the GEE FeatureCollection and exports them one at a time.
Applies Cloudfree Merging to the polygons in the GEE FeatureCollection and exports them one at a time .
'''
        Applies Cloudfree Merging to the polygons in the GEE FeatureCollection and exports them one at a time.

        IMPORTANT NOTE: GEE only allows 3,000 exports in your export queue at any one time. Please make sure
        your feature collection does not exceed 3,000.

        "loop_start": This method loops through your entire FeatureCollection by default. If you wish to instead
        start the loop at a specific value, input that value here. Default is 0.

        "limit": Again, this method loops through the entire FeatureCollection by default.
        If you want to only export the first X polygons instead, input that X here.

        "minutes_to_wait": If GEE times out, this method waits 60 minutes by default before starting again where it left off.
        You may change the number of minutes it waits here.

        "polygon_id_list": If you only want to export certain polygons, enter their IDs into a list and input that list here.
        ''' # source = self.source # sensor = self.sensor # export_folder = self.export_folder # polygons = self.polygons # date_range_list = self.date_range_list # debug = self.debug #task_list = [] ### ERROR? ### ## Originally this was range(1, n_features), but we're pretty sure ## that should be 0 so we changed it. # tile = None
self
loop_start
limit
polygon_id_list