usgsdownload/usgs.py
def download_file(self, url, download_dir, sceneName):
        """ Downloads large files in pieces   """
        try:
            # Log
            logger.info('\nStarting download..')
            print('\n Starting download..\n')

            # Request
            req = urllib.request.urlopen(url)

            try:
                if req.info().get_content_type() == 'text/html':
                    logger.error("error : the file format is html")
                    lines = req.read()
                    if lines.find('Download Not Found') > 0:
                        raise TypeError('Download USGS not found for scene: %s' % self.sceneInfo.name)
                    else:
                        print(lines)
                        print(sys.exit(-1))
            except Exception as e:
                logger.error('Erro in USGS download for scene %s error: %s' % (self.sceneInfo.name, e))
                raise CredentialsUsgsError('User or Password invalid ! ')

            total_size = int(req.getheader('Content-Length').strip())

            if total_size < 50000:
                logger.error("Error: The file is too small to be a Landsat Image for scene %s" % self.sceneInfo.name)
                raise SmallLandsatImageError("Error: The file is too small to be a Landsat Image")

            total_size_fmt = sizeof_fmt(total_size)
            downloaded = 0
            CHUNK = 1024 * 1024 * 8
            with open(download_dir + '/' + sceneName, 'wb') as fp:
                start = time.clock()
                logger.debug('Downloading {0} ({1}):'.format(self.sceneInfo.name, total_size_fmt))
                print('Downloading {0} ({1}):'.format(self.sceneInfo.name, total_size_fmt))
                while True:
                    chunk = req.read(CHUNK)
                    downloaded += len(chunk)
                    done = int(50 * downloaded / total_size)
                    print('\r[{1}{2}]{0:3.0f}% {3}ps'.format(floor((float(downloaded) / total_size) * 100), '-' * done,
                                                             ' ' * (50 - done), sizeof_fmt((downloaded //
                                                                                          (time.clock() - start)) / 8)))
                    if not chunk:
                        logger.debug('Download {0} completed({1}):'.format(self.sceneInfo.name, total_size_fmt))
                        break
                    fp.write(chunk)
        except urllib.error.HTTPError as e:
            if e.code == 500:
                logger.error("File doesn't exist")
                print("\n File doesn't exist: %s " % e)
                raise RemoteFileDoesntExist("File doesn't exist")
            elif e.code == 403:
                # Log celery
                logger.error("HTTP Error:", e.code, url)
                logger.debug('\n trying to download it again scene: %s' % self.sceneInfo.name)
                # Log shell
                print("\n HTTP Error:", e.code, url)
                print('\n trying to download it again scene: %s' % self.sceneInfo.name)
                self.connect_earthexplorer()
                self.download_file(url, download_dir, sceneName)
            else:
                logger.error("HTTP Error:", e)
                print("HTTP Error:", e.code, url)
                raise e
        except urllib.error.URLError as e:
            print("URL Error:", e.reason, url)
            logger.error("URL Error: %s in %s" % (e, url))
            raise e
        except ConnectionResetError as e:
            print('Error ConnectionResetError: %s' % e)
            logger.error('Error ConnectionResetError: %s' % e)
            print('\n trying to download it again scene: %s' % self.sceneInfo.name)
            logger.debug('trying to download it again scene: %s' % self.sceneInfo.name)
            self.download_file(url, download_dir, sceneName)
        except urllib.error.HTTPError as e:
            print('\n HttpError: %s' % e)
            print('\n trying to download it again scene: %s' % self.sceneInfo.name)
            logger.error('HttpError: %s' % e)
            logger.debug('trying to download it again scene: %s' % self.sceneInfo.name)
            self.download_file(url, download_dir, sceneName)
        except Exception as error:
            logger.error('Error unknown %s in download %s at scene: %s' % (error, url, self.sceneInfo.name))
            print('Error unknown %s in download % at scene: %s' % (error, url, self.sceneInfo.name))
            logger.debug('trying to download it again scene: %s' % self.sceneInfo.name)
            self.download_file(url, download_dir, sceneName)

        percent = floor((float(downloaded) / total_size) * 100) or 0
        if percent != 100:
            logger.debug('trying to download it again scene: %s' % self.sceneInfo.name)
            logger.error('Download interrupted in %s%%, trying to download it again scene: %s' % (
                percent, self.sceneInfo.name))
            print('\n Download interrupted in %s%%, trying to download it again scene: %s' % (
                percent, self.sceneInfo.name))
            self.download_file(url, download_dir, sceneName)

        path_item = download_dir + '/' + sceneName
        info = {'total_size': total_size_fmt, 'scene': self.sceneInfo.name,
                'sucess': verify_sucess(total_size, path_item), 'file_path': path_item}
        return info
def download_file(self, url, download_dir, sceneName):
        """ Downloads large files in pieces   """
        try:
            # Log
            logger.info('\nStarting download..')
            print('\n Starting download..\n')

            # Request
            req = urllib.request.urlopen(url)

            try:
                if req.info().get_content_type() == 'text/html':
                    logger.error("error : the file format is html")
                    lines = req.read()
                    if lines.find('Download Not Found') > 0:
                        raise TypeError('Download USGS not found for scene: %s' % self.sceneInfo.name)
                    else:
                        print(lines)
                        print(sys.exit(-1))
            except Exception as e:
                logger.error('Erro in USGS download for scene %s error: %s' % (self.sceneInfo.name, e))
                raise CredentialsUsgsError('User or Password invalid ! ')

            total_size = int(req.getheader('Content-Length').strip())

            if total_size < 50000:
                logger.error("Error: The file is too small to be a Landsat Image for scene %s" % self.sceneInfo.name)
                raise SmallLandsatImageError("Error: The file is too small to be a Landsat Image")

            total_size_fmt = sizeof_fmt(total_size)
            downloaded = 0
            CHUNK = 1024 * 1024 * 8
            with open(download_dir + '/' + sceneName, 'wb') as fp:
                start = time.clock()
                logger.debug('Downloading {0} ({1}):'.format(self.sceneInfo.name, total_size_fmt))
                print('Downloading {0} ({1}):'.format(self.sceneInfo.name, total_size_fmt))
                while True:
                    chunk = req.read(CHUNK)
                    downloaded += len(chunk)
                    done = int(50 * downloaded / total_size)
                    print('\r[{1}{2}]{0:3.0f}% {3}ps'.format(floor((float(downloaded) / total_size) * 100), '-' * done,
                                                             ' ' * (50 - done), sizeof_fmt((downloaded //
                                                                                          (time.clock() - start)) / 8)))
                    if not chunk:
                        logger.debug('Download {0} completed({1}):'.format(self.sceneInfo.name, total_size_fmt))
                        break
                    fp.write(chunk)
        except urllib.error.HTTPError as e:
            if e.code == 500:
                logger.error("File doesn't exist")
                print("\n File doesn't exist: %s " % e)
                raise RemoteFileDoesntExist("File doesn't exist")
            elif e.code == 403:
                # Log celery
                logger.error("HTTP Error:", e.code, url)
                logger.debug('\n trying to download it again scene: %s' % self.sceneInfo.name)
                # Log shell
                print("\n HTTP Error:", e.code, url)
                print('\n trying to download it again scene: %s' % self.sceneInfo.name)
                self.connect_earthexplorer()
                self.download_file(url, download_dir, sceneName)
            else:
                logger.error("HTTP Error:", e)
                print("HTTP Error:", e.code, url)
                raise e
        except urllib.error.URLError as e:
            print("URL Error:", e.reason, url)
            logger.error("URL Error: %s in %s" % (e, url))
            raise e
        except ConnectionResetError as e:
            print('Error ConnectionResetError: %s' % e)
            logger.error('Error ConnectionResetError: %s' % e)
            print('\n trying to download it again scene: %s' % self.sceneInfo.name)
            logger.debug('trying to download it again scene: %s' % self.sceneInfo.name)
            self.download_file(url, download_dir, sceneName)
        except urllib.error.HTTPError as e:
            print('\n HttpError: %s' % e)
            print('\n trying to download it again scene: %s' % self.sceneInfo.name)
            logger.error('HttpError: %s' % e)
            logger.debug('trying to download it again scene: %s' % self.sceneInfo.name)
            self.download_file(url, download_dir, sceneName)
        except Exception as error:
            logger.error('Error unknown %s in download %s at scene: %s' % (error, url, self.sceneInfo.name))
            print('Error unknown %s in download % at scene: %s' % (error, url, self.sceneInfo.name))
            logger.debug('trying to download it again scene: %s' % self.sceneInfo.name)
            self.download_file(url, download_dir, sceneName)

        percent = floor((float(downloaded) / total_size) * 100) or 0
        if percent != 100:
            logger.debug('trying to download it again scene: %s' % self.sceneInfo.name)
            logger.error('Download interrupted in %s%%, trying to download it again scene: %s' % (
                percent, self.sceneInfo.name))
            print('\n Download interrupted in %s%%, trying to download it again scene: %s' % (
                percent, self.sceneInfo.name))
            self.download_file(url, download_dir, sceneName)

        path_item = download_dir + '/' + sceneName
        info = {'total_size': total_size_fmt, 'scene': self.sceneInfo.name,
                'sucess': verify_sucess(total_size, path_item), 'file_path': path_item}
        return info
def download_file ( self , url , download_dir , sceneName ) : try : # Log logger . info ( '\nStarting download..' ) print ( '\n Starting download..\n' ) # Request req = urllib . request . urlopen ( url ) try : if req . info ( ) . get_content_type ( ) == 'text/html' : logger . error ( "error : the file format is html" ) lines = req . read ( ) if lines . find ( 'Download Not Found' ) > 0 : raise TypeError ( 'Download USGS not found for scene: %s' % self . sceneInfo . name ) else : print ( lines ) print ( sys . exit ( - 1 ) ) except Exception as e : logger . error ( 'Erro in USGS download for scene %s error: %s' % ( self . sceneInfo . name , e ) ) raise CredentialsUsgsError ( 'User or Password invalid ! ' ) total_size = int ( req . getheader ( 'Content-Length' ) . strip ( ) ) if total_size < 50000 : logger . error ( "Error: The file is too small to be a Landsat Image for scene %s" % self . sceneInfo . name ) raise SmallLandsatImageError ( "Error: The file is too small to be a Landsat Image" ) total_size_fmt = sizeof_fmt ( total_size ) downloaded = 0 CHUNK = 1024 * 1024 * 8 with open ( download_dir + '/' + sceneName , 'wb' ) as fp : start = time . clock ( ) logger . debug ( 'Downloading {0} ({1}):' . format ( self . sceneInfo . name , total_size_fmt ) ) print ( 'Downloading {0} ({1}):' . format ( self . sceneInfo . name , total_size_fmt ) ) while True : chunk = req . read ( CHUNK ) downloaded += len ( chunk ) done = int ( 50 * downloaded / total_size ) print ( '\r[{1}{2}]{0:3.0f}% {3}ps' . format ( floor ( ( float ( downloaded ) / total_size ) * 100 ) , '-' * done , ' ' * ( 50 - done ) , sizeof_fmt ( ( downloaded // ( time . clock ( ) - start ) ) / 8 ) ) ) if not chunk : logger . debug ( 'Download {0} completed({1}):' . format ( self . sceneInfo . name , total_size_fmt ) ) break fp . write ( chunk ) except urllib . error . HTTPError as e : if e . code == 500 : logger . error ( "File doesn't exist" ) print ( "\n File doesn't exist: %s " % e ) raise RemoteFileDoesntExist ( "File doesn't exist" ) elif e . code == 403 : # Log celery logger . error ( "HTTP Error:" , e . code , url ) logger . debug ( '\n trying to download it again scene: %s' % self . sceneInfo . name ) # Log shell print ( "\n HTTP Error:" , e . code , url ) print ( '\n trying to download it again scene: %s' % self . sceneInfo . name ) self . connect_earthexplorer ( ) self . download_file ( url , download_dir , sceneName ) else : logger . error ( "HTTP Error:" , e ) print ( "HTTP Error:" , e . code , url ) raise e except urllib . error . URLError as e : print ( "URL Error:" , e . reason , url ) logger . error ( "URL Error: %s in %s" % ( e , url ) ) raise e except ConnectionResetError as e : print ( 'Error ConnectionResetError: %s' % e ) logger . error ( 'Error ConnectionResetError: %s' % e ) print ( '\n trying to download it again scene: %s' % self . sceneInfo . name ) logger . debug ( 'trying to download it again scene: %s' % self . sceneInfo . name ) self . download_file ( url , download_dir , sceneName ) except urllib . error . HTTPError as e : print ( '\n HttpError: %s' % e ) print ( '\n trying to download it again scene: %s' % self . sceneInfo . name ) logger . error ( 'HttpError: %s' % e ) logger . debug ( 'trying to download it again scene: %s' % self . sceneInfo . name ) self . download_file ( url , download_dir , sceneName ) except Exception as error : logger . error ( 'Error unknown %s in download %s at scene: %s' % ( error , url , self . sceneInfo . name ) ) print ( 'Error unknown %s in download % at scene: %s' % ( error , url , self . sceneInfo . name ) ) logger . debug ( 'trying to download it again scene: %s' % self . sceneInfo . name ) self . download_file ( url , download_dir , sceneName ) percent = floor ( ( float ( downloaded ) / total_size ) * 100 ) or 0 if percent != 100 : logger . debug ( 'trying to download it again scene: %s' % self . sceneInfo . name ) logger . error ( 'Download interrupted in %s%%, trying to download it again scene: %s' % ( percent , self . sceneInfo . name ) ) print ( '\n Download interrupted in %s%%, trying to download it again scene: %s' % ( percent , self . sceneInfo . name ) ) self . download_file ( url , download_dir , sceneName ) path_item = download_dir + '/' + sceneName info = { 'total_size' : total_size_fmt , 'scene' : self . sceneInfo . name , 'sucess' : verify_sucess ( total_size , path_item ) , 'file_path' : path_item } return info
Downloads large files in pieces
Downloads large files in pieces