home/exp1000/acquire_and_label_images.py
def package_files_for_downlinking(self, file_ext, downlink_log_if_no_images, do_clustering, experiment_start_time, files_from_previous_runs, do_logging):
        """Package the files for downlinking.
        
        Logging is optional via the do_logging flag in case we start the experiment by tarring files leftover from a previous run that was abruptly interrupted.
        In that case we don't want to prematurely write a new log file for the current experiment run or else it's going to end up being tarred with the previous experiment run(s).
        """
        try:

            # Don't use gzip if files are already a compression file type.
            # Check against a list of file types in case multiple compression types are supported.
            tar_options = '-cf' if file_ext in SUPPORTED_COMPRESSION_TYPES else '-czf'
            tar_ext = 'tar' if file_ext in SUPPORTED_COMPRESSION_TYPES else 'tar.gz' 

            # The destination tar file path for the packaged files.
            tar_path = '{TG}/opssat_smartcam_{FILE_EXT}_exp{expID}_{D}.{TAR_EXT}'.format(\
                    TG=TOGROUND_PATH,\
                    FILE_EXT=file_ext,\
                    expID=EXP_ID,\
                    D=experiment_start_time.strftime("%Y%m%d_%H%M%S") + ("_previous" if files_from_previous_runs else ""),\
                    TAR_EXT=tar_ext)

            # Count how many images were kept and moved to the experiment's toGround folder.
            image_count = len(list(Path(TOGROUND_PATH).rglob('*.' + file_ext)))

            # Count how many log files were produced.
            log_count = len(list(Path(LOG_PATH).rglob('*.log')))
            
            # Tar images(s) for downlink if at least 1 image was classified with a label of interest.
            if image_count > 0:

                # Log that we are tarring some images.
                if do_logging:
                    logger.info("Tarring {T} file(s) for downlink.".format(T=image_count))

                # The paths of the image files to tar depends on whether or not we are clustering.
                img_files_to_tar = '{G}/**/*.{FILE_EXT}'.format(G=TOGROUND_PATH, FILE_EXT=file_ext)

                # Include cluster subfolders.
                if do_clustering:
                    img_files_to_tar = img_files_to_tar + ' {G}/**/**/*.{FILE_EXT}'.format(G=TOGROUND_PATH, FILE_EXT=file_ext)

                # Use tar to package image and log files into the filestore's toGround folder.
                tar_cmd = 'tar {TAR_O} {TAR_PATH} '.format(TAR_O=tar_options, TAR_PATH=tar_path) + img_files_to_tar + ' {L}/*.log {L}/*.csv --remove-files'.format(L=LOG_PATH)
                os.system(tar_cmd)

                # Return experiment toGround path to tar file.
                return tar_path

            elif downlink_log_if_no_images is True and log_count > 0:

                # Log that we are only tarring log files.
                if do_logging:
                    logger.info("No image(s) kept but tarring logs for downlink.")

                # The destination tar file path for the packaged files.
                tar_path = '{TG}/opssat_smartcam_{FILE_EXT}_exp{expID}_{D}.{TAR_EXT}'.format(\
                    TG=TOGROUND_PATH,\
                    FILE_EXT='logs',\
                    expID=EXP_ID,\
                    D=experiment_start_time.strftime("%Y%m%d_%H%M%S") + ("_previous" if files_from_previous_runs else ""),\
                    TAR_EXT=tar_ext)

                # Use tar to package log files into the filestore's toGround folder.
                os.system('tar {TAR_O} {TAR_PATH} {L}/*.log {L}/*.csv --remove-files'.format(\
                    TAR_O=tar_options,\
                    TAR_PATH=tar_path,\
                    L=LOG_PATH))

                # Return experiment toGround path to tar file.
                return tar_path

            else:
                # No images and no logs. Unlikely.
                if do_logging:
                    logger.info("No images(s) kept nor logs produced for downlink.")

                # Return None.
                return None

        except:
            # In case this happens, the image will be tarred at the end of the next experiment's run unless explicitely deleted.
            if do_logging:
                logger.exception("Failed to tar kept image for downlink (if any).")

            # Return None.
            return None
Package the files for downlinking.
        
        Logging is optional via the do_logging flag in case we start the experiment by tarring files leftover from a previous run that was abruptly interrupted.
        In that case we don't want to prematurely write a new log file for the current experiment run or else it's going to end up being tarred with the previous experiment run(s).
        
Package the files for downlinking.
Logging is optional via the do_logging flag in case we start the experiment by tarring files leftover from a previous run that was abruptly interrupted.
In that case we don't want to prematurely write a new log file for the current experiment run or else it's going to end up being tarred with the previous experiment run(s).
Package the files for downlinking . Logging is optional via the do_logging flag in case we start the experiment by tarring files leftover from a previous run that was abruptly interrupted . In that case we don ' t want to prematurely write a new log file for the current experiment run or else it ' s going to end up being tarred with the previous experiment run ( s ) .
def package_files_for_downlinking(self, file_ext, downlink_log_if_no_images, do_clustering, experiment_start_time, files_from_previous_runs, do_logging):
        try:
            tar_options = '-cf' if file_ext in SUPPORTED_COMPRESSION_TYPES else '-czf'
            tar_ext = 'tar' if file_ext in SUPPORTED_COMPRESSION_TYPES else 'tar.gz' 
            tar_path = '{TG}/opssat_smartcam_{FILE_EXT}_exp{expID}_{D}.{TAR_EXT}'.format(\
                    TG=TOGROUND_PATH,\
                    FILE_EXT=file_ext,\
                    expID=EXP_ID,\
                    D=experiment_start_time.strftime("%Y%m%d_%H%M%S") + ("_previous" if files_from_previous_runs else ""),\
                    TAR_EXT=tar_ext)
            image_count = len(list(Path(TOGROUND_PATH).rglob('*.' + file_ext)))
            log_count = len(list(Path(LOG_PATH).rglob('*.log')))
            if image_count > 0:
                if do_logging:
                    logger.info("Tarring {T} file(s) for downlink.".format(T=image_count))
                img_files_to_tar = '{G}/**/*.{FILE_EXT}'.format(G=TOGROUND_PATH, FILE_EXT=file_ext)
                if do_clustering:
                    img_files_to_tar = img_files_to_tar + ' {G}/**/**/*.{FILE_EXT}'.format(G=TOGROUND_PATH, FILE_EXT=file_ext)
                tar_cmd = 'tar {TAR_O} {TAR_PATH} '.format(TAR_O=tar_options, TAR_PATH=tar_path) + img_files_to_tar + ' {L}/*.log {L}/*.csv --remove-files'.format(L=LOG_PATH)
                os.system(tar_cmd)
                return tar_path
            elif downlink_log_if_no_images is True and log_count > 0:
                if do_logging:
                    logger.info("No image(s) kept but tarring logs for downlink.")
                tar_path = '{TG}/opssat_smartcam_{FILE_EXT}_exp{expID}_{D}.{TAR_EXT}'.format(\
                    TG=TOGROUND_PATH,\
                    FILE_EXT='logs',\
                    expID=EXP_ID,\
                    D=experiment_start_time.strftime("%Y%m%d_%H%M%S") + ("_previous" if files_from_previous_runs else ""),\
                    TAR_EXT=tar_ext)
                os.system('tar {TAR_O} {TAR_PATH} {L}/*.log {L}/*.csv --remove-files'.format(\
                    TAR_O=tar_options,\
                    TAR_PATH=tar_path,\
                    L=LOG_PATH))
                return tar_path
            else:
                if do_logging:
                    logger.info("No images(s) kept nor logs produced for downlink.")
                return None
        except:
            if do_logging:
                logger.exception("Failed to tar kept image for downlink (if any).")
            return None
def package_files_for_downlinking ( self , file_ext , downlink_log_if_no_images , do_clustering , experiment_start_time , files_from_previous_runs , do_logging ) : try : tar_options = '-cf' if file_ext in SUPPORTED_COMPRESSION_TYPES else '-czf' tar_ext = 'tar' if file_ext in SUPPORTED_COMPRESSION_TYPES else 'tar.gz' tar_path = '{TG}/opssat_smartcam_{FILE_EXT}_exp{expID}_{D}.{TAR_EXT}' . format ( TG = TOGROUND_PATH , FILE_EXT = file_ext , expID = EXP_ID , D = experiment_start_time . strftime ( "%Y%m%d_%H%M%S" ) + ( "_previous" if files_from_previous_runs else "" ) , TAR_EXT = tar_ext ) image_count = len ( list ( Path ( TOGROUND_PATH ) . rglob ( '*.' + file_ext ) ) ) log_count = len ( list ( Path ( LOG_PATH ) . rglob ( '*.log' ) ) ) if image_count > 0 : if do_logging : logger . info ( "Tarring {T} file(s) for downlink." . format ( T = image_count ) ) img_files_to_tar = '{G}/**/*.{FILE_EXT}' . format ( G = TOGROUND_PATH , FILE_EXT = file_ext ) if do_clustering : img_files_to_tar = img_files_to_tar + ' {G}/**/**/*.{FILE_EXT}' . format ( G = TOGROUND_PATH , FILE_EXT = file_ext ) tar_cmd = 'tar {TAR_O} {TAR_PATH} ' . format ( TAR_O = tar_options , TAR_PATH = tar_path ) + img_files_to_tar + ' {L}/*.log {L}/*.csv --remove-files' . format ( L = LOG_PATH ) os . system ( tar_cmd ) return tar_path elif downlink_log_if_no_images is True and log_count > 0 : if do_logging : logger . info ( "No image(s) kept but tarring logs for downlink." ) tar_path = '{TG}/opssat_smartcam_{FILE_EXT}_exp{expID}_{D}.{TAR_EXT}' . format ( TG = TOGROUND_PATH , FILE_EXT = 'logs' , expID = EXP_ID , D = experiment_start_time . strftime ( "%Y%m%d_%H%M%S" ) + ( "_previous" if files_from_previous_runs else "" ) , TAR_EXT = tar_ext ) os . system ( 'tar {TAR_O} {TAR_PATH} {L}/*.log {L}/*.csv --remove-files' . format ( TAR_O = tar_options , TAR_PATH = tar_path , L = LOG_PATH ) ) return tar_path else : if do_logging : logger . info ( "No images(s) kept nor logs produced for downlink." ) return None except : if do_logging : logger . exception ( "Failed to tar kept image for downlink (if any)." ) return None
Package the files for downlinking.
Package the files for downlinking .
"""Package the files for downlinking.
        
        Logging is optional via the do_logging flag in case we start the experiment by tarring files leftover from a previous run that was abruptly interrupted.
        In that case we don't want to prematurely write a new log file for the current experiment run or else it's going to end up being tarred with the previous experiment run(s).
        """ # Don't use gzip if files are already a compression file type. # Check against a list of file types in case multiple compression types are supported. # The destination tar file path for the packaged files. # Count how many images were kept and moved to the experiment's toGround folder. # Count how many log files were produced. # Tar images(s) for downlink if at least 1 image was classified with a label of interest. # Log that we are tarring some images. # The paths of the image files to tar depends on whether or not we are clustering. # Include cluster subfolders. # Use tar to package image and log files into the filestore's toGround folder. # Return experiment toGround path to tar file. # Log that we are only tarring log files. # The destination tar file path for the packaged files. # Use tar to package log files into the filestore's toGround folder. # Return experiment toGround path to tar file. # No images and no logs. Unlikely. # Return None. # In case this happens, the image will be tarred at the end of the next experiment's run unless explicitely deleted. # Return None.
self
file_ext
downlink_log_if_no_images
do_clustering
experiment_start_time
files_from_previous_runs
do_logging