gbdx_cloud_harness/controller.py
def _run_app(self):
        """
        Method for running a custom Application Templates.
        NOTES:
            * The default name of the application is app.py. So this function is going to look
            for app.py, unless the --file option is provide with a different file name.
            * The generated source bundle will package everything in the work_path. If large files
            not required for the application source, they need to be ignored. Use a file called "pkg_ignore"
            to identify folders and files to ignore.
        USAGE: cloud-harness run <file_name> [--remote] [--verbose] [--upload] [--download] [--dry-run]
        """
        is_remote_run = self._arguments.get('--remote')
        filename = self._arguments.get('<file_name>')
        upload_ports = self._arguments.get('--upload')
        download_ports = self._arguments.get('--download')
        is_verbose = self._arguments.get('--verbose')
        # A dry run means, allow port sot be pushed up, but don't allow execution and monitoring.
        is_dry_run = self._arguments.get('--dry-run')

        if download_ports:  # TODO temporary until implemented.
            raise NotImplementedError("Downloading of output ports is not implemented yet.")

        # Check if the filename passed is actually a class object (gbdxtools functionality)
        if not isinstance(filename, str) and issubclass(filename, TaskTemplate):
            template_class = filename
            template_file = inspect.getfile(template_class)
            config_file = self._write_config_file(template_file)

        else:
            template_file = self._get_template_abs_path(filename)

            if not os.path.isfile(template_file):
                raise ValueError('The location %s does not exist' % template_file)

            config_file = self._write_config_file(template_file)

            template_class = self._get_class(template_file)

        with template_class() as template:
            if is_remote_run:  # Means the user is running with --remote, push to S3 and submit workflow.

                task = template.task

                # Set the source bundle directory to where the template_file is.
                task.source_bundle.value = os.path.join(os.path.dirname(template_file), 'tmp_%s' % str(uuid.uuid4()))

                task.run_name = '{task_name}_src'.format(task_name=task.name)

                src_bundle_dir = task.source_bundle.value

                # Create source bundle to be executed on the GBDX platform
                self._archive_source(os.path.dirname(src_bundle_dir), src_bundle_dir)

                port_service = PortService(task)

                if upload_ports:
                    # Push all port data to S3
                    port_service.upload_input_ports()
                else:
                    # Only push source bundle port
                    port_service.upload_input_ports(port_list=[self.SOURCE_BUNDLE_PORT])

                # Delete source bundle directory and config after upload.
                shutil.rmtree(src_bundle_dir)
                os.remove(config_file)

                # Get the new task object with uploaded port locations.
                task = port_service.task

                # Validate task
                task.is_valid(remote=True)

                workflow = Workflow(task)

                if is_verbose:
                    temp_wf = workflow.json
                    printer(temp_wf)

                if is_dry_run:
                    return task

                try:
                    workflow.execute()
                    printer(workflow.id)
                except Exception as e:
                    printer(e.message)
                    template.reason = "Execution Failed: %s" % e.message
                    return

                # Monitor events of workflow
                is_done = workflow.monitor_run()

                if not is_done:
                    template.reason = "Execution Failed during Run"

                if download_ports:
                    # TODO port_service.download_output_port()
                    pass

            else:
                # For local and Docker container execution.

                # Check that all output locations exist.
                template.check_and_create_outputs()

                # Validate task
                template.task.is_valid()

                if is_verbose:
                    printer(template.task.json())
                    all_ports = template.task.ports[0] + template.task.ports[1]
                    printer([port.__str__() for port in all_ports])

                if is_dry_run:
                    template.reason = "Execution Skipped"
                    return

                # Run Task Locally
                try:
                    template.invoke()
                except Exception as e:
                    template.reason = "Failed Exception: %s" % e

                if template.reason is None or template.reason == '':
                    template.reason = "Execution Completed"
def _run_app(self):
        """
        Method for running a custom Application Templates.
        NOTES:
            * The default name of the application is app.py. So this function is going to look
            for app.py, unless the --file option is provide with a different file name.
            * The generated source bundle will package everything in the work_path. If large files
            not required for the application source, they need to be ignored. Use a file called "pkg_ignore"
            to identify folders and files to ignore.
        USAGE: cloud-harness run <file_name> [--remote] [--verbose] [--upload] [--download] [--dry-run]
        """
        is_remote_run = self._arguments.get('--remote')
        filename = self._arguments.get('<file_name>')
        upload_ports = self._arguments.get('--upload')
        download_ports = self._arguments.get('--download')
        is_verbose = self._arguments.get('--verbose')
        # A dry run means, allow port sot be pushed up, but don't allow execution and monitoring.
        is_dry_run = self._arguments.get('--dry-run')

        if download_ports:  # TODO temporary until implemented.
            raise NotImplementedError("Downloading of output ports is not implemented yet.")

        # Check if the filename passed is actually a class object (gbdxtools functionality)
        if not isinstance(filename, str) and issubclass(filename, TaskTemplate):
            template_class = filename
            template_file = inspect.getfile(template_class)
            config_file = self._write_config_file(template_file)

        else:
            template_file = self._get_template_abs_path(filename)

            if not os.path.isfile(template_file):
                raise ValueError('The location %s does not exist' % template_file)

            config_file = self._write_config_file(template_file)

            template_class = self._get_class(template_file)

        with template_class() as template:
            if is_remote_run:  # Means the user is running with --remote, push to S3 and submit workflow.

                task = template.task

                # Set the source bundle directory to where the template_file is.
                task.source_bundle.value = os.path.join(os.path.dirname(template_file), 'tmp_%s' % str(uuid.uuid4()))

                task.run_name = '{task_name}_src'.format(task_name=task.name)

                src_bundle_dir = task.source_bundle.value

                # Create source bundle to be executed on the GBDX platform
                self._archive_source(os.path.dirname(src_bundle_dir), src_bundle_dir)

                port_service = PortService(task)

                if upload_ports:
                    # Push all port data to S3
                    port_service.upload_input_ports()
                else:
                    # Only push source bundle port
                    port_service.upload_input_ports(port_list=[self.SOURCE_BUNDLE_PORT])

                # Delete source bundle directory and config after upload.
                shutil.rmtree(src_bundle_dir)
                os.remove(config_file)

                # Get the new task object with uploaded port locations.
                task = port_service.task

                # Validate task
                task.is_valid(remote=True)

                workflow = Workflow(task)

                if is_verbose:
                    temp_wf = workflow.json
                    printer(temp_wf)

                if is_dry_run:
                    return task

                try:
                    workflow.execute()
                    printer(workflow.id)
                except Exception as e:
                    printer(e.message)
                    template.reason = "Execution Failed: %s" % e.message
                    return

                # Monitor events of workflow
                is_done = workflow.monitor_run()

                if not is_done:
                    template.reason = "Execution Failed during Run"

                if download_ports:
                    # TODO port_service.download_output_port()
                    pass

            else:
                # For local and Docker container execution.

                # Check that all output locations exist.
                template.check_and_create_outputs()

                # Validate task
                template.task.is_valid()

                if is_verbose:
                    printer(template.task.json())
                    all_ports = template.task.ports[0] + template.task.ports[1]
                    printer([port.__str__() for port in all_ports])

                if is_dry_run:
                    template.reason = "Execution Skipped"
                    return

                # Run Task Locally
                try:
                    template.invoke()
                except Exception as e:
                    template.reason = "Failed Exception: %s" % e

                if template.reason is None or template.reason == '':
                    template.reason = "Execution Completed"
def _run_app ( self ) : is_remote_run = self . _arguments . get ( '--remote' ) filename = self . _arguments . get ( '<file_name>' ) upload_ports = self . _arguments . get ( '--upload' ) download_ports = self . _arguments . get ( '--download' ) is_verbose = self . _arguments . get ( '--verbose' ) # A dry run means, allow port sot be pushed up, but don't allow execution and monitoring. is_dry_run = self . _arguments . get ( '--dry-run' ) if download_ports : # TODO temporary until implemented. raise NotImplementedError ( "Downloading of output ports is not implemented yet." ) # Check if the filename passed is actually a class object (gbdxtools functionality) if not isinstance ( filename , str ) and issubclass ( filename , TaskTemplate ) : template_class = filename template_file = inspect . getfile ( template_class ) config_file = self . _write_config_file ( template_file ) else : template_file = self . _get_template_abs_path ( filename ) if not os . path . isfile ( template_file ) : raise ValueError ( 'The location %s does not exist' % template_file ) config_file = self . _write_config_file ( template_file ) template_class = self . _get_class ( template_file ) with template_class ( ) as template : if is_remote_run : # Means the user is running with --remote, push to S3 and submit workflow. task = template . task # Set the source bundle directory to where the template_file is. task . source_bundle . value = os . path . join ( os . path . dirname ( template_file ) , 'tmp_%s' % str ( uuid . uuid4 ( ) ) ) task . run_name = '{task_name}_src' . format ( task_name = task . name ) src_bundle_dir = task . source_bundle . value # Create source bundle to be executed on the GBDX platform self . _archive_source ( os . path . dirname ( src_bundle_dir ) , src_bundle_dir ) port_service = PortService ( task ) if upload_ports : # Push all port data to S3 port_service . upload_input_ports ( ) else : # Only push source bundle port port_service . upload_input_ports ( port_list = [ self . SOURCE_BUNDLE_PORT ] ) # Delete source bundle directory and config after upload. shutil . rmtree ( src_bundle_dir ) os . remove ( config_file ) # Get the new task object with uploaded port locations. task = port_service . task # Validate task task . is_valid ( remote = True ) workflow = Workflow ( task ) if is_verbose : temp_wf = workflow . json printer ( temp_wf ) if is_dry_run : return task try : workflow . execute ( ) printer ( workflow . id ) except Exception as e : printer ( e . message ) template . reason = "Execution Failed: %s" % e . message return # Monitor events of workflow is_done = workflow . monitor_run ( ) if not is_done : template . reason = "Execution Failed during Run" if download_ports : # TODO port_service.download_output_port() pass else : # For local and Docker container execution. # Check that all output locations exist. template . check_and_create_outputs ( ) # Validate task template . task . is_valid ( ) if is_verbose : printer ( template . task . json ( ) ) all_ports = template . task . ports [ 0 ] + template . task . ports [ 1 ] printer ( [ port . __str__ ( ) for port in all_ports ] ) if is_dry_run : template . reason = "Execution Skipped" return # Run Task Locally try : template . invoke ( ) except Exception as e : template . reason = "Failed Exception: %s" % e if template . reason is None or template . reason == '' : template . reason = "Execution Completed"
Method for running a custom Application Templates.
        NOTES:
            * The default name of the application is app.py. So this function is going to look
            for app.py, unless the --file option is provide with a different file name.
            * The generated source bundle will package everything in the work_path. If large files
            not required for the application source, they need to be ignored. Use a file called "pkg_ignore"
            to identify folders and files to ignore.
        USAGE: cloud-harness run <file_name> [--remote] [--verbose] [--upload] [--download] [--dry-run]
Method for running a custom Application Templates . NOTES : * The default name of the application is app . py . So this function is going to look for app . py unless the -- file option is provide with a different file name . * The generated source bundle will package everything in the work_path . If large files not required for the application source they need to be ignored . Use a file called pkg_ignore to identify folders and files to ignore . USAGE : cloud - harness run <file_name > [ -- remote ] [ -- verbose ] [ -- upload ] [ -- download ] [ -- dry - run ]