leaderboard/leaderboard/leaderboard_evaluator.py
def _load_and_run_scenario(self, args, config):
        """
        Load and run the scenario given by config.

        Depending on what code fails, the simulation will either stop the route and
        continue from the next one, or report a crash and stop.
        """
        crash_message = ""
        entry_status = "Started"

        print("\n\033[1m========= Preparing {} (repetition {}) =========".format(config.name, config.repetition_index))
        print("> Setting up the agent\033[0m")

        # Prepare the statistics of the route
        self.statistics_manager.set_route(config.name, config.index)

        # Set up the user's agent, and the timer to avoid freezing the simulation
        try:
            self._agent_watchdog.start()
            agent_class_name = getattr(self.module_agent, 'get_entry_point')()
            self.agent_instance = getattr(self.module_agent, agent_class_name)(args.agent_config)
            config.agent = self.agent_instance

            # Check and store the sensors
            if not self.sensors:
                self.sensors = self.agent_instance.sensors()
                track = self.agent_instance.track

                AgentWrapper.validate_sensor_configuration(self.sensors, track, args.track)

                self.sensor_icons = [sensors_to_icons[sensor['type']] for sensor in self.sensors]
                self.statistics_manager.save_sensors(self.sensor_icons, args.checkpoint)

            self._agent_watchdog.stop()

        except SensorConfigurationInvalid as e:
            # The sensors are invalid -> set the ejecution to rejected and stop
            print("\n\033[91mThe sensor's configuration used is invalid:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Agent's sensors were invalid"
            entry_status = "Rejected"

            self._register_statistics(config, args.checkpoint, entry_status, crash_message)
            self._cleanup()
            sys.exit(-1)

        except Exception as e:
            # The agent setup has failed -> start the next route
            print("\n\033[91mCould not set up the required agent:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Agent couldn't be set up"

            self._register_statistics(config, args.checkpoint, entry_status, crash_message)
            self._cleanup()
            return

        print("\033[1m> Loading the world\033[0m")

        # Load the world and the scenario
        try:
            self._load_and_wait_for_world(args, config.town, config.ego_vehicles)
            self._prepare_ego_vehicles(config.ego_vehicles, False)
            scenario = self._scenario_class(world=self.world, config=config, debug_mode=args.debug)
            self.statistics_manager.set_scenario(scenario.scenario)

            # Night mode
            if config.weather.sun_altitude_angle < 0.0:
                for vehicle in scenario.ego_vehicles:
                    vehicle.set_light_state(carla.VehicleLightState(self._vehicle_lights))

            # Load scenario and run it
            if args.record:
                self.client.start_recorder("{}/{}_rep{}.log".format(args.record, config.name, config.repetition_index))
            self.manager.load_scenario(scenario, self.agent_instance, config.repetition_index)

        except Exception as e:
            # The scenario is wrong -> set the ejecution to crashed and stop
            print("\n\033[91mThe scenario could not be loaded:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Simulation crashed"
            entry_status = "Crashed"

            self._register_statistics(config, args.checkpoint, entry_status, crash_message)

            if args.record:
                self.client.stop_recorder()

            self._cleanup()
            sys.exit(-1)

        print("\033[1m> Running the route\033[0m")

        # Run the scenario
        try:
            self.manager.run_scenario()

        except AgentError as e:
            # The agent has failed -> stop the route
            print("\n\033[91mStopping the route, the agent has crashed:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Agent crashed"

        except Exception as e:
            print("\n\033[91mError during the simulation:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Simulation crashed"
            entry_status = "Crashed"

        # Stop the scenario
        try:
            print("\033[1m> Stopping the route\033[0m")
            self.manager.stop_scenario()
            self._register_statistics(config, args.checkpoint, entry_status, crash_message)

            if args.record:
                self.client.stop_recorder()

            # Remove all actors
            scenario.remove_all_actors()

            self._cleanup()

        except Exception as e:
            print("\n\033[91mFailed to stop the scenario, the statistics might be empty:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()

            crash_message = "Simulation crashed"

        if crash_message == "Simulation crashed":
            sys.exit(-1)

        Load and run the scenario given by config.

        Depending on what code fails, the simulation will either stop the route and
        continue from the next one, or report a crash and stop.
        
Load and run the scenario given by config.
Depending on what code fails, the simulation will either stop the route and
continue from the next one, or report a crash and stop.
Load and run the scenario given by config . Depending on what code fails the simulation will either stop the route and continue from the next one or report a crash and stop .
def _load_and_run_scenario(self, args, config):
        crash_message = ""
        entry_status = "Started"
        print("\n\033[1m========= Preparing {} (repetition {}) =========".format(config.name, config.repetition_index))
        print("> Setting up the agent\033[0m")
        self.statistics_manager.set_route(config.name, config.index)
        try:
            self._agent_watchdog.start()
            agent_class_name = getattr(self.module_agent, 'get_entry_point')()
            self.agent_instance = getattr(self.module_agent, agent_class_name)(args.agent_config)
            config.agent = self.agent_instance
            if not self.sensors:
                self.sensors = self.agent_instance.sensors()
                track = self.agent_instance.track
                AgentWrapper.validate_sensor_configuration(self.sensors, track, args.track)
                self.sensor_icons = [sensors_to_icons[sensor['type']] for sensor in self.sensors]
                self.statistics_manager.save_sensors(self.sensor_icons, args.checkpoint)
            self._agent_watchdog.stop()
        except SensorConfigurationInvalid as e:
            print("\n\033[91mThe sensor's configuration used is invalid:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()
            crash_message = "Agent's sensors were invalid"
            entry_status = "Rejected"
            self._register_statistics(config, args.checkpoint, entry_status, crash_message)
            self._cleanup()
            sys.exit(-1)
        except Exception as e:
            print("\n\033[91mCould not set up the required agent:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()
            crash_message = "Agent couldn't be set up"
            self._register_statistics(config, args.checkpoint, entry_status, crash_message)
            self._cleanup()
            return
        print("\033[1m> Loading the world\033[0m")
        try:
            self._load_and_wait_for_world(args, config.town, config.ego_vehicles)
            self._prepare_ego_vehicles(config.ego_vehicles, False)
            scenario = self._scenario_class(world=self.world, config=config, debug_mode=args.debug)
            self.statistics_manager.set_scenario(scenario.scenario)
            if config.weather.sun_altitude_angle < 0.0:
                for vehicle in scenario.ego_vehicles:
                    vehicle.set_light_state(carla.VehicleLightState(self._vehicle_lights))
            if args.record:
                self.client.start_recorder("{}/{}_rep{}.log".format(args.record, config.name, config.repetition_index))
            self.manager.load_scenario(scenario, self.agent_instance, config.repetition_index)
        except Exception as e:
            print("\n\033[91mThe scenario could not be loaded:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()
            crash_message = "Simulation crashed"
            entry_status = "Crashed"
            self._register_statistics(config, args.checkpoint, entry_status, crash_message)
            if args.record:
                self.client.stop_recorder()
            self._cleanup()
            sys.exit(-1)
        print("\033[1m> Running the route\033[0m")
        try:
            self.manager.run_scenario()
        except AgentError as e:
            print("\n\033[91mStopping the route, the agent has crashed:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()
            crash_message = "Agent crashed"
        except Exception as e:
            print("\n\033[91mError during the simulation:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()
            crash_message = "Simulation crashed"
            entry_status = "Crashed"
        try:
            print("\033[1m> Stopping the route\033[0m")
            self.manager.stop_scenario()
            self._register_statistics(config, args.checkpoint, entry_status, crash_message)
            if args.record:
                self.client.stop_recorder()
            scenario.remove_all_actors()
            self._cleanup()
        except Exception as e:
            print("\n\033[91mFailed to stop the scenario, the statistics might be empty:")
            print("> {}\033[0m\n".format(e))
            traceback.print_exc()
            crash_message = "Simulation crashed"
        if crash_message == "Simulation crashed":
            sys.exit(-1)
def _load_and_run_scenario ( self , args , config ) : crash_message = "" entry_status = "Started" print ( "\n\033[1m========= Preparing {} (repetition {}) =========" . format ( config . name , config . repetition_index ) ) print ( "> Setting up the agent\033[0m" ) self . statistics_manager . set_route ( config . name , config . index ) try : self . _agent_watchdog . start ( ) agent_class_name = getattr ( self . module_agent , 'get_entry_point' ) ( ) self . agent_instance = getattr ( self . module_agent , agent_class_name ) ( args . agent_config ) config . agent = self . agent_instance if not self . sensors : self . sensors = self . agent_instance . sensors ( ) track = self . agent_instance . track AgentWrapper . validate_sensor_configuration ( self . sensors , track , args . track ) self . sensor_icons = [ sensors_to_icons [ sensor [ 'type' ] ] for sensor in self . sensors ] self . statistics_manager . save_sensors ( self . sensor_icons , args . checkpoint ) self . _agent_watchdog . stop ( ) except SensorConfigurationInvalid as e : print ( "\n\033[91mThe sensor's configuration used is invalid:" ) print ( "> {}\033[0m\n" . format ( e ) ) traceback . print_exc ( ) crash_message = "Agent's sensors were invalid" entry_status = "Rejected" self . _register_statistics ( config , args . checkpoint , entry_status , crash_message ) self . _cleanup ( ) sys . exit ( - 1 ) except Exception as e : print ( "\n\033[91mCould not set up the required agent:" ) print ( "> {}\033[0m\n" . format ( e ) ) traceback . print_exc ( ) crash_message = "Agent couldn't be set up" self . _register_statistics ( config , args . checkpoint , entry_status , crash_message ) self . _cleanup ( ) return print ( "\033[1m> Loading the world\033[0m" ) try : self . _load_and_wait_for_world ( args , config . town , config . ego_vehicles ) self . _prepare_ego_vehicles ( config . ego_vehicles , False ) scenario = self . _scenario_class ( world = self . world , config = config , debug_mode = args . debug ) self . statistics_manager . set_scenario ( scenario . scenario ) if config . weather . sun_altitude_angle < 0.0 : for vehicle in scenario . ego_vehicles : vehicle . set_light_state ( carla . VehicleLightState ( self . _vehicle_lights ) ) if args . record : self . client . start_recorder ( "{}/{}_rep{}.log" . format ( args . record , config . name , config . repetition_index ) ) self . manager . load_scenario ( scenario , self . agent_instance , config . repetition_index ) except Exception as e : print ( "\n\033[91mThe scenario could not be loaded:" ) print ( "> {}\033[0m\n" . format ( e ) ) traceback . print_exc ( ) crash_message = "Simulation crashed" entry_status = "Crashed" self . _register_statistics ( config , args . checkpoint , entry_status , crash_message ) if args . record : self . client . stop_recorder ( ) self . _cleanup ( ) sys . exit ( - 1 ) print ( "\033[1m> Running the route\033[0m" ) try : self . manager . run_scenario ( ) except AgentError as e : print ( "\n\033[91mStopping the route, the agent has crashed:" ) print ( "> {}\033[0m\n" . format ( e ) ) traceback . print_exc ( ) crash_message = "Agent crashed" except Exception as e : print ( "\n\033[91mError during the simulation:" ) print ( "> {}\033[0m\n" . format ( e ) ) traceback . print_exc ( ) crash_message = "Simulation crashed" entry_status = "Crashed" try : print ( "\033[1m> Stopping the route\033[0m" ) self . manager . stop_scenario ( ) self . _register_statistics ( config , args . checkpoint , entry_status , crash_message ) if args . record : self . client . stop_recorder ( ) scenario . remove_all_actors ( ) self . _cleanup ( ) except Exception as e : print ( "\n\033[91mFailed to stop the scenario, the statistics might be empty:" ) print ( "> {}\033[0m\n" . format ( e ) ) traceback . print_exc ( ) crash_message = "Simulation crashed" if crash_message == "Simulation crashed" : sys . exit ( - 1 )
Load and run the scenario given by config.
Load and run the scenario given by config .
"""
        Load and run the scenario given by config.

        Depending on what code fails, the simulation will either stop the route and
        continue from the next one, or report a crash and stop.
        """ # Prepare the statistics of the route # Set up the user's agent, and the timer to avoid freezing the simulation # Check and store the sensors # The sensors are invalid -> set the ejecution to rejected and stop # The agent setup has failed -> start the next route # Load the world and the scenario # Night mode # Load scenario and run it # The scenario is wrong -> set the ejecution to crashed and stop # Run the scenario # The agent has failed -> stop the route # Stop the scenario # Remove all actors
self
args
config