python/tvm/driver/tvmc/runner.py
def run_module(
    tvmc_package: TVMCPackage,
    device: str,
    hostname: Optional[str] = None,
    port: Union[int, str] = 9090,
    rpc_key: Optional[str] = None,
    inputs: Optional[Dict[str, np.ndarray]] = None,
    fill_mode: str = "random",
    repeat: int = 10,
    number: int = 10,
    profile: bool = False,
    options: dict = None,
):
    """Run a compiled graph executor module locally or remotely with
    optional input values.

    If input tensors are not specified explicitly, they can be filled
    with zeroes, ones or random data.

    Parameters
    ----------
    tvmc_package: TVMCPackage
        The compiled model package object that will be run.
    device: str,
        the device (e.g. "cpu" or "cuda") to be targeted by the RPC
        session, local or remote).
    hostname : str, optional
        The hostname of the target device on which to run.
    port : int, optional
        The port of the target device on which to run.
    rpc_key : str, optional
        The tracker key of the target device. If this is set, it
        will be assumed that remote points to a tracker.
    inputs : dict, optional
        A dictionary that maps input names to numpy values. If not provided,
        inputs will be generated using the fill_mode argument.
    fill_mode : str, optional
        The fill-mode to use when generating data for input tensors.
        Valid options are "zeros", "ones" and "random".
        Defaults to "random".
    repeat : int, optional
        How many times to repeat the run.
    number : int, optional
        The number of runs to measure within each repeat.
    profile : bool
        Whether to profile the run with the debug runtime.

    Returns
    -------
    outputs : dict
        a dictionary with output tensors, generated by the module
    times : list of str
        execution times generated by the time evaluator
    """
    if not isinstance(tvmc_package, TVMCPackage):
        raise TVMCException(
            "This model doesn't seem to have been compiled yet. "
            "Try calling tvmc.compile on the model before running it."
        )

    with ExitStack() as stack:
        # Currently only two package formats are supported: "classic" and
        # "mlf". The later can only be used for micro targets, i.e. with microTVM.
        if device == "micro":
            if tvmc_package.type != "mlf":
                raise TVMCException(f"Model {tvmc_package.package_path} is not a MLF archive.")

            project_dir = os.path.dirname(tvmc_package.package_path)

            # This is guaranteed to work since project_dir was already checked when
            # building the dynamic parser to accommodate the project options, so no
            # checks are in place when calling GeneratedProject.
            project_ = project.GeneratedProject.from_directory(project_dir, options)
        else:
            if tvmc_package.type == "mlf":
                raise TVMCException(
                    "You're trying to run a model saved using the Model Library Format (MLF). "
                    "MLF can only be used to run micro device ('--device micro')."
                )

        if hostname:
            if isinstance(port, str):
                port = int(port)
            # Remote RPC
            if rpc_key:
                logger.debug("Running on remote RPC tracker with key %s.", rpc_key)
                session = request_remote(rpc_key, hostname, port, timeout=1000)
            else:
                logger.debug("Running on remote RPC with no key.")
                session = rpc.connect(hostname, port)
        elif device == "micro":
            # Remote RPC (running on a micro target)
            logger.debug("Running on remote RPC (micro target).")
            try:
                session = tvm.micro.Session(project_.transport())
                stack.enter_context(session)
            except:
                raise TVMCException("Could not open a session with the micro target.")
        else:
            # Local
            logger.debug("Running a local session.")
            session = rpc.LocalSession()

        # Micro targets don't support uploading a model. The model to be run
        # must be already flashed into the micro target before one tries
        # to run it. Hence skip model upload for micro targets.
        if device != "micro":
            session.upload(tvmc_package.lib_path)
            lib = session.load_module(tvmc_package.lib_name)

        # TODO expand to other supported devices, as listed in tvm.rpc.client (@leandron)
        logger.debug("Device is %s.", device)
        if device == "cuda":
            dev = session.cuda()
        elif device == "cl":
            dev = session.cl()
        elif device == "metal":
            dev = session.metal()
        elif device == "vulkan":
            dev = session.vulkan()
        elif device == "rocm":
            dev = session.rocm()
        elif device == "micro":
            dev = session.device
            lib = session.get_system_lib()
        else:
            assert device == "cpu"
            dev = session.cpu()

        # TODO(gromero): Adjust for micro targets.
        if profile:
            logger.debug("Creating runtime with profiling enabled.")
            module = debug_executor.create(tvmc_package.graph, lib, dev, dump_root="./prof")
        else:
            if device == "micro":
                logger.debug("Creating runtime (micro) with profiling disabled.")
                module = tvm.micro.create_local_graph_executor(tvmc_package.graph, lib, dev)
            else:
                logger.debug("Creating runtime with profiling disabled.")
                module = runtime.create(tvmc_package.graph, lib, dev)

        logger.debug("Loading params into the runtime module.")
        module.load_params(tvmc_package.params)

        shape_dict, dtype_dict = get_input_info(tvmc_package.graph, tvmc_package.params)
        inputs_dict = make_inputs_dict(shape_dict, dtype_dict, inputs, fill_mode)

        logger.debug("Setting inputs to the module.")
        module.set_input(**inputs_dict)

        # Run must be called explicitly if profiling
        if profile:
            logger.info("Running the module with profiling enabled.")
            report = module.profile()
            # This print is intentional
            print(report)

        if device == "micro":
            # TODO(gromero): Fix time_evaluator() for micro targets. Once it's
            # fixed module.benchmark() can be used instead and this if/else can
            # be removed.
            module.run()
            times = []
        else:
            # call the benchmarking function of the executor
            times = module.benchmark(dev, number=number, repeat=repeat)

        logger.debug("Collecting the output tensors.")
        num_outputs = module.get_num_outputs()
        outputs = {}
        for i in range(num_outputs):
            output_name = "output_{}".format(i)
            outputs[output_name] = module.get_output(i).numpy()

        return TVMCResult(outputs, times)
Run a compiled graph executor module locally or remotely with
    optional input values.

    If input tensors are not specified explicitly, they can be filled
    with zeroes, ones or random data.

    Parameters
    ----------
    tvmc_package: TVMCPackage
        The compiled model package object that will be run.
    device: str,
        the device (e.g. "cpu" or "cuda") to be targeted by the RPC
        session, local or remote).
    hostname : str, optional
        The hostname of the target device on which to run.
    port : int, optional
        The port of the target device on which to run.
    rpc_key : str, optional
        The tracker key of the target device. If this is set, it
        will be assumed that remote points to a tracker.
    inputs : dict, optional
        A dictionary that maps input names to numpy values. If not provided,
        inputs will be generated using the fill_mode argument.
    fill_mode : str, optional
        The fill-mode to use when generating data for input tensors.
        Valid options are "zeros", "ones" and "random".
        Defaults to "random".
    repeat : int, optional
        How many times to repeat the run.
    number : int, optional
        The number of runs to measure within each repeat.
    profile : bool
        Whether to profile the run with the debug runtime.

    Returns
    -------
    outputs : dict
        a dictionary with output tensors, generated by the module
    times : list of str
        execution times generated by the time evaluator
    
Run a compiled graph executor module locally or remotely with
optional input values.

If input tensors are not specified explicitly, they can be filled
with zeroes, ones or random data.

Parameters

TVMCPackage
The compiled model package object that will be run.
device: str,
the device  to be targeted by the RPC
session, local or remote).
hostname : str, optional
The hostname of the target device on which to run.
port : int, optional
The port of the target device on which to run.
rpc_key : str, optional
The tracker key of the target device. If this is set, it
will be assumed that remote points to a tracker.
inputs : dict, optional
A dictionary that maps input names to numpy values. If not provided,
inputs will be generated using the fill_mode argument.
fill_mode : str, optional
The fill-mode to use when generating data for input tensors.

Returns

outputs : dict
a dictionary with output tensors, generated by the module
times : list of str
execution times generated by the time evaluator
Run a compiled graph executor module locally or remotely with optional input values . If input tensors are not specified explicitly they can be filled with zeroes ones or random data . Parameters TVMCPackage The compiled model package object that will be run . device : str the device to be targeted by the RPC session local or remote ) . hostname : str optional The hostname of the target device on which to run . port : int optional The port of the target device on which to run . rpc_key : str optional The tracker key of the target device . If this is set it will be assumed that remote points to a tracker . inputs : dict optional A dictionary that maps input names to numpy values . If not provided inputs will be generated using the fill_mode argument . fill_mode : str optional The fill - mode to use when generating data for input tensors . Returns outputs : dict a dictionary with output tensors generated by the module times : list of str execution times generated by the time evaluator
def run_module(
    tvmc_package: TVMCPackage,
    device: str,
    hostname: Optional[str] = None,
    port: Union[int, str] = 9090,
    rpc_key: Optional[str] = None,
    inputs: Optional[Dict[str, np.ndarray]] = None,
    fill_mode: str = "random",
    repeat: int = 10,
    number: int = 10,
    profile: bool = False,
    options: dict = None,
):
    if not isinstance(tvmc_package, TVMCPackage):
        raise TVMCException(
            "This model doesn't seem to have been compiled yet. "
            "Try calling tvmc.compile on the model before running it."
        )
    with ExitStack() as stack:
        if device == "micro":
            if tvmc_package.type != "mlf":
                raise TVMCException(f"Model {tvmc_package.package_path} is not a MLF archive.")
            project_dir = os.path.dirname(tvmc_package.package_path)
            project_ = project.GeneratedProject.from_directory(project_dir, options)
        else:
            if tvmc_package.type == "mlf":
                raise TVMCException(
                    "You're trying to run a model saved using the Model Library Format (MLF). "
                    "MLF can only be used to run micro device ('--device micro')."
                )
        if hostname:
            if isinstance(port, str):
                port = int(port)
            if rpc_key:
                logger.debug("Running on remote RPC tracker with key %s.", rpc_key)
                session = request_remote(rpc_key, hostname, port, timeout=1000)
            else:
                logger.debug("Running on remote RPC with no key.")
                session = rpc.connect(hostname, port)
        elif device == "micro":
             (running on a micro target)
            logger.debug("Running on remote RPC (micro target).")
            try:
                session = tvm.micro.Session(project_.transport())
                stack.enter_context(session)
            except:
                raise TVMCException("Could not open a session with the micro target.")
        else:
            logger.debug("Running a local session.")
            session = rpc.LocalSession()
        if device != "micro":
            session.upload(tvmc_package.lib_path)
            lib = session.load_module(tvmc_package.lib_name)
        logger.debug("Device is %s.", device)
        if device == "cuda":
            dev = session.cuda()
        elif device == "cl":
            dev = session.cl()
        elif device == "metal":
            dev = session.metal()
        elif device == "vulkan":
            dev = session.vulkan()
        elif device == "rocm":
            dev = session.rocm()
        elif device == "micro":
            dev = session.device
            lib = session.get_system_lib()
        else:
            assert device == "cpu"
            dev = session.cpu()
        if profile:
            logger.debug("Creating runtime with profiling enabled.")
            module = debug_executor.create(tvmc_package.graph, lib, dev, dump_root="./prof")
        else:
            if device == "micro":
                logger.debug("Creating runtime (micro) with profiling disabled.")
                module = tvm.micro.create_local_graph_executor(tvmc_package.graph, lib, dev)
            else:
                logger.debug("Creating runtime with profiling disabled.")
                module = runtime.create(tvmc_package.graph, lib, dev)
        logger.debug("Loading params into the runtime module.")
        module.load_params(tvmc_package.params)
        shape_dict, dtype_dict = get_input_info(tvmc_package.graph, tvmc_package.params)
        inputs_dict = make_inputs_dict(shape_dict, dtype_dict, inputs, fill_mode)
        logger.debug("Setting inputs to the module.")
        module.set_input(**inputs_dict)
        if profile:
            logger.info("Running the module with profiling enabled.")
            report = module.profile()
            print(report)
        if device == "micro":
            module.run()
            times = []
        else:
            times = module.benchmark(dev, number=number, repeat=repeat)
        logger.debug("Collecting the output tensors.")
        num_outputs = module.get_num_outputs()
        outputs = {}
        for i in range(num_outputs):
            output_name = "output_{}".format(i)
            outputs[output_name] = module.get_output(i).numpy()
        return TVMCResult(outputs, times)
def run_module ( tvmc_package : TVMCPackage , device : str , hostname : Optional [ str ] = None , port : Union [ int , str ] = 9090 , rpc_key : Optional [ str ] = None , inputs : Optional [ Dict [ str , np . ndarray ] ] = None , fill_mode : str = "random" , repeat : int = 10 , number : int = 10 , profile : bool = False , options : dict = None , ) : if not isinstance ( tvmc_package , TVMCPackage ) : raise TVMCException ( "This model doesn't seem to have been compiled yet. " "Try calling tvmc.compile on the model before running it." ) with ExitStack ( ) as stack : if device == "micro" : if tvmc_package . type != "mlf" : raise TVMCException ( f"Model {tvmc_package.package_path} is not a MLF archive." ) project_dir = os . path . dirname ( tvmc_package . package_path ) project_ = project . GeneratedProject . from_directory ( project_dir , options ) else : if tvmc_package . type == "mlf" : raise TVMCException ( "You're trying to run a model saved using the Model Library Format (MLF). " "MLF can only be used to run micro device ('--device micro')." ) if hostname : if isinstance ( port , str ) : port = int ( port ) if rpc_key : logger . debug ( "Running on remote RPC tracker with key %s." , rpc_key ) session = request_remote ( rpc_key , hostname , port , timeout = 1000 ) else : logger . debug ( "Running on remote RPC with no key." ) session = rpc . connect ( hostname , port ) elif device == "micro" : logger . debug ( "Running on remote RPC (micro target)." ) try : session = tvm . micro . Session ( project_ . transport ( ) ) stack . enter_context ( session ) except : raise TVMCException ( "Could not open a session with the micro target." ) else : logger . debug ( "Running a local session." ) session = rpc . LocalSession ( ) if device != "micro" : session . upload ( tvmc_package . lib_path ) lib = session . load_module ( tvmc_package . lib_name ) logger . debug ( "Device is %s." , device ) if device == "cuda" : dev = session . cuda ( ) elif device == "cl" : dev = session . cl ( ) elif device == "metal" : dev = session . metal ( ) elif device == "vulkan" : dev = session . vulkan ( ) elif device == "rocm" : dev = session . rocm ( ) elif device == "micro" : dev = session . device lib = session . get_system_lib ( ) else : assert device == "cpu" dev = session . cpu ( ) if profile : logger . debug ( "Creating runtime with profiling enabled." ) module = debug_executor . create ( tvmc_package . graph , lib , dev , dump_root = "./prof" ) else : if device == "micro" : logger . debug ( "Creating runtime (micro) with profiling disabled." ) module = tvm . micro . create_local_graph_executor ( tvmc_package . graph , lib , dev ) else : logger . debug ( "Creating runtime with profiling disabled." ) module = runtime . create ( tvmc_package . graph , lib , dev ) logger . debug ( "Loading params into the runtime module." ) module . load_params ( tvmc_package . params ) shape_dict , dtype_dict = get_input_info ( tvmc_package . graph , tvmc_package . params ) inputs_dict = make_inputs_dict ( shape_dict , dtype_dict , inputs , fill_mode ) logger . debug ( "Setting inputs to the module." ) module . set_input ( ** inputs_dict ) if profile : logger . info ( "Running the module with profiling enabled." ) report = module . profile ( ) print ( report ) if device == "micro" : module . run ( ) times = [ ] else : times = module . benchmark ( dev , number = number , repeat = repeat ) logger . debug ( "Collecting the output tensors." ) num_outputs = module . get_num_outputs ( ) outputs = { } for i in range ( num_outputs ) : output_name = "output_{}" . format ( i ) outputs [ output_name ] = module . get_output ( i ) . numpy ( ) return TVMCResult ( outputs , times )
Run a compiled graph executor module locally or remotely with
optional input values.
Run a compiled graph executor module locally or remotely with optional input values .
"""Run a compiled graph executor module locally or remotely with
    optional input values.

    If input tensors are not specified explicitly, they can be filled
    with zeroes, ones or random data.

    Parameters
    ----------
    tvmc_package: TVMCPackage
        The compiled model package object that will be run.
    device: str,
        the device (e.g. "cpu" or "cuda") to be targeted by the RPC
        session, local or remote).
    hostname : str, optional
        The hostname of the target device on which to run.
    port : int, optional
        The port of the target device on which to run.
    rpc_key : str, optional
        The tracker key of the target device. If this is set, it
        will be assumed that remote points to a tracker.
    inputs : dict, optional
        A dictionary that maps input names to numpy values. If not provided,
        inputs will be generated using the fill_mode argument.
    fill_mode : str, optional
        The fill-mode to use when generating data for input tensors.
        Valid options are "zeros", "ones" and "random".
        Defaults to "random".
    repeat : int, optional
        How many times to repeat the run.
    number : int, optional
        The number of runs to measure within each repeat.
    profile : bool
        Whether to profile the run with the debug runtime.

    Returns
    -------
    outputs : dict
        a dictionary with output tensors, generated by the module
    times : list of str
        execution times generated by the time evaluator
    """ # Currently only two package formats are supported: "classic" and # "mlf". The later can only be used for micro targets, i.e. with microTVM. # This is guaranteed to work since project_dir was already checked when # building the dynamic parser to accommodate the project options, so no # checks are in place when calling GeneratedProject. # Remote RPC # Remote RPC (running on a micro target) # Local # Micro targets don't support uploading a model. The model to be run # must be already flashed into the micro target before one tries # to run it. Hence skip model upload for micro targets. # TODO expand to other supported devices, as listed in tvm.rpc.client (@leandron) # TODO(gromero): Adjust for micro targets. # Run must be called explicitly if profiling # This print is intentional # TODO(gromero): Fix time_evaluator() for micro targets. Once it's # fixed module.benchmark() can be used instead and this if/else can # be removed. # call the benchmarking function of the executor
tvmc_package
TVMCPackage
device
str
hostname
Optional[str]
port
Union[int, str]
rpc_key
Optional[str]
inputs
Optional[Dict[str, np.ndarray]]
fill_mode
str
repeat
int
number
int
profile
bool
options
dict