# (c) Copyright IBM Corporation 2020.

# LICENSE: Apache License 2.0 (Apache-2.0)
# http://www.apache.org/licenses/LICENSE-2.0

import abc
import logging
import time
from collections import defaultdict
from typing import List

import numpy as np
from dataclasses import dataclass

logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s')

import lrtc_lib.data_access.data_access_factory as data_access_factory
import lrtc_lib.experiment_runners.experiments_results_handler as res_handler
from lrtc_lib.oracle_data_access import oracle_data_access_api
from lrtc_lib.active_learning.diversity_calculator import DiversityCalculator
from lrtc_lib.active_learning.knn_outlier_calculator import KnnOutlierCalculator
from lrtc_lib.active_learning.strategies import ActiveLearningStrategies
from lrtc_lib.data_access.core.data_structs import TextElement
from lrtc_lib.data_access.data_access_api import DataAccessApi
from lrtc_lib.data_access.data_access_factory import get_data_access
from lrtc_lib.orchestrator import orchestrator_api
from lrtc_lib.orchestrator.orchestrator_api import DeleteModels
from lrtc_lib.train_and_infer_service.model_type import ModelType
from lrtc_lib.training_set_selector.train_and_dev_set_selector_api import TrainingSetSelectionStrategy


@dataclass
class ExperimentParams:
    experiment_name: str
    train_dataset_name: str
    dev_dataset_name: str
    test_dataset_name: str
    category_name: str
    workspace_id: str
    model: ModelType
    active_learning_strategies: list
    repeat_id: int
    train_params: dict


def compute_batch_scores(config, elements):
    data_access = get_data_access()
    unlabeled = data_access.sample_unlabeled_text_elements(config.workspace_id, config.train_dataset_name,
                                                           config.category_name, 10 ** 6)["results"]
    unlabeled_emb = np.array(orchestrator_api.infer(config.workspace_id, config.category_name, unlabeled)["embeddings"])
    batch_emb = np.array(orchestrator_api.infer(config.workspace_id, config.category_name, elements)["embeddings"])

    outlier_calculator = KnnOutlierCalculator(unlabeled_emb)
    outlier_value = outlier_calculator.compute_batch_score(batch_emb)
    representativeness_value = 1 / outlier_value
    diversity_calculator = DiversityCalculator(unlabeled_emb)
    diversity_value = diversity_calculator.compute_batch_score(batch_emb)
    return diversity_value, representativeness_value


class ExperimentRunner(object, metaclass=abc.ABCMeta):
    NO_AL = 'no_active_learning'

    def __init__(self, first_model_positives_num: int, first_model_negatives_num: int,
                 active_learning_suggestions_num: int):
        """
        Init the ExperimentsRunner
        :param first_model_positives_num: the number of positives instances to provide for the first model.
        :param first_model_negatives_num: the number of negative instances to provide for the first model.
        :param active_learning_suggestions_num: the number of instances to be suggested by the active learning strategy
        for the training of the second model.

        """
        self.first_model_positives_num = first_model_positives_num
        self.first_model_negatives_num = first_model_negatives_num
        self.active_learning_suggestions_num = active_learning_suggestions_num
        self.data_access: DataAccessApi = data_access_factory.get_data_access()
        self.cached_first_model_scores = False
        orchestrator_api.set_training_set_selection_strategy(TrainingSetSelectionStrategy.ALL_LABELED)

    def run(self, config: ExperimentParams, active_learning_iterations_num: int, results_file_path: str,
            delete_workspaces: bool = True):

        # key: active learning name, value: list of results oevr iterations (first model has no iterations)
        results_per_active_learning = defaultdict(dict)
        # train first model
        iteration = 0
        res_dict = self.train_first_model(config=config)
        res_handler.save_results(results_file_path, [res_dict])
        results_per_active_learning[self.NO_AL][iteration] = res_dict

        original_workspace_id = config.workspace_id

        for al in config.active_learning_strategies:
            orchestrator_api.set_active_learning_strategy(al)
            if not orchestrator_api.is_model_compatible_with_active_learning(al, config.model):
                logging.info(f'skipping active learning strategy {al.name} for model {config.model.name} '
                             f'since the strategy does not support this model.')
                continue

            al_workspace_id = original_workspace_id + "-" + al.name
            if orchestrator_api.workspace_exists(al_workspace_id):
                orchestrator_api.delete_workspace(al_workspace_id)
            orchestrator_api.copy_workspace(original_workspace_id, al_workspace_id)
            config.workspace_id = al_workspace_id

            for iteration in range(1, active_learning_iterations_num + 1):
                logging.info(f'Run AL strategy: {al.name}, iteration num: {iteration}, repeat num: {config.repeat_id}\t'
                             f'workspace: {config.workspace_id}')

                res_dict, train_id = self.run_active_learning_iteration(config, al, iteration)
                res_handler.save_results(results_file_path, [res_dict])
                results_per_active_learning[al.name][iteration] = res_dict

            if delete_workspaces:
                orchestrator_api.delete_workspace(config.workspace_id, DeleteModels.ALL_BUT_FIRST_MODEL)
        if delete_workspaces:
            orchestrator_api.delete_workspace(original_workspace_id)
        return results_per_active_learning

    def train_first_model(self, config: ExperimentParams):
        if orchestrator_api.workspace_exists(config.workspace_id):
            orchestrator_api.delete_workspace(config.workspace_id)

        orchestrator_api.create_workspace(config.workspace_id, config.train_dataset_name,
                                          dev_dataset_name=config.dev_dataset_name)
        orchestrator_api.create_new_category(config.workspace_id, config.category_name, "No description for you")

        dev_text_elements_uris = orchestrator_api.get_all_text_elements_uris(config.dev_dataset_name)
        dev_text_elements_and_labels = oracle_data_access_api.get_gold_labels(config.dev_dataset_name,
                                                                              dev_text_elements_uris)
        if dev_text_elements_and_labels is not None:
            orchestrator_api.set_labels(config.workspace_id, dev_text_elements_and_labels)

        random_seed = sum([ord(c) for c in config.workspace_id])
        logging.info(str(config))
        logging.info(f'random seed: {random_seed}')

        self.set_first_model_positives(config, random_seed)
        self.set_first_model_negatives(config, random_seed)

        # train first model
        logging.info(f'Starting first model training (model: {config.model.name})\tworkspace: {config.workspace_id}')
        new_model_id = orchestrator_api.train(config.workspace_id, config.category_name, config.model, train_params=config.train_params)
        if new_model_id is None:
            raise Exception(f'a new model was not trained\tworkspace: {config.workspace_id}')

        eval_dataset = config.test_dataset_name
        res_dict = self.evaluate(config, al=self.NO_AL, iteration=0, eval_dataset=eval_dataset)
        res_dict.update(self.generate_al_batch_dict(config))  # ensures AL-related keys are in the results dictionary

        logging.info(f'Evaluation on dataset: {eval_dataset}, iteration: 0, first model (id: {new_model_id}) '
                     f'repeat: {config.repeat_id}, is: {res_dict}\t'
                     f'workspace: {config.workspace_id}')

        return res_dict

    def run_active_learning_iteration(self, config: ExperimentParams, al, iteration):
        # get suggested elements for labeling (and their gold labels)
        suggested_text_elements, suggested_uris_and_gold_labels = \
            self.get_suggested_elements_and_gold_labels(config, al)

        # calculate metrics for the batch suggested by the active learning strategy
        al_batch_dict = self.generate_al_batch_dict(config, suggested_text_elements)

        # set gold labels as the user-provided labels of the elements suggested by the active learning strategy
        orchestrator_api.set_labels(config.workspace_id, suggested_uris_and_gold_labels)

        # train a new model with the additional elements suggested by the active learning strategy
        new_model_id = orchestrator_api.train(config.workspace_id, config.category_name, config.model, train_params=config.train_params)
        if new_model_id is None:
            raise Exception('New model was not trained')

        # evaluate the new model
        eval_dataset = config.test_dataset_name
        res_dict = self.evaluate(config, al.name, iteration, eval_dataset, suggested_text_elements)
        res_dict.update(al_batch_dict)

        logging.info(f'Evaluation on dataset: {eval_dataset}, with AL: {al.name}, iteration: {iteration}, '
                     f'repeat: {config.repeat_id}, model (id: {new_model_id}) is: {res_dict}\t'
                     f'workspace: {config.workspace_id}')
        return res_dict, new_model_id

    def get_suggested_elements_and_gold_labels(self, config, al):
        start = time.time()
        suggested_text_elements_for_labeling = \
            orchestrator_api.get_elements_to_label(config.workspace_id, config.category_name,
                                                   self.active_learning_suggestions_num)
        end = time.time()
        logging.info(f'{len(suggested_text_elements_for_labeling)} instances '
                     f'suggested by active learning strategy: {al.name} '
                     f'for dataset: {config.train_dataset_name} and category: {config.category_name}.\t'
                     f'runtime: {end - start}\tworkspace: {config.workspace_id}')
        uris_for_labeling = [elem.uri for elem in suggested_text_elements_for_labeling]
        uris_and_gold_labels = oracle_data_access_api.get_gold_labels(config.train_dataset_name, uris_for_labeling,
                                                                      config.category_name)
        return suggested_text_elements_for_labeling, uris_and_gold_labels

    def evaluate(self, config: ExperimentParams, al, iteration, eval_dataset,
                 suggested_text_elements_for_labeling=None):
        metadata_dict = res_handler.generate_metadata_dict(config, eval_dataset, al, iteration)
        labels_counts_dict = res_handler.generate_train_labels_counts_dict(config)
        performance_dict = res_handler.generate_performance_metrics_dict(config, eval_dataset)
        experiment_specific_metrics_dict = \
            self.generate_additional_metrics_dict(config, suggested_text_elements_for_labeling)
        res_dict = {**metadata_dict, **labels_counts_dict, **performance_dict, **experiment_specific_metrics_dict}
        return res_dict

    @abc.abstractmethod
    def set_first_model_positives(self, config, random_seed) -> List[TextElement]:
        """
        Set the positive instances for the training of the first model.
        :param config: experiment config for this run
        :param random_seed: a seed for the Random being used for sampling
        :return: a list of TextElements and a log message

        """
        func_name = self.set_first_model_positives.__name__
        raise NotImplementedError('users must define ' + func_name + ' to use this base class')

    @abc.abstractmethod
    def set_first_model_negatives(self, config, random_seed) -> List[TextElement]:
        """
        Set the negative instances for the training of the first model.
        :param config: experiment config for this run
        :param random_seed: a seed for the Random being used for sampling
        :return: a list of TextElements and a log message

        """
        func_name = self.set_first_model_negatives.__name__
        raise NotImplementedError('users must define ' + func_name + ' to use this base class')

    @staticmethod
    def generate_al_batch_dict(config, batch_elements=None):
        batch_dict = {}
        model_supports_embeddings = \
            orchestrator_api.is_model_compatible_with_active_learning(ActiveLearningStrategies.DAL, config.model)
        if batch_elements is not None and model_supports_embeddings:
            diversity_value, representativeness_value = compute_batch_scores(config, batch_elements)
            batch_dict["diversity"] = diversity_value
            batch_dict["representativeness"] = representativeness_value
        else:
            batch_dict["diversity"] = "NA"
            batch_dict["representativeness"] = "NA"
        return batch_dict

    def generate_additional_metrics_dict(self, config, suggested_text_elements_for_labeling):
        return {}

[((15, 0, 15, 115), 'logging.basicConfig', 'logging.basicConfig', (), '', False, 'import logging\n'), ((47, 18, 47, 35), 'lrtc_lib.data_access.data_access_factory.get_data_access', 'get_data_access', ({}, {}), '()', False, 'from lrtc_lib.data_access.data_access_factory import get_data_access\n'), ((53, 25, 53, 60), 'lrtc_lib.active_learning.knn_outlier_calculator.KnnOutlierCalculator', 'KnnOutlierCalculator', ({(53, 46, 53, 59): 'unlabeled_emb'}, {}), '(unlabeled_emb)', False, 'from lrtc_lib.active_learning.knn_outlier_calculator import KnnOutlierCalculator\n'), ((56, 27, 56, 61), 'lrtc_lib.active_learning.diversity_calculator.DiversityCalculator', 'DiversityCalculator', ({(56, 47, 56, 60): 'unlabeled_emb'}, {}), '(unlabeled_emb)', False, 'from lrtc_lib.active_learning.diversity_calculator import DiversityCalculator\n'), ((77, 42, 77, 79), 'lrtc_lib.data_access.data_access_factory.get_data_access', 'data_access_factory.get_data_access', ({}, {}), '()', True, 'import lrtc_lib.data_access.data_access_factory as data_access_factory\n'), ((79, 8, 79, 102), 'lrtc_lib.orchestrator.orchestrator_api.set_training_set_selection_strategy', 'orchestrator_api.set_training_set_selection_strategy', ({(79, 61, 79, 101): 'TrainingSetSelectionStrategy.ALL_LABELED'}, {}), '(\n    TrainingSetSelectionStrategy.ALL_LABELED)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((85, 38, 85, 55), 'collections.defaultdict', 'defaultdict', ({(85, 50, 85, 54): 'dict'}, {}), '(dict)', False, 'from collections import defaultdict\n'), ((89, 8, 89, 63), 'lrtc_lib.experiment_runners.experiments_results_handler.save_results', 'res_handler.save_results', ({(89, 33, 89, 50): 'results_file_path', (89, 52, 89, 62): '[res_dict]'}, {}), '(results_file_path, [res_dict])', True, 'import lrtc_lib.experiment_runners.experiments_results_handler as res_handler\n'), ((122, 11, 122, 65), 'lrtc_lib.orchestrator.orchestrator_api.workspace_exists', 'orchestrator_api.workspace_exists', ({(122, 45, 122, 64): 'config.workspace_id'}, {}), '(config.workspace_id)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((125, 8, 126, 83), 'lrtc_lib.orchestrator.orchestrator_api.create_workspace', 'orchestrator_api.create_workspace', (), '', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((127, 8, 127, 113), 'lrtc_lib.orchestrator.orchestrator_api.create_new_category', 'orchestrator_api.create_new_category', ({(127, 45, 127, 64): 'config.workspace_id', (127, 66, 127, 86): 'config.category_name', (127, 88, 127, 112): '"""No description for you"""'}, {}), "(config.workspace_id, config.\n    category_name, 'No description for you')", False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((129, 33, 129, 101), 'lrtc_lib.orchestrator.orchestrator_api.get_all_text_elements_uris', 'orchestrator_api.get_all_text_elements_uris', ({(129, 77, 129, 100): 'config.dev_dataset_name'}, {}), '(config.dev_dataset_name)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((130, 39, 131, 101), 'lrtc_lib.oracle_data_access.oracle_data_access_api.get_gold_labels', 'oracle_data_access_api.get_gold_labels', ({(130, 78, 130, 101): 'config.dev_dataset_name', (131, 78, 131, 100): 'dev_text_elements_uris'}, {}), '(config.dev_dataset_name,\n    dev_text_elements_uris)', False, 'from lrtc_lib.oracle_data_access import oracle_data_access_api\n'), ((137, 8, 137, 51), 'logging.info', 'logging.info', ({(137, 21, 137, 50): 'f"""random seed: {random_seed}"""'}, {}), "(f'random seed: {random_seed}')", False, 'import logging\n'), ((143, 8, 143, 117), 'logging.info', 'logging.info', ({(143, 21, 143, 116): 'f"""Starting first model training (model: {config.model.name})\tworkspace: {config.workspace_id}"""'}, {}), "(\n    f'Starting first model training (model: {config.model.name})\\tworkspace: {config.workspace_id}'\n    )", False, 'import logging\n'), ((144, 23, 144, 136), 'lrtc_lib.orchestrator.orchestrator_api.train', 'orchestrator_api.train', (), '', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((152, 8, 154, 57), 'logging.info', 'logging.info', ({(152, 21, 154, 56): 'f"""Evaluation on dataset: {eval_dataset}, iteration: 0, first model (id: {new_model_id}) repeat: {config.repeat_id}, is: {res_dict}\tworkspace: {config.workspace_id}"""'}, {}), "(\n    f'Evaluation on dataset: {eval_dataset}, iteration: 0, first model (id: {new_model_id}) repeat: {config.repeat_id}, is: {res_dict}\\tworkspace: {config.workspace_id}'\n    )", False, 'import logging\n'), ((167, 8, 167, 88), 'lrtc_lib.orchestrator.orchestrator_api.set_labels', 'orchestrator_api.set_labels', ({(167, 36, 167, 55): 'config.workspace_id', (167, 57, 167, 87): 'suggested_uris_and_gold_labels'}, {}), '(config.workspace_id, suggested_uris_and_gold_labels\n    )', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((170, 23, 170, 136), 'lrtc_lib.orchestrator.orchestrator_api.train', 'orchestrator_api.train', (), '', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((179, 8, 181, 57), 'logging.info', 'logging.info', ({(179, 21, 181, 56): 'f"""Evaluation on dataset: {eval_dataset}, with AL: {al.name}, iteration: {iteration}, repeat: {config.repeat_id}, model (id: {new_model_id}) is: {res_dict}\tworkspace: {config.workspace_id}"""'}, {}), "(\n    f'Evaluation on dataset: {eval_dataset}, with AL: {al.name}, iteration: {iteration}, repeat: {config.repeat_id}, model (id: {new_model_id}) is: {res_dict}\\tworkspace: {config.workspace_id}'\n    )", False, 'import logging\n'), ((185, 16, 185, 27), 'time.time', 'time.time', ({}, {}), '()', False, 'import time\n'), ((187, 12, 188, 88), 'lrtc_lib.orchestrator.orchestrator_api.get_elements_to_label', 'orchestrator_api.get_elements_to_label', ({(187, 51, 187, 70): 'config.workspace_id', (187, 72, 187, 92): 'config.category_name', (188, 51, 188, 87): 'self.active_learning_suggestions_num'}, {}), '(config.workspace_id, config.\n    category_name, self.active_learning_suggestions_num)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((189, 14, 189, 25), 'time.time', 'time.time', ({}, {}), '()', False, 'import time\n'), ((195, 31, 196, 91), 'lrtc_lib.oracle_data_access.oracle_data_access_api.get_gold_labels', 'oracle_data_access_api.get_gold_labels', ({(195, 70, 195, 95): 'config.train_dataset_name', (195, 97, 195, 114): 'uris_for_labeling', (196, 70, 196, 90): 'config.category_name'}, {}), '(config.train_dataset_name,\n    uris_for_labeling, config.category_name)', False, 'from lrtc_lib.oracle_data_access import oracle_data_access_api\n'), ((201, 24, 201, 95), 'lrtc_lib.experiment_runners.experiments_results_handler.generate_metadata_dict', 'res_handler.generate_metadata_dict', ({(201, 59, 201, 65): 'config', (201, 67, 201, 79): 'eval_dataset', (201, 81, 201, 83): 'al', (201, 85, 201, 94): 'iteration'}, {}), '(config, eval_dataset, al, iteration)', True, 'import lrtc_lib.experiment_runners.experiments_results_handler as res_handler\n'), ((202, 29, 202, 82), 'lrtc_lib.experiment_runners.experiments_results_handler.generate_train_labels_counts_dict', 'res_handler.generate_train_labels_counts_dict', ({(202, 75, 202, 81): 'config'}, {}), '(config)', True, 'import lrtc_lib.experiment_runners.experiments_results_handler as res_handler\n'), ((203, 27, 203, 94), 'lrtc_lib.experiment_runners.experiments_results_handler.generate_performance_metrics_dict', 'res_handler.generate_performance_metrics_dict', ({(203, 73, 203, 79): 'config', (203, 81, 203, 93): 'eval_dataset'}, {}), '(config, eval_dataset)', True, 'import lrtc_lib.experiment_runners.experiments_results_handler as res_handler\n'), ((237, 12, 237, 113), 'lrtc_lib.orchestrator.orchestrator_api.is_model_compatible_with_active_learning', 'orchestrator_api.is_model_compatible_with_active_learning', ({(237, 70, 237, 98): 'ActiveLearningStrategies.DAL', (237, 100, 237, 112): 'config.model'}, {}), '(\n    ActiveLearningStrategies.DAL, config.model)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((50, 29, 50, 105), 'lrtc_lib.orchestrator.orchestrator_api.infer', 'orchestrator_api.infer', ({(50, 52, 50, 71): 'config.workspace_id', (50, 73, 50, 93): 'config.category_name', (50, 95, 50, 104): 'unlabeled'}, {}), '(config.workspace_id, config.category_name, unlabeled)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((51, 25, 51, 100), 'lrtc_lib.orchestrator.orchestrator_api.infer', 'orchestrator_api.infer', ({(51, 48, 51, 67): 'config.workspace_id', (51, 69, 51, 89): 'config.category_name', (51, 91, 51, 99): 'elements'}, {}), '(config.workspace_id, config.category_name, elements)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((95, 12, 95, 61), 'lrtc_lib.orchestrator.orchestrator_api.set_active_learning_strategy', 'orchestrator_api.set_active_learning_strategy', ({(95, 58, 95, 60): 'al'}, {}), '(al)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((102, 15, 102, 65), 'lrtc_lib.orchestrator.orchestrator_api.workspace_exists', 'orchestrator_api.workspace_exists', ({(102, 49, 102, 64): 'al_workspace_id'}, {}), '(al_workspace_id)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((104, 12, 104, 83), 'lrtc_lib.orchestrator.orchestrator_api.copy_workspace', 'orchestrator_api.copy_workspace', ({(104, 44, 104, 65): 'original_workspace_id', (104, 67, 104, 82): 'al_workspace_id'}, {}), '(original_workspace_id, al_workspace_id)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((118, 12, 118, 68), 'lrtc_lib.orchestrator.orchestrator_api.delete_workspace', 'orchestrator_api.delete_workspace', ({(118, 46, 118, 67): 'original_workspace_id'}, {}), '(original_workspace_id)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((123, 12, 123, 66), 'lrtc_lib.orchestrator.orchestrator_api.delete_workspace', 'orchestrator_api.delete_workspace', ({(123, 46, 123, 65): 'config.workspace_id'}, {}), '(config.workspace_id)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((133, 12, 133, 90), 'lrtc_lib.orchestrator.orchestrator_api.set_labels', 'orchestrator_api.set_labels', ({(133, 40, 133, 59): 'config.workspace_id', (133, 61, 133, 89): 'dev_text_elements_and_labels'}, {}), '(config.workspace_id, dev_text_elements_and_labels)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((96, 19, 96, 94), 'lrtc_lib.orchestrator.orchestrator_api.is_model_compatible_with_active_learning', 'orchestrator_api.is_model_compatible_with_active_learning', ({(96, 77, 96, 79): 'al', (96, 81, 96, 93): 'config.model'}, {}), '(al, config.model)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((97, 16, 98, 80), 'logging.info', 'logging.info', ({(97, 29, 98, 79): 'f"""skipping active learning strategy {al.name} for model {config.model.name} since the strategy does not support this model."""'}, {}), "(\n    f'skipping active learning strategy {al.name} for model {config.model.name} since the strategy does not support this model.'\n    )", False, 'import logging\n'), ((103, 16, 103, 66), 'lrtc_lib.orchestrator.orchestrator_api.delete_workspace', 'orchestrator_api.delete_workspace', ({(103, 50, 103, 65): 'al_workspace_id'}, {}), '(al_workspace_id)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n'), ((108, 16, 109, 65), 'logging.info', 'logging.info', ({(108, 29, 109, 64): 'f"""Run AL strategy: {al.name}, iteration num: {iteration}, repeat num: {config.repeat_id}\tworkspace: {config.workspace_id}"""'}, {}), "(\n    f'Run AL strategy: {al.name}, iteration num: {iteration}, repeat num: {config.repeat_id}\\tworkspace: {config.workspace_id}'\n    )", False, 'import logging\n'), ((112, 16, 112, 71), 'lrtc_lib.experiment_runners.experiments_results_handler.save_results', 'res_handler.save_results', ({(112, 41, 112, 58): 'results_file_path', (112, 60, 112, 70): '[res_dict]'}, {}), '(results_file_path, [res_dict])', True, 'import lrtc_lib.experiment_runners.experiments_results_handler as res_handler\n'), ((116, 16, 116, 104), 'lrtc_lib.orchestrator.orchestrator_api.delete_workspace', 'orchestrator_api.delete_workspace', ({(116, 50, 116, 69): 'config.workspace_id', (116, 71, 116, 103): 'DeleteModels.ALL_BUT_FIRST_MODEL'}, {}), '(config.workspace_id, DeleteModels.\n    ALL_BUT_FIRST_MODEL)', False, 'from lrtc_lib.orchestrator import orchestrator_api\n')]