"""
Copyright 2018 BlazeMeter Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import ast
import math
import re
import string
from collections import OrderedDict
from urllib import parse

import astunparse

from bzt import TaurusConfigError, TaurusInternalException
from bzt.engine import Scenario
from bzt.requests_model import HTTPRequest, HierarchicRequestParser, TransactionBlock, SetVariables
from bzt.requests_model import IncludeScenarioBlock, SetUpBlock, TearDownBlock
from bzt.utils import iteritems, dehumanize_time, ensure_is_dict, BetterDict
from .ast_helpers import ast_attr, ast_call, gen_empty_line_stmt, gen_store, gen_subscript, gen_try_except, gen_raise
from .jmeter_functions import JMeterExprCompiler


def normalize_class_name(text):
    allowed_chars = "%s%s%s" % (string.digits, string.ascii_letters, '_')
    split_separator = re.split(r'[\-_]', text)
    return ''.join([capitalize_class_name(part, allowed_chars) for part in split_separator])


def capitalize_class_name(text, allowed_chars):
    return filter_string(text, allowed_chars).capitalize()


def filter_string(text, allowed_chars):
    return ''.join(c for c in text if c in allowed_chars)


def normalize_method_name(text):
    allowed_chars = "%s%s%s" % (string.digits, string.ascii_letters, '- ')
    return filter_string(text, allowed_chars).replace(' ', '_').replace('-', '_')


def create_class_name(label):
    return 'TestAPI' if label.startswith('autogenerated') else 'Test%s' % normalize_class_name(label)


def create_method_name(label):
    return 'test_requests' if label.startswith('autogenerated') else normalize_method_name(label)


class ApiritifScriptGenerator(object):
    BYS = {
        'xpath': "XPATH",
        'css': "CSS_SELECTOR",
        'name': "NAME",
        'id': "ID",
        'linktext': "LINK_TEXT"
    }

    TO_BYS = {
        'byxpath': "xpath",
        'bycss': "css",
        'byname': "name",
        'byid': "id",
        'bylinktext': "linktext",
        'byelement': "byelement",
        'byshadow': "shadow"
    }

    ACTION_CHAINS = {
        'doubleclick': "double_click",
        'contextclick': "context_click",
        'mousedown': "click_and_hold",
        'mouseup': "release",
        'mousemove': "move_to_element",
        'mouseover': "move_to_element",
        'mouseout': "move_to_element_with_offset"
    }

    ACTIONS = "|".join(['click', 'doubleClick', 'contextClick', 'mouseDown', 'mouseUp', 'mouseMove', 'mouseOut',
                        'mouseOver', 'select', 'wait', 'keys', 'pauseFor', 'clear', 'assert',
                        'assertText', 'assertValue', 'assertDialog', 'answerDialog', 'submit',
                        'close', 'script', 'editcontent',
                        'switch', 'switchFrame', 'go', 'echo', 'type', 'element', 'drag',
                        'storeText', 'storeValue', 'store', 'open', 'screenshot', 'rawCode',
                        'resize', 'maximize', 'alert', 'waitFor'
                        ])

    ACTIONS_WITH_WAITER = ['go', 'click', 'doubleclick', 'contextclick', 'drag', 'select', 'type', 'script']

    EXECUTION_BLOCKS = "|".join(['if', 'loop', 'foreach'])

    # Python AST docs: https://greentreesnakes.readthedocs.io/en/latest/

    IMPORTS = """import os
import re
from %s import webdriver
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support import expected_conditions as econd
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.keys import Keys
"""

    BY_TAGS = ("byName", "byID", "byCSS", "byXPath", "byLinkText", "byElement", "byShadow")
    COMMON_TAGS = ("Cookies", "Title", "Window", "Eval", "ByIdx", "String")
    EXTERNAL_HANDLER_START = 'action_start'
    EXTERNAL_HANDLER_END = 'action_end'
    EXTERNAL_HANDLER_TAGS = (EXTERNAL_HANDLER_START, EXTERNAL_HANDLER_END)
    DEPRECATED_LOG_TAG = 'log'

    ACCESS_TARGET = 'target'
    ACCESS_PLAIN = 'plain'
    SUPPORTED_BLOCKS = (HTTPRequest, TransactionBlock, SetVariables, IncludeScenarioBlock, SetUpBlock, TearDownBlock)

    FINALLY_MARKER = 'finally'
    OPTIONS = 'options'

    def __init__(self, scenario, label, wdlog=None, executor=None, ignore_unknown_actions=False,
                 generate_markers=None, capabilities=None, wd_addr=None, test_mode="selenium",
                 generate_external_handler=False, selenium_version=None):
        self.scenario = scenario
        self.selenium_extras = set()
        self.data_sources = list(scenario.get_data_sources())
        self.executor = executor
        self.label = label
        self.log = self.scenario.engine.log.getChild(self.__class__.__name__)
        self.tree = None
        self.verbose = False
        self.expr_compiler = JMeterExprCompiler(parent_log=self.log)
        self.service_methods = []
        self.selenium_version = selenium_version

        self.remote_address = wd_addr
        self.capabilities = capabilities or {}
        self.window_size = None
        self.wdlog = wdlog
        self.browser = None
        self.appium = False
        self.ignore_unknown_actions = ignore_unknown_actions
        self.generate_markers = generate_markers
        self.generate_external_handler = generate_external_handler
        self.test_mode = test_mode
        self.replace_dialogs = True

    def _parse_action_params(self, expr, name):
        res = expr.match(name)
        if not res:
            msg = "Unsupported action: %s" % name
            if self.ignore_unknown_actions:
                self.log.warning(msg)
                return
            else:
                raise TaurusConfigError(msg)

        atype = res.group(1).lower()
        tag = res.group(2).lower() if res.group(2) else ""
        selector = None
        if len(res.groups()) > 3:
            selector = res.group(4)

        return atype, tag, selector

    @staticmethod
    def _trim_quotes(selector):
        if selector.startswith('"') and selector.endswith('"'):
            selector = selector[1:-1]
        elif selector.startswith("'") and selector.endswith("'"):
            selector = selector[1:-1]
        return selector

    def _parse_string_action(self, name, param):
        tags = "|".join(self.BY_TAGS + self.COMMON_TAGS)
        all_actions = self.ACTIONS + "|" + "|".join((self.DEPRECATED_LOG_TAG, self.EXECUTION_BLOCKS))
        expr = re.compile(r"^(%s)(%s)?(\(([\S\s]*)\))?$" % (all_actions, tags), re.IGNORECASE)
        atype, tag, selector = self._parse_action_params(expr, name)
        value = None
        selectors = []
        if selector:
            selector = self._trim_quotes(selector)
        else:
            selector = ""

        # Need to shuffle the variables to get the same output for both of the versions of
        # action types, this is unfortunately cumbersome as the param/value can be on different
        # places:
        # action_name(selector): param
        # action_name(param)
        # action_name(value): param, e.g. storeString(value): var_name
        if selector:
            if tag in self.TO_BYS.keys():
                tag_name = self.TO_BYS[tag]
                selectors = [{tag_name: selector}]
            elif param is None:
                param = selector
            else:
                value = selector

        if atype == "drag":
            # param should be e.g. elementByXPath(/xpath)
            element_action = self._parse_action(param)
            selectors = (selectors, element_action[4])
        elif atype == "switchframe":
            # for switchFrameByName we need to get the param
            param = selector
        elif atype == "waitfor":
            value = param
            args = selector.rsplit(",", 1)
            if len(args) != 2:
                raise TaurusConfigError("Incorrect amount of arguments (%s) for waitFor (2 expected)." % len(args))
            param = args[1].strip()
            selectors = [{self.TO_BYS[tag]: self._trim_quotes(args[0].strip())}]
        elif atype in ['answerdialog', 'assertdialog']:
            param, value = value, param

        return atype, tag, param, value, selectors

    def _parse_dict_action(self, action_config):
        name = action_config["type"]
        selectors = []
        if action_config.get("locators"):
            selectors = action_config.get("locators")
        if action_config.get("element"):
            selectors.extend(self._gen_selector_byelement(action_config))
        if action_config.get("shadow"):
            selectors = [{"shadow": action_config.get("shadow")}]
        if action_config.get("source") and action_config.get("target"):
            source = action_config.get("source")
            target = action_config.get("target")
            if self._is_foreach_element(source):
                source = self._gen_selector_byelement(source[0])
            if self._is_foreach_element(target):
                target = self._gen_selector_byelement(target[0])
            selectors = (source, target)
        param = action_config["param"]
        value = action_config["value"]
        tags = "|".join(self.COMMON_TAGS) + "|ByName"  # ByName is needed in switchFrameByName
        all_actions = self.ACTIONS + "|" + "|".join(self.EXTERNAL_HANDLER_TAGS)
        expr = re.compile("^(%s)(%s)?$" % (all_actions, tags), re.IGNORECASE)
        action_params = self._parse_action_params(expr, name)

        return action_params[0], action_params[1], param, value, selectors

    @staticmethod
    def _gen_selector_byelement(config):
        return [{"byelement": config.get("element")}]

    def _parse_action(self, action_config):
        if isinstance(action_config, str):
            name = action_config
            param = None
        elif isinstance(action_config, dict):
            if action_config.get("type"):
                return self._parse_dict_action(action_config)
            block = self._get_execution_block(action_config)
            if len(block) == 1:
                name, param = (block[0], action_config.get(block[0]))
            else:
                name, param = next(iteritems(action_config))
        else:
            raise TaurusConfigError("Unsupported value for action: %s" % action_config)

        return self._parse_string_action(name, param)

    def _get_execution_block(self, action_config):
        # get the list of execution blocks in this action if there are any or empty list
        return list(set(action_config.keys()).intersection(self.EXECUTION_BLOCKS.split("|")))

    @staticmethod
    def _is_foreach_element(locators):
        # action performed in foreach loop on element
        return len(locators) == 1 and (locators[0].get("byelement") or locators[0].get("element"))

    @staticmethod
    def _is_shadow_locator(locators):
        return len(locators) == 1 and locators[0].get("shadow")

    def _gen_dynamic_locator(self, var_w_locator, locators):
        if self._is_foreach_element(locators):
            return ast.Name(id=locators[0].get("byelement"))
        el = self._get_byelement(locators)
        target = el if el else "self.driver"
        method = "%s.find_element" % target

        if self._is_shadow_locator(locators):
            self.selenium_extras.add("find_element_by_shadow")
            return ast_call(
                func=ast_attr("find_element_by_shadow"),
                args=[
                    self._gen_expr(locators[0].get("shadow"))
                ]
            )
        return ast_call(
            func=ast_attr(method),
            args=[
                gen_subscript(var_w_locator, 0),
                gen_subscript(var_w_locator, 1)
            ])

    def _gen_ast_locators_dict(self, locators):
        args = []
        for loc in locators:
            locator_type = list(loc.keys())[0]
            locator_value = loc[locator_type]
            args.append(ast.Dict([ast.Str(locator_type, kind="")], [self._gen_expr(locator_value)]))
        return args

    def _gen_loc_method_call(self, method, var_name, locators, parent_el=None):
        args = [ast.List(elts=self._gen_ast_locators_dict(locators))]
        if parent_el:
            args.append(ast.Name(id=parent_el))
        return ast.Assign(
            targets=[ast.Name(id=var_name, ctx=ast.Store(), kind="")],
            value=ast_call(func=method,
                           args=args))

    def _gen_get_locator_call(self, var_name, locators):
        # don't generate 'get_locator' for byElement action or shadow locator
        if self._is_foreach_element(locators) or self._is_shadow_locator(locators):
            return []
        parent_el = self._get_byelement(locators)
        locs = [l for l in locators if not l.get("byelement")]  # remove the byelement locator from the list
        return self._gen_loc_method_call("get_locator", var_name, locs, parent_el)

    def _get_byelement(self, locators):
        for loc in locators:
            el = loc.get("byelement")
            if el:
                return el
        return None

    def _gen_get_elements_call(self, var_name, locators):
        return self._gen_loc_method_call("get_elements", var_name, locators)

    def _gen_locator(self, tag, selector):
        return ast_call(
            func=ast_attr("self.driver.find_element"),
            args=[
                ast_attr("By.%s" % self.BYS[tag]),
                self._gen_expr(selector)])

    def _gen_window_mngr(self, atype, param):
        elements = []
        if atype == "switch":
            method = "switch_window"
            self.selenium_extras.add(method)
            elements.append(ast_call(
                func=ast_attr(method),
                args=[self._gen_expr(param)]))
        elif atype == "resize":
            if not re.compile(r"\d+,\d+").match(param):
                if re.compile(r"\d+, \d+").match(param):
                    param = param.replace(', ', ',')
                else:
                    return elements
            x, y = param.split(",")
            elements.append(ast_call(
                func=ast_attr("self.driver.set_window_size"),
                args=[self._gen_expr(x), self._gen_expr(y)]))
        elif atype == "maximize":
            args = []
            elements.append(ast_call(
                func=ast_attr("self.driver.maximize_window"),
                args=args))
        elif atype == "open":
            method = "open_window"
            self.selenium_extras.add(method)
            elements.append(ast_call(
                func=ast_attr(method),
                args=[self._gen_expr(param.strip())]))
        elif atype == "close":
            method = "close_window"
            self.selenium_extras.add(method)
            args = []
            if param:
                args.append(self._gen_expr(param))
            elements.append(ast_call(
                func=ast_attr(method),
                args=args))
        return elements

    def _gen_frame_mngr(self, tag, selector):
        method = "switch_frame"
        self.selenium_extras.add(method)
        elements = []
        if not selector:
            raise TaurusConfigError("Can not generate action for 'switchFrame'. Selector is empty.")
        if tag == "byidx" or selector.startswith("index=") or selector in ["relative=top", "relative=parent"]:
            if tag == "byidx":
                selector = "index=%s" % selector

            elements.append(ast_call(
                func=ast_attr(method),
                args=[ast.Str(selector, kind="")]))
        else:
            if not tag:
                if "=" in selector:
                    parts = selector.partition("=")
                    tag = parts[0].strip()
                    selector = self._trim_quotes(parts[2].strip())
                else:
                    tag = "name"  # if tag is not present default it to name
            elif tag.startswith('by'):
                tag = tag[2:]  # remove the 'by' prefix
            elements.append(ast_call(
                func=ast_attr(method),
                args=[self._gen_locator(tag, selector)]))
        return elements

    def _gen_chain_mngr(self, atype, selectors):
        elements = []
        if atype in self.ACTION_CHAINS:
            elements.append(self._gen_get_locator_call("var_loc_chain", selectors))
            locator = self._gen_dynamic_locator("var_loc_chain", selectors)
            operator = ast_attr(fields=(
                ast_call(func="ActionChains", args=[ast_attr("self.driver")]),
                self.ACTION_CHAINS[atype.lower()]))
            args = [locator, ast.Num(-10, kind=""), ast.Num(-10, kind="")] if atype == "mouseout" else [locator]
            elements.append(ast_call(
                func=ast_attr(
                    fields=(
                        ast_call(
                            func=operator,
                            args=args),
                        "perform"))))
        elif atype == "drag":
            if not selectors or not selectors[0]:
                raise TaurusConfigError("Can not generate action for 'drag'. Source is empty.")
            if not selectors[1]:
                raise TaurusConfigError("Can not generate action for 'drag'. Target is empty.")
            source = selectors[0]
            target = selectors[1]

            elements = [self._gen_get_locator_call("source", source),
                        self._gen_get_locator_call("target", target)]

            operator = ast_attr(
                fields=(
                    ast_call(
                        func="ActionChains",
                        args=[ast_attr("self.driver")]),
                    "drag_and_drop"))
            elements.append(ast_call(
                func=ast_attr(
                    fields=(
                        ast_call(
                            func=operator,
                            args=[self._gen_dynamic_locator("source", source),
                                  self._gen_dynamic_locator("target", target)]),
                        "perform"))))
        return elements

    def _gen_assert_store_mngr(self, atype, tag, name, value, selectors):
        elements = []
        if not name:
            raise TaurusConfigError("Missing param for %s action." % atype)
        if tag == 'title':
            if atype.startswith('assert'):
                elements.append(ast_call(
                    func=ast_attr("self.assertEqual"),
                    args=[ast_attr("self.driver.title"), self._gen_expr(name)]))
            else:
                elements.append(gen_store(
                    name=self._gen_expr(name.strip()),
                    value=self._gen_expr(ast_attr("self.driver.title"))))
        elif atype == 'store' and tag == 'string':
            elements.append(gen_store(
                name=self._gen_expr(name.strip()),
                value=self._gen_expr(value.strip())))
        elif atype == 'assert' and tag == 'eval':
            elements.append(ast_call(
                func=ast_attr("self.assertTrue"),
                args=[self._gen_eval_js_expression(name), ast.Str(name, kind="")]))
        elif atype == 'store' and tag == 'eval':
            escaped_value = self._escape_js_blocks(value)
            elements.append(
                gen_store(
                    self._gen_expr(name.strip()),
                    value=self._gen_eval_js_expression(escaped_value))
            )
        else:
            target = None

            if atype in ["asserttext", "storetext"]:
                target = "innerText"
            elif atype in ["assertvalue", "storevalue"]:
                target = "value"

            if target:
                elements.append(self._gen_get_locator_call("var_loc_as", selectors))
                locator_attr = ast_call(
                    func=ast_attr(
                        fields=(
                            self._gen_dynamic_locator("var_loc_as", selectors),
                            "get_attribute")),
                    args=[ast.Str(target, kind="")])

                if atype.startswith("assert"):
                    elements.append(ast_call(
                        func=ast_attr(fields="self.assertEqual"),
                        args=[
                            ast_call(
                                func=ast_attr(
                                    fields=(
                                        self._gen_expr(locator_attr),
                                        "strip"))),
                            ast_call(
                                func=ast_attr(
                                    fields=(
                                        self._gen_expr(name),
                                        "strip")))]))
                elif atype.startswith('store'):
                    elements.append(gen_store(
                        self._gen_expr(name.strip()),
                        value=self._gen_expr(locator_attr)))

        return elements

    def _gen_keys_mngr(self, atype, param, selectors):
        elements = []
        args = []
        action = None
        elements.append(self._gen_get_locator_call("var_loc_keys", selectors))

        if atype == "click":
            action = "click"
        elif atype == "submit":
            action = "submit"
        elif atype in ["keys", "type"]:
            if atype == "type":
                elements.append(ast_call(
                    func=ast_attr(
                        fields=(
                            self._gen_dynamic_locator("var_loc_keys", selectors),
                            "clear"))))
            action = "send_keys"
            if isinstance(param, str) and param.startswith("KEY_"):
                args = [ast_attr("Keys.%s" % param.split("KEY_")[1])]
            else:
                args = [self._gen_expr(str(param))]

        if action:
            elements.append(ast_call(
                func=ast_attr(
                    fields=(
                        self._gen_dynamic_locator("var_loc_keys", selectors),
                        action)),
                args=args))
        return elements

    def _gen_edit_mngr(self, param, locators):
        if not param:
            raise TaurusConfigError("Missing param for editContent action.")
        var_name = "var_edit_content"

        elements = [self._gen_get_locator_call(var_name, locators)]
        locator = self._gen_dynamic_locator(var_name, locators)
        tag = gen_subscript(var_name, 0)
        selector = gen_subscript(var_name, 1)

        if self._is_foreach_element(locators):
            el = locators[0].get("byelement")
            exc_msg = "The element '%s' (tag name: '%s', text: '%s') is not a contenteditable element"
            exc_args = [ast.Str(el, kind=""), ast_attr(el + ".tag_name"), ast_attr(el + ".text")]
        elif self._is_shadow_locator(locators):
            el = locators[0].get("shadow")
            exc_msg = "The element (shadow: '%s') is not a contenteditable element"
            exc_args = [ast.Str(el, kind="")]
        else:
            exc_msg = "The element (%s: %r) is not a contenteditable element"
            exc_args = [tag, selector]

        exc_type = ast_call(
            func="NoSuchElementException",
            args=[
                ast.BinOp(
                    left=ast.Str(exc_msg, kind=""),
                    op=ast.Mod(),
                    right=ast.Tuple(elts=exc_args))
            ]
        )

        raise_kwargs = {
            "exc": exc_type,
            "cause": None}

        body = ast.Expr(ast_call(func=ast_attr("self.driver.execute_script"),
                                 args=[
                                     ast.BinOp(
                                         left=ast.Str("arguments[0].innerHTML = '%s';", kind=""),
                                         op=ast.Mod(),
                                         right=self._gen_expr(param.strip())),
                                     locator]))

        element = ast.If(
            test=ast_call(
                func=ast_attr(
                    fields=(locator, "get_attribute")),
                args=[ast.Str("contenteditable", kind="")]),
            body=[body],
            orelse=[ast.Raise(**raise_kwargs)])

        elements.append(element)
        return elements

    def _gen_screenshot_mngr(self, param):
        elements = []
        if param:
            elements.append(ast_call(
                func=ast_attr("self.driver.save_screenshot"),
                args=[self._gen_expr(param)]))
        else:
            elements.append(ast.Assign(
                targets=[ast.Name(id="filename")],
                value=ast_call(
                    func=ast_attr("os.path.join"),
                    args=[
                        ast_call(
                            func=ast_attr("os.getenv"),
                            args=[ast.Str('TAURUS_ARTIFACTS_DIR', kind="")]),
                        ast.BinOp(
                            left=ast.Str('screenshot-%d.png', kind=""),
                            op=ast.Mod(),
                            right=ast.BinOp(
                                left=ast_call(func="time"),
                                op=ast.Mult(),
                                right=ast.Num(1000, kind="")))])))
            elements.append(ast_call(
                func=ast_attr("self.driver.save_screenshot"),
                args=[ast.Name(id="filename")]))
        return elements

    def _gen_alert(self, param):
        elements = []
        switch, args = "self.driver.switch_to.alert.", []
        if param == "OK":
            elements.append(ast_call(
                func=ast_attr(switch + "accept"),
                args=args))
        elif param == "Dismiss":
            elements.append(ast_call(
                func=ast_attr(switch + "dismiss"),
                args=args))
        return elements

    def _gen_sleep_mngr(self, param):
        elements = [ast_call(
            func="sleep",
            args=[ast.Num(dehumanize_time(param), kind="")])]

        return elements

    def _gen_select_mngr(self, param, selectors):
        elements = [self._gen_get_locator_call("var_loc_select", selectors), ast_call(
            func=ast_attr(
                fields=(
                    ast_call(func="Select", args=[self._gen_dynamic_locator("var_loc_select", selectors)]),
                    "select_by_visible_text")),
            args=[self._gen_expr(param)])]
        return elements

    def _gen_action(self, action_config):
        action = self._parse_action(action_config)
        if action:
            atype, tag, param, value, selectors = action
        else:
            atype = tag = param = value = selectors = None

        action_elements = []

        if atype in self.EXTERNAL_HANDLER_TAGS:
            action_elements.append(ast_call(
                func=ast_attr(atype),
                args=[self._gen_expr(self._gen_expr(param))]
            ))
        elif atype == self.DEPRECATED_LOG_TAG:
            self.log.warning("'log' is deprecated. It will be removed in the next release.")
            return []
        elif tag == "window":
            action_elements.extend(self._gen_window_mngr(atype, param))
        elif atype == "switchframe":
            action_elements.extend(self._gen_frame_mngr(tag, param))
        elif atype in self.ACTION_CHAINS or atype == "drag":
            action_elements.extend(self._gen_chain_mngr(atype, selectors))
        elif atype == "select":
            action_elements.extend(self._gen_select_mngr(param, selectors))
        elif atype == 'assertdialog':
            action_elements.extend(self._gen_assert_dialog(param, value))
        elif atype == 'answerdialog':
            action_elements.extend(self._gen_answer_dialog(param, value))
        elif atype is not None and (atype.startswith("assert") or atype.startswith("store")):
            action_elements.extend(self._gen_assert_store_mngr(atype, tag, param, value, selectors))

        elif atype in ("click", "type", "keys", "submit"):
            action_elements.extend(self._gen_keys_mngr(atype, param, selectors))

        elif atype == 'echo' and tag == 'string':
            if len(param) > 0 and not selectors:
                action_elements.append(ast_call(
                    func="print",
                    args=[self._gen_expr(param.strip())]))

        elif atype == "script" and tag == "eval":
            escaped_param = self._escape_js_blocks(param)
            action_elements.append(ast_call(func=ast_attr("self.driver.execute_script"),
                                            args=[self._gen_expr(escaped_param)]))
        elif atype == "rawcode":
            action_elements.append(ast.parse(param))
        elif atype == 'go':
            if param:
                action_elements.append(ast_call(func=ast_attr("self.driver.get"),
                                                args=[self._gen_expr(param.strip())]))
                action_elements.append(self._gen_replace_dialogs())
        elif atype == "editcontent":
            action_elements.extend(self._gen_edit_mngr(param, selectors))
        elif atype.startswith('waitfor'):
            action_elements.extend(self._gen_wait_for(atype, param, value, selectors))
        elif atype == 'pausefor':
            action_elements.extend(self._gen_sleep_mngr(param))
        elif atype == 'clear' and tag == 'cookies':
            action_elements.append(ast_call(
                func=ast_attr("self.driver.delete_all_cookies")))
        elif atype == 'screenshot':
            action_elements.extend(self._gen_screenshot_mngr(param))
        elif atype == 'alert':
            action_elements.extend(self._gen_alert(param))
        elif atype == 'if':
            action_elements.append(self._gen_condition_mngr(param, action_config))
        elif atype == 'loop':
            action_elements.append(self._gen_loop_mngr(action_config))
        elif atype == 'foreach':
            action_elements.append(self._gen_foreach_mngr(action_config))

        if not action_elements and not self.ignore_unknown_actions:
            raise TaurusInternalException("Could not build code for action: %s" % action_config)

        if atype.lower() in self.ACTIONS_WITH_WAITER:
            action_elements.append(ast_call(func=ast_attr("waiter"), args=[]))

        return [ast.Expr(element) for element in action_elements]

    def _gen_foreach_mngr(self, action_config):
        self.selenium_extras.add("get_elements")
        exc = TaurusConfigError("Foreach loop must contain locators and do")
        elements = []
        locators = action_config.get('locators', exc)
        body = []
        for action in action_config.get('do', exc):
            body = body + self._gen_action(action)

        body_list = []
        # filter out empty AST expressions that cause empty lines in the generated code
        for item in body:
            if isinstance(item.value, list):
                if len(item.value) > 0:
                    body_list.append(item)
            else:
                body_list.append(item)

        elements.append(self._gen_get_elements_call("elements", locators))
        elements.append(
            ast.For(target=ast.Name(id=action_config.get('foreach'), ctx=ast.Store()), iter=ast.Name(id="elements"),
                    body=body_list,
                    orelse=[]))

        return elements

    def _gen_wait_for(self, atype, param, value, selectors):
        self.selenium_extras.add("wait_for")
        supported_conds = ["present", "visible", "clickable", "notpresent", "notvisible", "notclickable"]

        if not value:
            value = 10  # if timeout value is not present set it by default to 10s
        timeout = dehumanize_time(value)

        if param.lower() not in supported_conds:
            raise TaurusConfigError("Invalid condition in %s: '%s'. Supported conditions are: %s." %
                                    (atype, param, ", ".join(supported_conds)))

        return [ast_call(func="wait_for",
                         args=[ast.Str(param, kind=""),
                               ast.List(elts=self._gen_ast_locators_dict(selectors)),
                               ast.Num(timeout, kind="")])]

    def _gen_answer_dialog(self, type, value):
        if type not in ['alert', 'prompt', 'confirm']:
            raise TaurusConfigError("answerDialog type must be one of the following: 'alert', 'prompt' or 'confirm'")
        if type == 'confirm' and str(value).lower() not in ['#ok', '#cancel']:
            raise TaurusConfigError("answerDialog of type confirm must have value either '#Ok' or '#Cancel'")
        if type == 'alert' and str(value).lower() != '#ok':
            raise TaurusConfigError("answerDialog of type alert must have value '#Ok'")
        dlg_method = "dialogs_answer_on_next_%s" % type
        self.selenium_extras.add(dlg_method)
        return [ast_call(func=ast_attr(dlg_method), args=[ast.Str(value, kind="")])]

    def _gen_assert_dialog(self, type, value):
        if type not in ['alert', 'prompt', 'confirm']:
            raise TaurusConfigError("assertDialog type must be one of the following: 'alert', 'prompt' or 'confirm'")
        elements = []
        dlg_method = "dialogs_get_next_%s" % type
        self.selenium_extras.add(dlg_method)
        elements.append(ast.Assign(targets=[ast.Name(id='dialog', ctx=ast.Store())],
                                   value=ast_call(
                                       func=ast_attr(dlg_method))))
        elements.append(ast_call(
            func=ast_attr("self.assertIsNotNone"),
            args=[ast.Name(id='dialog'), ast.Str("No dialog of type %s appeared" % type, kind="")]))
        elements.append(ast_call(
            func=ast_attr("self.assertEqual"),
            args=[ast.Name(id='dialog'), ast.Str(value, kind=""), ast.Str("Dialog message didn't match", kind="")]))

        return elements

    def _gen_replace_dialogs(self):
        """
        Generates the call to DialogsManager to replace dialogs
        """
        if not self.replace_dialogs:
            return []

        method = "dialogs_replace"
        self.selenium_extras.add(method)
        return [
            gen_empty_line_stmt(),
            ast_call(
                func=ast_attr(method))
        ]

    @staticmethod
    def _convert_to_number(arg):
        if isinstance(arg, str) and arg.isdigit():
            return int(arg)
        return arg

    def _gen_loop_mngr(self, action_config):
        extra_method = "get_loop_range"
        self.selenium_extras.add(extra_method)
        exc = TaurusConfigError("Loop must contain start, end and do")
        start = self._convert_to_number(action_config.get('start', exc))
        end = self._convert_to_number(action_config.get('end', exc))
        step = self._convert_to_number(action_config.get('step')) or 1
        elements = []

        body = [
            ast.Assign(
                targets=[self._gen_expr("${%s}" % action_config['loop'])],
                value=ast_call(func=ast_attr("str"), args=[ast.Name(id=action_config['loop'])]))
        ]
        actions = action_config.get('do', exc)
        if len(actions) == 0:
            raise exc
        for action in actions:
            body.append(self._gen_action(action))

        range_args = [self.expr_compiler.gen_expr(start),
                      self.expr_compiler.gen_expr(end),
                      self.expr_compiler.gen_expr(step)]

        elements.append(
            ast.For(target=ast.Name(id=action_config.get('loop'),
                                    ctx=ast.Store()),
                    iter=ast_call(func=ast_attr(extra_method),
                                  args=range_args),
                    body=body,
                    orelse=[]))

        return elements

    def _gen_eval_js_expression(self, js_expr):
        return ast_call(func=ast_attr("self.driver.execute_script"), args=[self._gen_expr("return %s;" % js_expr)])

    def _gen_condition_mngr(self, param, action_config):
        if not action_config.get('then'):
            raise TaurusConfigError("Missing then branch in if statement")

        test = ast.Assign(targets=[ast.Name(id='test', ctx=ast.Store())],
                          value=self._gen_eval_js_expression(param))

        body = []
        for action in action_config.get('then'):
            body.append(self._gen_action(action))

        orelse = []
        if action_config.get('else'):
            for action in action_config.get('else'):
                orelse.append(self._gen_action(action))

        return [test,
                [ast.If(
                    test=[ast.Name(id='test')],
                    body=body,
                    orelse=orelse)]]

    def _check_platform(self):
        mobile_browsers = ["chrome", "safari"]
        mobile_platforms = ["android", "ios"]

        browser = self.capabilities.get("browserName", "")
        browser = self.scenario.get("browser", browser)
        browser = browser.lower()
        if browser == "microsoftedge":
            browser = "edge"
        local_browsers = ["firefox", "chrome", "ie", "opera", "edge"] + mobile_browsers

        browser_platform = None
        if browser:
            browser_split = browser.split("-")
            browser = browser_split[0]
            if len(browser_split) > 1:
                browser_platform = browser_split[1]

        if self.remote_address:
            if browser and browser != "remote":
                msg = "Forcing browser to Remote, because of remote WebDriver address, use '%s' as browserName"
                self.log.warning(msg % browser)
                self.capabilities["browserName"] = browser
            browser = "remote"
            if self.generate_markers is None:  # if not set by user - set to true
                self.generate_markers = True
        elif browser in mobile_browsers and browser_platform in mobile_platforms:
            self.appium = True
            self.remote_address = "http://localhost:4723/wd/hub"
            self.capabilities["platformName"] = browser_platform
            self.capabilities["browserName"] = browser
            browser = "remote"  # Force using remote web driver
        elif not browser:
            browser = "firefox"
        elif browser not in local_browsers:  # browser isn't supported
            raise TaurusConfigError("Unsupported browser name: %s" % browser)
        return browser

    def _get_scenario_timeout(self):
        return dehumanize_time(self.scenario.get("timeout", "30s"))

    def _gen_webdriver(self):
        self.log.debug("Generating setUp test method")

        browser = self._check_platform()
        body = [self._get_options(browser)]

        if browser == 'firefox':
            body.extend(self._get_firefox_profile() + [self._get_firefox_webdriver()])

        elif browser == 'chrome':
            body.extend(self._get_chrome_profile() + [self._get_chrome_webdriver()])

        elif browser == 'edge':
            body.extend([self._get_edge_webdriver()])

        elif browser == 'remote':
            if self.selenium_version.startswith("4"):
                remote_profile = self._get_remote_profile() + [self._get_remote_webdriver()]
            else:
                remote_profile = self._get_remote_webdriver()
            body.append(remote_profile)

        else:
            body.append(ast.Assign(
                targets=[ast_attr("self.driver")],
                value=ast_call(
                    func=ast_attr("webdriver.%s" % browser))))

        body.append(self._get_timeout())
        body.extend(self._get_extra_mngrs())

        return body

    def _wrap_with_try_except(self, web_driver_cmds):
        body = [ast.Assign(targets=[ast_attr("self.driver")], value=ast_attr("None")),
                self._gen_new_session_start()]

        exception_variables = [ast.Name(id='ex_type'), ast.Name(id='ex'), ast.Name(id='tb')]
        exception_handler = [
            ast.Assign(targets=[ast.Tuple(elts=exception_variables)],
                       value=ast_call(func=ast_attr('sys.exc_info'), args=[]))]

        exception_handler.extend(self._gen_new_session_end(True))

        exception_handler.append(
            ast.Expr(value=ast_call(
                func=ast_attr('apiritif.log.error'),
                args=[
                    self._gen_expr(ast_attr("str(traceback.format_exception(ex_type, ex, tb))"))
                ])))
        exception_handler.append(gen_raise())

        body.append(gen_try_except(try_body=web_driver_cmds, exception_body=exception_handler))
        body.append(self._gen_new_session_end())

        return body

    def _get_timeout(self):
        return ast.Expr(
            ast_call(
                func=ast_attr("self.driver.implicitly_wait"),
                args=[ast_attr("timeout")]))

    def _get_extra_mngrs(self):
        mngrs = []
        mgr = "WindowManager"
        if mgr in self.selenium_extras:
            mngrs.append(ast.Assign(
                targets=[ast_attr("self.wnd_mng")],
                value=ast_call(
                    func=ast.Name(id=mgr))))

        mgr = "FrameManager"
        if mgr in self.selenium_extras:
            mngrs.append(ast.Assign(
                targets=[ast_attr("self.frm_mng")],
                value=ast_call(
                    func=ast.Name(id=mgr))))
        return mngrs

    def _get_headless_setup(self):
        if self.scenario.get("headless", False):
            self.log.info("Headless mode works only with Selenium 3.8.0+, be sure to have it installed")
            if self.selenium_version.startswith("4"):
                return [ast.Assign(
                    targets=[ast_attr("options.headless")], value=ast_attr("True"))]
            return [ast.Expr(
                ast_call(func=ast_attr("options.set_headless")))]
        else:
            return []

    def _get_options(self, browser):
        if browser == 'remote':
            browser = self.capabilities.get("browserName", "").lower()
            if browser in ["microsoftedge", "edge"]:
                browser = 'edge'
            elif browser == 'safari' and 'api/v4/grid/wd/hub' in self.remote_address:
                browser = 'MiniBrowser'  # use MiniBrowser instead of safari as a remote browser in Blazemeter
        if browser == 'firefox':
            options = self._get_firefox_options()
        elif browser == 'chrome':
            options = self._get_chrome_options()
        elif browser == 'edge' and self.selenium_version.startswith("4"):
            options = self._get_edge_options()
        elif browser == 'MiniBrowser':
            options = self._get_webkitgtk_options()
        else:
            if self.selenium_version.startswith("4"):
                options = [ast.Assign(targets=[ast.Name(id="options")], value=ast_call(func=ast_attr("ArgOptions")))]
            else:
                options = [ast.Assign(targets=[ast_attr("options")], value=ast_attr("None"))]

        if self.OPTIONS in self.executor.settings:
            self.log.debug(f'Generating selenium option {self.executor.settings.get(self.OPTIONS)}. '
                           f'Browser {browser}. Selenium version {self.selenium_version}')
            options.extend(self._get_selenium_options(browser))

        return options

    def _get_firefox_options(self):
        firefox_options = [
            ast.Assign(
                targets=[ast.Name(id="options")],
                value=ast_call(
                    func=ast_attr("webdriver.FirefoxOptions")))]

        return firefox_options + self._get_headless_setup()

    def _get_chrome_options(self):
        chrome_options = [
            ast.Assign(
                targets=[ast.Name(id="options")],
                value=ast_call(
                    func=ast_attr("webdriver.ChromeOptions"))),
            ast.Expr(
                ast_call(
                    func=ast_attr("options.add_argument"),
                    args=[ast.Str("--no-sandbox", kind="")])),
            ast.Expr(
                ast_call(
                    func=ast_attr("options.add_argument"),
                    args=[ast.Str("--disable-dev-shm-usage", kind="")])),
            ast.Expr(
                ast_call(
                    func=ast_attr("options.add_argument"),
                    args=[ast.Str("--disable-gpu", kind="")])),

            ast.Expr(
                ast_call(
                    func=ast_attr("options.set_capability"),
                    args=[ast.Str("unhandledPromptBehavior", kind=""),
                          ast.Str("ignore", kind="")]))]

        return chrome_options + self._get_headless_setup()

    def _get_edge_options(self):
        edge_options = [
            ast.Assign(
                targets=[ast.Name(id="options")],
                value=ast_call(func=ast_attr("webdriver.EdgeOptions")))]

        return edge_options + self._get_headless_setup()

    def _get_webkitgtk_options(self):
        return [
            ast.Assign(
                targets=[ast.Name(id="options")],
                value=ast_call(func=ast_attr("webdriver.WebKitGTKOptions")))]

    def _get_firefox_profile(self):
        capabilities = sorted(self.capabilities.keys())
        cap_expr = []
        for capability in capabilities:
            cap_expr.append([
                ast.Expr(
                    ast_call(
                        func=ast_attr("options.set_capability"),
                        args=[ast.Str(capability, kind=""), ast.Str(self.capabilities[capability], kind="")]))

            ])

        return [
                   ast.Assign(
                       targets=[ast.Name(id="profile")],
                       value=ast_call(func=ast_attr("webdriver.FirefoxProfile"))),
                   ast.Expr(ast_call(
                       func=ast_attr("profile.set_preference"),
                       args=[ast.Str("webdriver.log.file", kind=""), ast.Str(self.wdlog, kind="")])),
                   ast.Expr(
                       ast_call(
                           func=ast_attr("options.set_capability"),
                           args=[ast.Str("unhandledPromptBehavior", kind=""), ast.Str("ignore", kind="")]))] + cap_expr

    def _get_chrome_profile(self):
        capabilities = sorted(self.capabilities.keys())
        cap_expr = []
        for capability in capabilities:
            cap_expr.append([
                ast.Expr(
                    ast_call(
                        func=ast_attr("options.set_capability"),
                        args=[ast.Str(capability, kind=""), ast.Str(self.capabilities[capability], kind="")]))

            ])

        return cap_expr

    def _get_remote_profile(self):
        capabilities = sorted(self.capabilities.keys())
        if "browserName" in capabilities and self.capabilities.get('browserName').lower() in ["microsoftedge", "edge"]:
            self.capabilities["browserName"] = "MicrosoftEdge"  # MicrosoftEdge in camel case is necessary
        cap_expr = []
        for capability in capabilities:
            cap_expr.append([
                ast.Expr(
                    ast_call(
                        func=ast_attr("options.set_capability"),
                        args=[ast.Str(capability, kind=""), ast.Str(self.capabilities[capability], kind="")]))

            ])

        return cap_expr

    def _get_firefox_webdriver(self):
        return ast.Assign(
            targets=[ast_attr("self.driver")],
            value=ast_call(
                func=ast_attr("webdriver.Firefox"),
                args=[ast.Name(id="profile")],
                keywords=[ast.keyword(
                    arg="options",
                    value=ast.Name(id="options"))]))

    def _get_chrome_webdriver(self):
        return ast.Assign(
            targets=[ast_attr("self.driver")],
            value=ast_call(
                func=ast_attr("webdriver.Chrome"),
                keywords=[
                    ast.keyword(
                        arg="service_log_path",
                        value=ast.Str(self.wdlog, kind="")),
                    ast.keyword(
                        arg="options",
                        value=ast.Name(id="options"))]))

    def _get_edge_webdriver(self):
        return ast.Assign(
            targets=[ast_attr("self.driver")],
            value=ast_call(
                func=ast_attr("webdriver.Edge")))

    def _get_remote_webdriver(self):
        if self.selenium_version.startswith("4"):
            return ast.Assign(
                targets=[ast_attr("self.driver")],
                value=ast_call(
                    func=ast_attr("webdriver.Remote"),
                    keywords=[
                        ast.keyword(
                            arg="command_executor",
                            value=ast.Str(self.remote_address, kind="")),
                        ast.keyword(
                            arg="options",
                            value=ast.Name(id="options"))]))
        else:
            keys = sorted(self.capabilities.keys())
            if "browserName" in keys and self.capabilities.get('browserName').lower() in ["microsoftedge", "edge"]:
                self.capabilities["browserName"] = "MicrosoftEdge"  # MicrosoftEdge in camel case is necessary
            values = [self.capabilities[key] for key in keys]

            return ast.Assign(
                targets=[ast_attr("self.driver")],
                value=ast_call(
                    func=ast_attr("webdriver.Remote"),
                    keywords=[
                        ast.keyword(
                            arg="command_executor",
                            value=ast.Str(self.remote_address, kind="")),
                        ast.keyword(
                            arg="desired_capabilities",
                            value=ast.Dict(
                                keys=[ast.Str(key, kind="") for key in keys],
                                values=[ast.Str(value, kind="") for value in values])),
                        ast.keyword(
                            arg="options",
                            value=ast.Name(id="options"))]))

    def _get_selenium_options(self, browser):
        options = []

        old_version = self.selenium_version.startswith("3")

        if old_version and browser not in ['firefox', 'chrome', 'MiniBrowser']:
            self.log.warning(
                f'Selenium options are not supported. Browser {browser}. Selenium version {self.selenium_version}')
        else:
            for opt in self.executor.settings.get(self.OPTIONS):
                if opt == "ignore-proxy":
                    options.extend(self._get_ignore_proxy(old_version))
                elif opt == "arguments":
                    options.extend(self._get_arguments())
                elif opt == "experimental-options":
                    options.extend(self._get_experimental_options(browser))
                elif opt == "preferences":
                    options.extend(self._get_preferences(browser))
                else:
                    self.log.warning(f'Unknown option {opt}')

        return options

    def _get_ignore_proxy(self, old_version):
        ignore_proxy = "ignore-proxy"

        if old_version:
            self.log.warning(f'Option {ignore_proxy} is not supported for this selenium version')
            return []
        if not self.executor.settings.get(self.OPTIONS).get(ignore_proxy):
            return []

        return [ast.Expr(ast_call(func=ast_attr("options.ignore_local_proxy_environment_variables")))]

    def _get_arguments(self):
        args = []
        arguments = "arguments"

        for arg in self.executor.settings.get(self.OPTIONS).get(arguments):
            args.extend([ast.Expr(
                ast_call(
                    func=ast_attr("options.add_argument"),
                    args=[ast.Str(arg, kind="")]))])

        return args

    def _get_experimental_options(self, browser):
        experimental_options = "experimental-options"

        if browser != "chrome":
            self.log.warning(f'Option {experimental_options} is not supported for {browser}')
            return []

        exp_opts = []
        for key, value in self.executor.settings.get(self.OPTIONS).get(experimental_options).items():
            exp_opts.append(ast.Expr(ast_call(
                func=ast_attr("options.add_experimental_option"),
                args=[
                    [ast.Str(key, kind="")],
                    [ast.Str(value, kind="")]])))
        return exp_opts

    def _get_preferences(self, browser):
        preferences = "preferences"

        if browser != "firefox":
            self.log.warning(f'Option {preferences} is not supported for {browser}')
            return []

        prefers = []
        for key, value in self.executor.settings.get(self.OPTIONS).get(preferences).items():
            prefers.append(ast.Expr(ast_call(
                func=ast_attr("options.set_preference"),
                args=[
                    [ast.Str(key, kind="")],
                    [ast.Str(value, kind="")]])))

        return prefers

    @staticmethod
    def _gen_impl_wait(timeout):
        return ast.Expr(
            ast_call(
                func=ast_attr("self.driver.implicitly_wait"),
                args=[ast.Num(dehumanize_time(timeout), kind="")]))

    def _gen_module(self):
        stmts = []

        if self.verbose:
            stmts.extend(self._gen_logging())

        stmts.extend(self._gen_data_source_readers())
        stmts.append(self._gen_classdef())

        stmts = self._gen_imports() + stmts

        return ast.Module(body=stmts)

    def _gen_imports(self):
        imports = [
            ast.Import(names=[ast.alias(name='logging', asname=None)]),
            ast.Import(names=[ast.alias(name='random', asname=None)]),
            ast.Import(names=[ast.alias(name='string', asname=None)]),
            ast.Import(names=[ast.alias(name='sys', asname=None)]),
            ast.Import(names=[ast.alias(name='unittest', asname=None)]),
            ast.ImportFrom(
                module="time",
                names=[
                    ast.alias(name="time", asname=None),
                    ast.alias(name="sleep", asname=None)],
                level=0),
            gen_empty_line_stmt(),
            ast.Import(names=[ast.alias(name='apiritif', asname=None)]),  # or "from apiritif import http, utils"?
            gen_empty_line_stmt()]

        if self.generate_external_handler:
            imports.append(ast.Import(names=[ast.alias(name='traceback', asname=None)]))
            self.selenium_extras.update(self.EXTERNAL_HANDLER_TAGS)

        if self.test_mode == "selenium":
            if self.appium:
                source = "appium"
            else:
                source = "selenium"

            imports.append(ast.parse(self.IMPORTS % source).body)
            if self.selenium_version.startswith("4"):
                imports.append(
                    ast.ImportFrom(
                        module="selenium.webdriver.common.options",
                        names=[ast.alias(name="ArgOptions", asname=None)],
                        level=0
                    )
                )
            self.selenium_extras.add("get_locator")
            self.selenium_extras.add("waiter")
            extra_names = [ast.alias(name=name, asname=None) for name in self.selenium_extras]
            imports.append(
                ast.ImportFrom(
                    module="bzt.resources.selenium_extras",
                    names=extra_names,
                    level=0))

        return imports

    def _gen_data_source_readers(self):
        readers = []
        for idx, source in enumerate(self.data_sources, start=1):
            keywords = []

            if "variable-names" in source:
                fieldnames = ast.keyword()
                fieldnames.arg = "fieldnames"
                str_names = source.get("variable-names").split(",")
                fieldnames.value = ast.List(elts=[ast.Str(s=fname, kind="") for fname in str_names])
                keywords.append(fieldnames)

            if "loop" in source:
                loop = ast.keyword()
                loop.arg = "loop"
                loop.value = ast.Name(id=source.get("loop"))
                keywords.append(loop)

            if "quoted" in source:
                quoted = ast.keyword()
                quoted.arg = "quoted"
                quoted.value = ast.Name(id=source.get("quoted"))
                keywords.append(quoted)

            if "delimiter" in source:
                delimiter = ast.keyword()
                delimiter.arg = "delimiter"
                delimiter.value = ast.Str(s=source.get("delimiter"), kind="")
                keywords.append(delimiter)

            if "encoding" in source:
                encoding = ast.keyword()
                encoding.arg = "encoding"
                encoding.value = ast.Str(s=source.get("encoding"), kind="")
                keywords.append(encoding)

            csv_file = self.scenario.engine.find_file(source["path"])
            reader = ast.Assign(
                targets=[ast.Name(id="reader_%s" % idx)],
                value=ast_call(
                    func=ast_attr("apiritif.CSVReaderPerThread"),
                    args=[ast.Str(s=csv_file, kind="")],
                    keywords=keywords))

            readers.append(reader)

        if readers:
            readers.append(gen_empty_line_stmt())

        return readers

    def _gen_classdef(self):
        class_body = [self._gen_test_methods()]
        class_body = [self._gen_class_setup()] + class_body  # order is important for selenium_extras set

        if self.test_mode == "selenium":
            class_body.append(self._gen_class_teardown())

        return ast.ClassDef(
            name=create_class_name(self.label),
            bases=[ast_attr("unittest.TestCase")],
            body=class_body,
            keywords=[],
            starargs=None,
            kwargs=None,
            decorator_list=[])

    def _gen_class_setup(self):
        data_sources = [self._gen_default_vars()]
        for idx in range(len(self.data_sources)):
            data_sources.append(ast.Expr(ast_call(func=ast_attr("reader_%s.read_vars" % (idx + 1)))))

        for idx in range(len(self.data_sources)):
            extend_vars = ast_call(
                func=ast_attr("self.vars.update"),
                args=[ast_call(
                    func=ast_attr("reader_%s.get_vars" % (idx + 1)))])
            data_sources.append(ast.Expr(extend_vars))

        if self.test_mode == "apiritif":
            target_init = self._gen_api_target()
        else:
            target_init = self._gen_webdriver()

        handlers = []
        if self.generate_markers:
            func_name = "add_flow_markers"
            self.selenium_extras.add(func_name)
            handlers.append(ast.Expr(ast_call(func=func_name)))

        stored_vars = {
            "timeout": "timeout",
            "func_mode": str(self.executor.engine.is_functional_mode())}

        if target_init:
            if self.test_mode == "selenium":
                stored_vars["driver"] = "self.driver"
                stored_vars["windows"] = "{}"

        has_ds = bool(list(self.scenario.get_data_sources()))
        stored_vars['scenario_name'] = [ast.Str(self.label, kind="")]
        if has_ds:
            stored_vars['data_sources'] = str(has_ds)

        store_call = ast_call(
            func=ast_attr("apiritif.put_into_thread_store"),
            keywords=[ast.keyword(arg=key, value=ast_attr(stored_vars[key])) for key in stored_vars],
            args=[])

        store_block = [ast.Expr(store_call)]

        timeout_setup = [ast.Expr(ast.Assign(
            targets=[ast_attr("timeout")],
            value=ast.Num(self._get_scenario_timeout(), kind="")))]
        body = data_sources + timeout_setup + target_init + handlers + store_block
        if self.generate_external_handler:
            body = self._wrap_with_try_except(body)
        setup = ast.FunctionDef(
            name="setUp",
            args=[ast_attr("self")],
            body=body,
            decorator_list=[])
        return [setup, gen_empty_line_stmt()]

    def _gen_class_teardown(self):
        body = [
            ast.If(
                test=ast_attr("self.driver"),
                body=ast.Expr(ast_call(func=ast_attr("self.driver.quit"))), orelse=[])]

        return ast.FunctionDef(name="tearDown", args=[ast_attr("self")], body=body, decorator_list=[])

    def _nfc_preprocess(self, requests):
        setup = []
        main = []
        teardown = []

        while requests:
            request = requests.pop(0)
            if isinstance(request, SetUpBlock):
                setup.extend(request.requests)
            elif isinstance(request, TearDownBlock):
                teardown.extend(request.requests)
            else:
                main.append(request)

        requests.extend(setup + main)
        if teardown:
            requests.extend([self.FINALLY_MARKER] + teardown)

    def _gen_test_methods(self):
        methods = []
        try_block_content = []
        finally_block_content = []
        finally_marker = False

        requests = self.scenario.get_requests(parser=HierarchicRequestParser, require_url=False)
        self._nfc_preprocess(requests)

        number_of_digits = int(math.log10(len(requests))) + 1
        index = 1

        while requests:
            request = requests.pop(0)
            if request == self.FINALLY_MARKER:
                finally_marker = True
                continue
            if not isinstance(request, self.SUPPORTED_BLOCKS):
                msg = "Apiritif script generator doesn't support '%s' blocks, skipping"
                self.log.warning(msg, request.NAME)
                continue

            # convert top-level http request to transaction
            if isinstance(request, HTTPRequest):
                request = TransactionBlock(
                    name=request.label,
                    requests=[request],
                    include_timers=[],
                    config=request.config,
                    scenario=request.scenario)

            if isinstance(request, TransactionBlock):
                body = [self._gen_transaction(request)]
                label = create_method_name(request.label[:40])
            elif isinstance(request, IncludeScenarioBlock):
                body = [self._gen_transaction(request)]
                label = create_method_name(request.scenario_name)
            elif isinstance(request, SetVariables):
                body = self._gen_set_vars(request)
                label = request.config.get("label", "set_variables")
            else:
                return

            counter = str(index).zfill(number_of_digits)
            index += 1
            method_name = '_' + counter + '_' + label

            if isinstance(request, SetVariables):
                self.service_methods.append(label)  # for sample excluding

            methods.append(self._gen_test_method(method_name, body))
            if finally_marker:
                finally_block_content.append(method_name)
            else:
                try_block_content.append(method_name)

        methods.append(self._gen_master_test_method(try_block_content, finally_block_content))
        return methods

    def _gen_set_vars(self, request):
        res = []
        for name in sorted(request.mapping.keys()):
            res.append(ast.Assign(
                targets=[self._gen_expr("${%s}" % name)],
                value=ast.Str(s="%s" % request.mapping[name], kind="")))

        return res

    def _gen_master_test_method(self, try_block, finally_block):
        if not try_block:
            raise TaurusConfigError("Supported transactions not found, test is empty")

        body = []
        for slave_name in try_block:
            body.append(ast.Expr(ast_call(func=ast_attr("self." + slave_name))))

        finally_body = []
        for slave_name in finally_block:
            finally_body.append(ast.Expr(ast_call(func=ast_attr("self." + slave_name))))

        if finally_body:
            teardown_marker = ast.Expr(ast_call(func=ast_attr("apiritif.set_stage"), args=[self._gen_expr("teardown")]))
            finally_body.insert(0, teardown_marker)
            body = [gen_try_except(try_body=body, final_body=finally_body)]

        name = 'test_' + create_method_name(self.label)
        return self._gen_test_method(name=name, body=body)

    @staticmethod
    def _gen_test_method(name, body):
        # 'test_01_get_posts'
        return ast.FunctionDef(
            name=name,
            args=[ast.Name(id='self', ctx=ast.Param())],
            body=body,
            decorator_list=[])

    def _gen_expr(self, value):
        return self.expr_compiler.gen_expr(value)

    @staticmethod
    def _escape_js_blocks(value):  # escapes plain { with {{
        if not value:
            return value
        value = value.replace("{", "{{").replace("}", "}}")
        while True:
            blocks = re.finditer(r"\${{[\w\d]*}}", value)
            block = next(blocks, None)
            if block:
                start, end = block.start(), block.end()
                line = "$" + value[start + 2:end - 1]
                value = value[:start] + line + value[end:]
            else:
                break
        return value

    def _gen_target_setup(self, key, value):
        return ast.Expr(ast_call(
            func=ast_attr("self.target.%s" % key),
            args=[self._gen_expr(value)]))

    def _access_method(self):
        keepalive = self.scenario.get("keepalive", None)
        default_address = self.scenario.get("default-address", None)
        store_cookie = self.scenario.get("store-cookie", None)

        if default_address is not None or keepalive or store_cookie:
            return ApiritifScriptGenerator.ACCESS_TARGET
        else:
            return ApiritifScriptGenerator.ACCESS_PLAIN

    def _gen_api_target(self):
        keepalive = self.scenario.get("keepalive", None)
        base_path = self.scenario.get("base-path", None)
        auto_assert_ok = self.scenario.get("auto-assert-ok", True)
        store_cookie = self.scenario.get("store-cookie", None)
        timeout = self.scenario.get("timeout", None)
        follow_redirects = self.scenario.get("follow-redirects", True)

        if keepalive is None:
            keepalive = True
        if store_cookie is None:
            store_cookie = True

        target = []
        if self._access_method() == ApiritifScriptGenerator.ACCESS_TARGET:

            target.extend([
                self._init_target(),
                self._gen_target_setup('keep_alive', keepalive),
                self._gen_target_setup('auto_assert_ok', auto_assert_ok),
                self._gen_target_setup('use_cookies', store_cookie),
                self._gen_target_setup('allow_redirects', follow_redirects),
            ])
            if base_path:
                target.append(self._gen_target_setup('base_path', base_path))
            if timeout is not None:
                target.append(self._gen_target_setup('timeout', dehumanize_time(timeout)))
            target.append(gen_empty_line_stmt())
        return target

    def _init_target(self):
        default_address = self.scenario.get("default-address", "")

        target_call = ast_call(
            func=ast_attr("apiritif.http.target"),
            args=[self._gen_expr(default_address)])

        target = ast.Assign(
            targets=[ast_attr("self.target")],
            value=target_call)

        return target

    def _extract_named_args(self, req):
        named_args = OrderedDict()

        no_target = self._access_method() != ApiritifScriptGenerator.ACCESS_TARGET
        if req.timeout is not None:
            named_args['timeout'] = dehumanize_time(req.timeout)
        elif "timeout" in self.scenario and no_target:
            named_args['timeout'] = dehumanize_time(self.scenario.get("timeout"))

        follow_redirects = req.priority_option('follow-redirects', None)
        if follow_redirects is not None:
            named_args['allow_redirects'] = follow_redirects

        headers = {}
        headers.update(self.scenario.get("headers"))
        headers.update(req.headers)

        if headers:
            named_args['headers'] = self._gen_expr(headers)

        merged_headers = dict([(key.lower(), value) for key, value in iteritems(headers)])
        content_type = merged_headers.get("content-type")

        if content_type == 'application/json' and isinstance(req.body, (dict, list)):  # json request body
            named_args['json'] = self._gen_expr(req.body)
        elif req.method.lower() == "get" and isinstance(req.body, dict):  # request URL params (?a=b&c=d)
            named_args['params'] = self._gen_expr(req.body)
        elif isinstance(req.body, dict):  # form data
            named_args['data'] = self._gen_expr(list(iteritems(req.body)))
        elif isinstance(req.body, str):
            named_args['data'] = self._gen_expr(req.body)
        elif req.body:
            msg = "Cannot handle 'body' option of type %s: %s"
            raise TaurusConfigError(msg % (type(req.body), req.body))

        cert = self.scenario.get("certificate")
        cert_pass = self.scenario.get("passphrase", None)
        if cert:
            named_args['encrypted_cert'] = (self.executor.engine.find_file(cert), cert_pass)

        if cert_pass and not cert:
            self.log.warning("Passphrase was found, but certificate is missing!")

        return named_args

    # generate transactions recursively
    def _gen_transaction(self, trans_conf, transaction_class="apiritif.smart_transaction"):
        body = []
        if isinstance(trans_conf, IncludeScenarioBlock):
            included = self.executor.get_scenario(trans_conf.scenario_name)
            included_requests = included.get_requests(parser=HierarchicRequestParser, require_url=False)
            trans_conf = TransactionBlock(
                name=trans_conf.scenario_name,
                requests=included_requests,
                include_timers=[],
                config=included.data,
                scenario=included)
        for request in trans_conf.requests:
            if isinstance(request, TransactionBlock) or isinstance(request, IncludeScenarioBlock):
                body.append(self._gen_transaction(request, transaction_class="apiritif.transaction"))
            elif isinstance(request, SetVariables):
                body.append(self._gen_set_vars(request))
            else:
                body.append(self._gen_http_request(request))

        transaction = ast.With(
            context_expr=ast_call(
                func=ast_attr(transaction_class),
                args=[self._gen_expr(trans_conf.label)]),
            optional_vars=None,
            body=body)

        return transaction

    def _escape_values_for_external_handler(self, action):
        if isinstance(action, dict):
            action_type = action.get("type")
            if isinstance(action_type, str) and action_type.endswith("Eval"):
                value = self._escape_js_blocks(action.get("value"))
                action["value"] = value
        return action

    def _gen_http_request(self, req):
        lines = []
        think_time = dehumanize_time(req.get_think_time())

        if req.url:
            if self.test_mode == "selenium":
                if req.timeout:
                    lines.append(self._gen_impl_wait(req.timeout))
                default_address = self.scenario.get("default-address")
                parsed_url = parse.urlparse(req.url)
                if default_address and not parsed_url.netloc:
                    url = default_address + req.url
                else:
                    url = req.url

                get_url_lines = [ast.Expr(
                    ast_call(
                        func=ast_attr("self.driver.get"),
                        args=[self._gen_expr(url)]))]
                if self.generate_external_handler:
                    action = BetterDict.from_dict({f"go({url})": None})
                    get_url_lines = self._gen_action_start(action) + get_url_lines + self._gen_action_end(action)

                lines.extend(get_url_lines)

                if "actions" in req.config:
                    self.replace_dialogs = self._is_dialog_replacement_needed(req.config.get("actions"))
                    lines.append(self._gen_replace_dialogs())

            else:
                method = req.method.lower()
                named_args = self._extract_named_args(req)

                if self._access_method() == ApiritifScriptGenerator.ACCESS_TARGET:
                    requestor = ast_attr("self.target")
                else:
                    requestor = ast_attr("apiritif.http")

                keywords = [ast.keyword(
                    arg=name,
                    value=self._gen_expr(value)) for name, value in iteritems(named_args)]

                lines.append(ast.Assign(
                    targets=[ast.Name(id="response")],
                    value=ast_call(
                        func=ast_attr((requestor, method)),
                        args=[self._gen_expr(req.url)],
                        keywords=keywords)))

        elif "actions" not in req.config:
            self.log.warning("'url' and/or 'actions' are mandatory for request but not found: '%s'", req.config)
            return [ast.Pass()]

        if self.test_mode == "selenium":
            actions = req.config.get("actions")
            self.replace_dialogs = self._is_dialog_replacement_needed(actions)

            for action in actions:
                action_lines = self._gen_action(action)
                if self.generate_external_handler:
                    action = self._escape_values_for_external_handler(action)
                    action_lines = self._gen_action_start(action) + action_lines + self._gen_action_end(action)

                lines.extend(action_lines)

            if "assert" in req.config:
                lines.append(ast.Assign(
                    targets=[ast.Name(id="body")],
                    value=ast_attr("self.driver.page_source")))
                for assert_config in req.config.get("assert"):
                    lines.extend(self._gen_sel_assertion(assert_config))

        else:
            lines.extend(self._gen_assertions(req))
            lines.extend(self._gen_jsonpath_assertions(req))
            lines.extend(self._gen_xpath_assertions(req))

        lines.extend(self._gen_extractors(req))

        if think_time:
            lines.append(ast.Expr(
                ast_call(
                    func=ast_attr("sleep"),
                    args=[self._gen_expr(think_time)])))

        return lines

    def _is_dialog_replacement_needed(self, actions):
        for action in actions:
            action_config = self._parse_action(action)
            if action_config:
                if action_config[0] in ['assertdialog', 'answerdialog']:
                    return True
        return False

    def _gen_new_session_start(self):
        return self._gen_action({
            'type': self.EXTERNAL_HANDLER_START,
            'value': None,
            'param': {
                'type': 'new_session',
                'value': None,
                'param': self.capabilities,
            }
        })

    def _gen_new_session_end(self, msg=False):
        val = {
            'type': 'new_session',
            'param': self.capabilities,
        }

        if msg:
            val['message'] = self._gen_expr(ast_attr("str(traceback.format_exception(ex_type, ex, tb))"))

        return self._gen_action({
            'type': self.EXTERNAL_HANDLER_END,
            'value': None,
            'param': val,
        })

    def _gen_action_start(self, action):
        atype, tag, param, value, selectors = self._parse_action(action)
        return self._gen_action({
            'type': self.EXTERNAL_HANDLER_START,
            'value': None,
            'param': {
                'type': atype,
                'tag': tag,
                'param': param,
                'value': value,
                'selectors': selectors,
            },
        })

    def _gen_action_end(self, action):
        atype, tag, param, value, selectors = self._parse_action(action)
        return self._gen_action({
            'type': self.EXTERNAL_HANDLER_END,
            'value': None,
            'param': {
                'type': atype,
                'tag': tag,
                'param': param,
                'value': value,
                'selectors': selectors,
            },
        })

    def _gen_sel_assertion(self, assertion_config):
        self.log.debug("Generating assertion, config: %s", assertion_config)
        assertion_elements = []

        if isinstance(assertion_config, str):
            assertion_config = {"contains": [assertion_config]}

        for val in assertion_config["contains"]:
            regexp = assertion_config.get("regexp", True)
            reverse = assertion_config.get("not", False)
            subject = assertion_config.get("subject", "body")
            if subject != "body":
                raise TaurusConfigError("Only 'body' subject supported ")

            assert_message = "'%s' " % val
            if not reverse:
                assert_message += 'not '
            assert_message += 'found in BODY'

            if regexp:
                if reverse:
                    method = "self.assertEqual"
                else:
                    method = "self.assertNotEqual"
                assertion_elements.append(
                    ast.Assign(
                        targets=[ast.Name(id="re_pattern")],
                        value=ast_call(
                            func=ast_attr("re.compile"),
                            args=[ast.Str(val, kind="")])))

                assertion_elements.append(ast.Expr(
                    ast_call(
                        func=ast_attr(method),
                        args=[
                            ast.Num(0, kind=""),
                            ast_call(
                                func=ast.Name(id="len"),
                                args=[ast_call(
                                    func=ast_attr("re.findall"),
                                    args=[ast.Name(id="re_pattern"), ast.Name(id="body")])]),
                            ast.Str("Assertion: %s" % assert_message, kind="")])))

            else:
                if reverse:
                    method = "self.assertNotIn"
                else:
                    method = "self.assertIn"
                assertion_elements.append(
                    ast.Expr(
                        ast_call(
                            func=ast_attr(method),
                            args=[
                                ast.Str(val, kind=""),
                                ast.Name(id="body"),
                                ast.Str("Assertion: %s" % assert_message, kind="")])))

        return assertion_elements

    def _gen_default_vars(self):
        variables = self.scenario.get("variables")
        names = sorted(variables.keys())
        values = [variables[name] for name in names]

        return ast.Assign(
            targets=[ast_attr("self.vars")],
            value=ast.Dict(
                keys=[self._gen_expr(name) for name in names],
                values=[self._gen_expr(val) for val in values]))

    def _gen_assertions(self, request):
        stmts = []
        assertions = request.config.get("assert", [])
        for idx, assertion in enumerate(assertions):
            assertion = ensure_is_dict(assertions, idx, "contains")
            if not isinstance(assertion['contains'], list):
                assertion['contains'] = [assertion['contains']]
            subject = assertion.get("subject", Scenario.FIELD_BODY)
            if subject in (Scenario.FIELD_BODY, Scenario.FIELD_HEADERS):
                for member in assertion["contains"]:
                    func_table = {
                        (Scenario.FIELD_BODY, False, False): "assert_in_body",
                        (Scenario.FIELD_BODY, False, True): "assert_not_in_body",
                        (Scenario.FIELD_BODY, True, False): "assert_regex_in_body",
                        (Scenario.FIELD_BODY, True, True): "assert_regex_not_in_body",
                        (Scenario.FIELD_HEADERS, False, False): "assert_in_headers",
                        (Scenario.FIELD_HEADERS, False, True): "assert_not_in_headers",
                        (Scenario.FIELD_HEADERS, True, False): "assert_regex_in_headers",
                        (Scenario.FIELD_HEADERS, True, True): "assert_regex_not_in_headers",
                    }
                    method = func_table[(subject, assertion.get('regexp', True), assertion.get('not', False))]
                    stmts.append(ast.Expr(
                        ast_call(
                            func=ast_attr("response.%s" % method),
                            args=[self._gen_expr(member)])))

            elif subject == Scenario.FIELD_RESP_CODE:
                for member in assertion["contains"]:
                    method = "assert_status_code" if not assertion.get('not', False) else "assert_not_status_code"
                    stmts.append(ast.Expr(
                        ast_call(
                            func=ast_attr("response.%s" % method),
                            args=[self._gen_expr(member)])))
        return stmts

    def _gen_jsonpath_assertions(self, request):
        stmts = []
        jpath_assertions = request.config.get("assert-jsonpath", [])
        for idx, assertion in enumerate(jpath_assertions):
            assertion = ensure_is_dict(jpath_assertions, idx, "jsonpath")
            exc = TaurusConfigError('JSON Path not found in assertion: %s' % assertion)
            query = assertion.get('jsonpath', exc)
            expected = assertion.get('expected-value', None)
            method = "assert_not_jsonpath" if assertion.get('invert', False) else "assert_jsonpath"
            stmts.append(ast.Expr(
                ast_call(
                    func=ast_attr("response.%s" % method),
                    args=[self._gen_expr(query)],
                    keywords=[ast.keyword(arg="expected_value", value=self._gen_expr(expected))])))

        return stmts

    def _gen_xpath_assertions(self, request):
        stmts = []
        jpath_assertions = request.config.get("assert-xpath", [])
        for idx, assertion in enumerate(jpath_assertions):
            assertion = ensure_is_dict(jpath_assertions, idx, "xpath")
            exc = TaurusConfigError('XPath not found in assertion: %s' % assertion)
            query = assertion.get('xpath', exc)
            parser_type = 'html' if assertion.get('use-tolerant-parser', True) else 'xml'
            validate = assertion.get('validate-xml', False)
            method = "assert_not_xpath" if assertion.get('invert', False) else "assert_xpath"
            stmts.append(ast.Expr(
                ast_call(
                    func=ast_attr("response.%s" % method),
                    args=[self._gen_expr(query)],
                    keywords=[ast.keyword(arg="parser_type", value=self._gen_expr(parser_type)),
                              ast.keyword(arg="validate", value=self._gen_expr(validate))])))
        return stmts

    def _gen_extractors(self, request):
        stmts = []
        jextractors = request.config.get("extract-jsonpath")
        for varname in jextractors:
            cfg = ensure_is_dict(jextractors, varname, "jsonpath")
            stmts.append(ast.Assign(
                targets=[self.expr_compiler.gen_var_accessor(varname, ast.Store())],
                value=ast_call(
                    func=ast_attr("response.extract_jsonpath"),
                    args=[self._gen_expr(cfg['jsonpath']), self._gen_expr(cfg.get('default', 'NOT_FOUND'))])))

        extractors = request.config.get("extract-regexp")
        for varname in extractors:
            cfg = ensure_is_dict(extractors, varname, "regexp")
            stmts.append(ast.Assign(
                targets=[self.expr_compiler.gen_var_accessor(varname, ast.Store())],
                value=ast_call(
                    func=ast_attr("response.extract_regex"),
                    args=[self._gen_expr(cfg['regexp']), self._gen_expr(cfg.get('default', 'NOT_FOUND'))])))

        xpath_extractors = request.config.get("extract-xpath")
        for varname in xpath_extractors:
            cfg = ensure_is_dict(xpath_extractors, varname, "xpath")
            parser_type = 'html' if cfg.get('use-tolerant-parser', True) else 'xml'
            validate = cfg.get('validate-xml', False)
            stmts.append(ast.Assign(
                targets=[self.expr_compiler.gen_var_accessor(varname, ast.Store())],
                value=ast_call(
                    func=ast_attr("response.extract_xpath"),
                    args=[self._gen_expr(cfg['xpath'])],
                    keywords=[ast.keyword(arg="default", value=cfg.get('default', 'NOT_FOUND')),
                              ast.keyword(arg="parser_type", value=parser_type),
                              ast.keyword(arg="validate", value=validate)])))
        return stmts

    def _build_tree(self):
        mod = self._gen_module()
        mod.lineno = 0
        mod.col_offset = 0
        mod = ast.fix_missing_locations(mod)
        return mod

    def build_source_code(self):
        self.tree = self._build_tree()

    def save(self, filename):
        with open(filename, 'wt', encoding='utf8') as fds:
            fds.write("# coding=utf-8\n")
            fds.write(astunparse.unparse(self.tree))

    def _gen_logging(self):
        set_log = ast.Assign(
            targets=[ast.Name(id="log")],
            value=ast_call(
                func=ast_attr("logging.getLogger"),
                args=[ast.Str(s="apiritif.http", kind="")]))
        add_handler = ast_call(
            func=ast_attr("log.addHandler"),
            args=[ast_call(
                func=ast_attr("logging.StreamHandler"),
                args=[ast_attr("sys.stdout")])])
        set_level = ast_call(
            func=ast_attr("log.setLevel"),
            args=[ast_attr("logging.DEBUG")])

        return [set_log, gen_empty_line_stmt(), add_handler,
                gen_empty_line_stmt(), set_level, gen_empty_line_stmt()]
