import great_expectations as ge

from ruamel import yaml

from great_expectations.core.batch import BatchRequest

from great_expectations.rule_based_profiler.rule.rule import Rule
from great_expectations.rule_based_profiler.rule_based_profiler import RuleBasedProfiler, RuleBasedProfilerResult

from great_expectations.rule_based_profiler.domain_builder import (
    DomainBuilder,
    ColumnDomainBuilder,
)
from great_expectations.rule_based_profiler.parameter_builder import (
    MetricMultiBatchParameterBuilder,
)
from great_expectations.rule_based_profiler.expectation_configuration_builder import (
    DefaultExpectationConfigurationBuilder,
)

data_context: ge.DataContext = ge.get_context()

"""
Explanation: How to Build a RuleBasedProfiler


This Notebook will demonstrate the steps we need to take to generate a simple RuleBasedProfiler by initializing the components in memory.


We will start from a new Great Expectations Data Context (ie great_expectations folder after running great_expectations init), and begin by adding the Datasource, and progressively adding more components
End of explanation
"""


data_path: str = "../../../../test_sets/taxi_yellow_tripdata_samples"

datasource_config = {
    "name": "taxi_multi_batch_datasource",
    "class_name": "Datasource",
    "module_name": "great_expectations.datasource",
    "execution_engine": {
        "module_name": "great_expectations.execution_engine",
        "class_name": "PandasExecutionEngine",
    },
    "data_connectors": {
        "default_inferred_data_connector_name": {
            "class_name": "InferredAssetFilesystemDataConnector",
            "base_directory": data_path,
            "default_regex": {
                "group_names": ["data_asset_name", "month"],
                "pattern": "(yellow_tripdata_sample_2018)-(\\d.*)\\.csv",
            },
        },
        "default_inferred_data_connector_name_all_years": {
            "class_name": "InferredAssetFilesystemDataConnector",
            "base_directory": data_path,
            "default_regex": {
                "group_names": ["data_asset_name", "year", "month"],
                "pattern": "(yellow_tripdata_sample)_(\\d.*)-(\\d.*)\\.csv",
            },
        },
    },
}

data_context.test_yaml_config(yaml.dump(datasource_config))


# add_datasource only if it doesn't already exist in our configuration
try:
    data_context.get_datasource(datasource_config["name"])
except ValueError:
    data_context.add_datasource(**datasource_config)

"""
Explanation: Set-up: Adding taxi_data Datasource

Add taxi_data as a new Datasource
We are using an InferredAssetFilesystemDataConnector to connect to data in the test_sets/taxi_yellow_tripdata_samples folder and get one DataAsset (yellow_tripdata_sample_2018) that has 12 Batches (1 Batch/month).
End of explanation
"""


single_batch_batch_request: BatchRequest = BatchRequest(
    datasource_name="taxi_multi_batch_datasource",
    data_connector_name="default_inferred_data_connector_name",
    data_asset_name="yellow_tripdata_sample_2018",
    data_connector_query={"index": -1},
)

multi_batch_batch_request: BatchRequest = BatchRequest(
    datasource_name="taxi_multi_batch_datasource",
    data_connector_name="default_inferred_data_connector_name",
    data_asset_name="yellow_tripdata_sample_2018",
)

"""
Explanation: BatchRequests

In this example, we will be using two BatchRequests using our Datasource. 
single_batch_batch_request : which gives the most recent (December) data from the 2018 taxi_data dataset. 
multi_batch_batch_request: which gives all 12 Batches of data from the 2018 taxi_data datataset.
End of explanation
"""


domain_builder: DomainBuilder = ColumnDomainBuilder(
    include_column_name_suffixes=["_amount"],
    data_context=data_context,
)
domains: list = domain_builder.get_domains(rule_name="my_rule", batch_request=single_batch_batch_request)

# assert that the domains we get are the ones we expect
assert len(domains) == 4
assert domains == [
    {"rule_name": "my_rule", "domain_type": "column", "domain_kwargs": {"column": "fare_amount"}, "details": {"inferred_semantic_domain_type": {"fare_amount": "numeric",}},},
    {"rule_name": "my_rule", "domain_type": "column", "domain_kwargs": {"column": "tip_amount"}, "details": {"inferred_semantic_domain_type": {"tip_amount": "numeric",}},},
    {"rule_name": "my_rule", "domain_type": "column", "domain_kwargs": {"column": "tolls_amount"}, "details": {"inferred_semantic_domain_type": {"tolls_amount": "numeric",}},},
    {"rule_name": "my_rule", "domain_type": "column", "domain_kwargs": {"column": "total_amount"}, "details": {"inferred_semantic_domain_type": {"total_amount": "numeric",}},},
]

"""
Explanation: Example 1:  RuleBasedProfiler with just a DomainBuilder and ExpectationConfigurationBuilder
Build a DomainBuilder
In the process of building a RuleBasedProfiler, one of the first components we want to build/test
is a DomainBuilder, which returns the Domains (tables, columns, set of columns, etc) that the our resulting Expectations will be run on. In our example, the DomainBuilder will output a list of columns that follow a certain pattern, namely have '_amount' in their suffix. To this end we will be using a ColumnDomainBuilder which allows you to choose columns based on their suffix, name, or semantic type (like numeric or string) and our DomainBuilder will output a list of 4 columns : fare_amount, tip_amount, tolls_amount and total_amount.
The RuleBasedProfiler also contains additional DomainBuilders that allow you to do more sophisticated filtering on your data. 
These include:
 * CategoricalColumnDomainBuilder: which allows you to choose columns based on their cardinality (number of unique values).
 * MapMetricColumnDomainBuilder: which allows you to choose columns based on Map Metrics, which give a yes/no answer for individual values or rows. 
In addition, there are DomainBuilders that do not perform any additional filtering, but are required by the Expectations that are being built by the RuleBasedProfiler. 
 * TableDomainBuilder:  Outputs Table Domain, which is required by Expectations that act on tables, like (expect_table_row_count_to_equal, or expect_table_columns_to_match_set). 
ColumnDomainBuilder
End of explanation
"""


default_expectation_configuration_builder = DefaultExpectationConfigurationBuilder(
    expectation_type="expect_column_values_to_not_be_null",
    column="$domain.domain_kwargs.column", # Get the column from domain_kwargs that are retrieved from the DomainBuilder
)

simple_rule: Rule = Rule(
    name="rule_with_no_parameters",
    variables=None,
    domain_builder=domain_builder,
    expectation_configuration_builders=[default_expectation_configuration_builder],
)

"""
Explanation: To continue our example, we will continue building a RuleBasedProfiler using our ColumnDomainBuilder
Build Rule

The first Rule that we build will output expect_column_values_to_not_be_null because it does not take in  additional information other than Domain. We will add ParameterBuilders in a subsequent example.
End of explanation
"""


from great_expectations.core import ExpectationSuite
from great_expectations.rule_based_profiler.rule_based_profiler import RuleBasedProfiler

my_rbp: RuleBasedProfiler = RuleBasedProfiler(
    name="my_simple_rbp", data_context=data_context, config_version=1.0
)

my_rbp.add_rule(rule=simple_rule)


profiler_result: RuleBasedProfilerResult

profiler_result = my_rbp.run(batch_request=single_batch_batch_request)

assert len(profiler_result.expectation_configurations) == 4

profiler_result.expectation_configurations

"""
Explanation: Create RuleBasedProfiler and add Rule

We create a simple RuleBasedProfiler and add the Rule that we added in the previous step is added to the Profiler. When we run the Profiler, the output is an ExpectationSuite with 4 Expectations, which we expect.
End of explanation
"""


domain_builder: DomainBuilder = ColumnDomainBuilder(
    include_column_name_suffixes=["_amount"],
    data_context=data_context,
)
domains: list = domain_builder.get_domains(rule_name="my_rule", batch_request=single_batch_batch_request)

domains

"""
Explanation: As expected our simple RuleBasedProfiler will output 4 Expectations, one for each of our 4 columns. 
Example 2: RuleBasedProfiler with DomainBuilder, ParameterBuilder ExpectationConfigurationBuilder
Build a DomainBuilder

Using the same ColumnDomainBuilder from our previous example.
End of explanation
"""


numeric_range_parameter_builder: MetricMultiBatchParameterBuilder = (
    MetricMultiBatchParameterBuilder(
        data_context=data_context,
        metric_name="column.min",
        metric_domain_kwargs="$domain.domain_kwargs",  # domain kwarg values are accessible using fully qualified parameters
        name="my_column_min",
    )
)

"""
Explanation: Build a ParameterBuilder
ParameterBuilders help calcluate "reasonable" parameters for Expectations based on data that is specified by a BatchRequest.
The largest categories include: 
- metric_multi_batch_parameter_builder: Which is able to calculate a numeric Metric (like column.min) across multiple Batches (or just one Batch).
- value_set_multi_batch_parameter_builder: Which is able to build a value set across multiple Batches (or just one Batch). 
In some cases, there is a better way to build a value set using regex or dates. 
- regex_pattern_string_parameter_builder: Which contains a set of default regex patterns and builds a value set of the best-matching patterns. Users are also able to pass in new patterns as a parameter. 
- simple_date_format_string_parameter_builder: Which contains a set of default datetime-format patterns and builds a value set of the best-matching patterns. Users are also able to pass in new patterns as a parameter. 
Across multiple-Batches, we can build more-sophisticated parameters by using sampling methods. 
- numeric_range_multi_batch_parameter_builder: Which is able to provide range estimations across Batches using sampling methods. For instance, if we expect a table's row_count to change between Batches, we could calculate the min / max values of row_count by using the NumericMetricRangeMultiBatchParameterBuilder. These parameters could then be used by ExpectTableRowCountToBeBetween
In our example we will be using a MetricMultiBatchParameterBuilder to estimate the column.min Metric for the 4 columns defined by our Domain Builder. These are passed in as metric_domain_kwargs and are accessible using the fully qualified parameter $domain.domain_kwargs.
End of explanation
"""


config_builder: DefaultExpectationConfigurationBuilder = (
    DefaultExpectationConfigurationBuilder(
        expectation_type="expect_column_values_to_be_greater_than",
        value="$parameter.my_column_min.value[-1]", # the parameter is accessible using a fully qualified parameter
        column="$domain.domain_kwargs.column", # domain kwarg values are accessible using fully qualified parameters
        name="my_column_min",
    )
)

"""
Explanation: Build an ExpectationConfigurationBuilder
ExpectationConfigurationBuilder is being built for expect_column_values_to_be_greater_than which will use the column.min values that are calculated using the ParameterBuilder. These are now accessible using the fully qualified parameter $parameter.my_column_min.value[-1].  The [-1] indicates that we will use the min value from the latest Batch (the only Batch in this case since our BatchRequest only returns a single Batch).
End of explanation
"""


simple_rule: Rule = Rule(
    name="rule_with_parameters",
    variables=None,
    domain_builder=domain_builder,
    parameter_builders=[numeric_range_parameter_builder],
    expectation_configuration_builders=[config_builder],
)

my_rbp = RuleBasedProfiler(name="my_rbp", data_context=data_context, config_version=1.0)


"""
Explanation: Build a Rule, RuleBasedProfiler, and run
Now we build a rule with our ParameterBuilder, DomainBuilder and ExpectationConfigurationBuilder.
End of explanation
"""


my_rbp.add_rule(rule=simple_rule)

profiler_result = my_rbp.run(batch_request=single_batch_batch_request)

assert len(profiler_result.expectation_configurations) == 4

profiler_result.expectation_configurations

"""
Explanation: Add the Rule to our RuleBasedProfiler and run.
End of explanation
"""


variables: dict = {
    "multi_batch_batch_request": multi_batch_batch_request,
    "estimator_name": "bootstrap",
    "false_positive_rate": 5.0e-2, 
}

"""
Explanation: The resulting ExpectationSuite now contain values (-80.0, 0.0 etc) that were calculated from the Batch of data defined by the BatchRequest.
Example 3: RuleBasedProfiler with multiple ParameterBuilders, ExpectationConfigurationBuilders and  Variables
The third example is more complex, using multiple batches, multiple ParameterBuilders, ExpectationConfigurationBuilders and also introducing the use of variables.
The goal of this example is to build a RuleBasedProfiler that outputs an ExpectationSuite containing 2 Expectation types
- expect_column_min_to_be_between : Defined as "Expect the column minimum to be between a min and max value".
- expect_column_max_to_be_between : Defined as "Expect the column maxmimum to be between a min and max value".
for 2 columns in our taxi_data dataset
- fare_amount 
- tip_amount
with the min_value and max_value parameters for each of the Expectations estimated over 12 Batches of taxi_data, for a total of 4 Expectations.
To estimate the parameters, we will be using a NumericMetricRangeMultiBatchParameterBuilder, which is able to provide range estimations across Batches using sampling methods.  We will also be using a variables dictionary to share defined variables across Rule components like DomainBuilders, ParameterBuilders and ExpectationConfigurationBuilders.
Instantiating variables dictionary
RuleBasedProfilers allow for the definition of variables, which can be shared across Rules and Rule components. When building a complex RuleBasedProfiler with multiple Rules or components, using variables will help you keep track of values without having to input them multiple times.
Once loaded into the RuleBasedProfiler configuration, the variables are accessible using the fully qualified variable name $variables.[key_in_variables_dictionary], similar to how domain kwarg values and parameter values are accessible using a fully qualified name that begins with $.
In the example below, the estimator_name is accessible using $variables.estimator_name.
End of explanation
"""


my_rbp = RuleBasedProfiler(name="my_complex_rbp", data_context=data_context, variables=variables, config_version=1.0)

"""
Explanation: Instantiating RuleBasedProfiler with variables
Pass the variables dictionary into the RuleBasedProfiler constructor.
End of explanation
"""


from great_expectations.rule_based_profiler.domain_builder import ColumnDomainBuilder

domain_builder: DomainBuilder = ColumnDomainBuilder(
    include_column_names=["tip_amount", "fare_amount"],
    data_context=data_context,
)

"""
Explanation: Instantiating ColumnDomainBuilder
The ColumnDomainBuilder is instantiated using column names tip_amount and fare_amount. The BatchRequest is passed in as a $variable.
End of explanation
"""


from great_expectations.rule_based_profiler.parameter_builder import NumericMetricRangeMultiBatchParameterBuilder

min_range_parameter_builder: NumericMetricRangeMultiBatchParameterBuilder = NumericMetricRangeMultiBatchParameterBuilder(
    name="min_range_parameter_builder",
    metric_name="column.min",
    metric_domain_kwargs="$domain.domain_kwargs",
    false_positive_rate='$variables.false_positive_rate',
    estimator="$variables.estimator_name",
    data_context=data_context,
)

max_range_parameter_builder: NumericMetricRangeMultiBatchParameterBuilder = NumericMetricRangeMultiBatchParameterBuilder(
    name="max_range_parameter_builder",
    metric_name="column.max",
    metric_domain_kwargs="$domain.domain_kwargs",
    false_positive_rate="$variables.false_positive_rate",
    estimator="$variables.estimator_name",
    data_context=data_context,
)

"""
Explanation: Instantiating ParameterBuilders
Our Rule will contain 2 NumericMetricRangeMultiBatchParameterBuilders, one for each of our 2 Expectation types. One will be estimating the Parameter values for the column.min Metric, and the other will be estimating Parameter values for the column.max Metric.  metric_domain_kwargs are passed in from our DomainBuilder using $domain.domain_kwargs.
Also note the use of 3 Variables we defined above: 

$variables.estimator_name: This is "oneshot" in our case. 
$variables.false_positive_rate: This is 5.0e-2 or 5% in our case.
$variables.multi_batch_batch_request: This the multi_batch_batch_request, which gives all 12 Batches of data from the 2018 taxi_data datataset.
End of explanation
"""


expect_column_min: DefaultExpectationConfigurationBuilder = DefaultExpectationConfigurationBuilder(
    expectation_type="expect_column_min_to_be_between",
    column="$domain.domain_kwargs.column",
    min_value="$parameter.min_range_parameter_builder.value[0]",
    max_value="$parameter.min_range_parameter_builder.value[1]",
)

expect_column_max: DefaultExpectationConfigurationBuilder = DefaultExpectationConfigurationBuilder(
    expectation_type="expect_column_max_to_be_between",
    column="$domain.domain_kwargs.column", 
    min_value="$parameter.max_range_parameter_builder.value[0]",
    max_value="$parameter.max_range_parameter_builder.value[1]",
)

"""
Explanation: Instantiating ExpectationConfigurationBuilders
Our Rule will contain 2 ExpectationConfigurationBuilders, one for each of our 2 Expectation types: 

expect_column_min_to_be_between
expect_column_max_to_be_between

The Expectations are both ColumnExpectations, so the column parameter will be accessed from the Domain kwargs using $domain.domain_kwargs.column. 
The Expectations also take in a min_value and max_value parameter, which our NumericMetricRangeMultiBatchParameterBuilders are estimating. For expect_column_min_to_be_between, these estimated values are accessible using

$parameter.min_range_parameter_builder.value[0] for the min_value, with min_range_parameter_builder being the name of our ParameterBuilder that estimates the column.min metric.
$parameter.min_range.value[1] for the max_value.

The equivalent $parameter for expect_column_max_to_be_between would be $parameter.max_range.value[0] and $parameter.max_range_parameter_builder.value[1] respectively.
End of explanation
"""


more_complex_rule: Rule = Rule(
    name="rule_with_parameters",
    variables=None,
    domain_builder=domain_builder,
    parameter_builders=[min_range_parameter_builder, max_range_parameter_builder],
    expectation_configuration_builders=[expect_column_min, expect_column_max],
)

my_rbp.add_rule(rule=more_complex_rule)

profiler_result = my_rbp.run(batch_request=multi_batch_batch_request)

profiler_result.expectation_configurations

"""
Explanation: Instantiating RuleBasedProfiler and Running
We instantiate a Rule with our DomainBuilder, ParameterBuilders and ExpectationConfigurationBuilders and load into our RuleBasedProfiler.
End of explanation
"""


from great_expectations.rule_based_profiler.domain_builder import ColumnDomainBuilder

"""
Explanation: As expected, the resulting ExpectationSuite contains our minimum and maximum values, with tip_amount ranging from $-2.16 to $195.05 (a generous tip), and fare_amount ranging from $-98.90 (a refund) to $405,904.54 (a very very long trip).
Appendix

Here we have additional example configuration of DomainBuilder and ParameterBuilders that were not included in the previous 3 Examples. 

DomainBuilders
ColumnDomainBuilder
This DomainBuilder outputs column Domains, which are required by ColumnExpectations like (expect_column_median_to_be_between). There are a few ways that the ColumnDomainBuilder can be used. 

In the simplest usecase, the ColumnDomainBuilder can output all columns in the dataset as a Domain, or include/exclude columns if you already know which ones you would like. Column suffixes (like _amount) can be used to select columns of interest, as we saw in our examples above.

The ColumnDomainBuilder also allows you to choose columns based on their semantic types (such as numeric, or text).

Semantic types are defined as an Enum object called SemanticDomainTypes, which can be found here : https://github.com/great-expectations/great_expectations/blob/develop/great_expectations/rule_based_profiler/types/domain.py
End of explanation
"""


domain_builder: DomainBuilder = ColumnDomainBuilder(
    data_context=data_context,
)
domains: list = domain_builder.get_domains(rule_name="my_rule", batch_request=single_batch_batch_request)
assert len(domains) == 18 # all columns in yellow_tripdata_sample_2018

"""
Explanation: In the simplest usecase, the ColumnDomainBuilder can output all of the columns in yellow_tripdata_sample_2018
End of explanation
"""


domain_builder: DomainBuilder = ColumnDomainBuilder(
    include_column_names=["vendor_id"],
    data_context=data_context,
)
domains: list = domain_builder.get_domains(rule_name="my_rule", batch_request=single_batch_batch_request)
domains

domain_builder: DomainBuilder = ColumnDomainBuilder(
    exclude_column_names=["vendor_id"],
    data_context=data_context,
)
domains: list = domain_builder.get_domains(rule_name="my_rule", batch_request=single_batch_batch_request)
assert len(domains) == 17 # all columns in yellow_tripdata_sample_2018 with vendor_id excluded

domains

"""
Explanation: Columns can also be included or excluded by name
End of explanation
"""


domain_builder: DomainBuilder = ColumnDomainBuilder(
    include_semantic_types=['numeric'],
    data_context=data_context,
)

domains: list = domain_builder.get_domains(rule_name="my_rule", batch_request=single_batch_batch_request)
assert len(domains) == 15 # columns in yellow_trip_data_sample_2018 that are numeric

"""
Explanation: As described above, the ColumnDomainBuilder also allows you to choose columns based on their semantic types (such as numeric, or text). This is passed in as part of the include_semantic_types parameter.
End of explanation
"""


from great_expectations.rule_based_profiler.domain_builder import MultiColumnDomainBuilder

domain_builder: DomainBuilder = MultiColumnDomainBuilder(
    include_column_names=["vendor_id", "fare_amount", "tip_amount"],
    data_context=data_context,
)
domains: list = domain_builder.get_domains(rule_name="my_rule", batch_request=single_batch_batch_request)
assert len(domains) == 1 # 3 columns are part of a single multi-column domain. 

expected_columns: list = ["vendor_id", "fare_amount", "tip_amount"]

assert domains[0]["domain_kwargs"]["column_list"] == expected_columns

"""
Explanation: MultiColumnDomainBuilder
This DomainBuilder outputs multicolumn Domains by taking in a column list in the include_column_names parameter.
End of explanation
"""


from great_expectations.rule_based_profiler.domain_builder import ColumnPairDomainBuilder

domain_builder: DomainBuilder = ColumnPairDomainBuilder(
    include_column_names=["vendor_id", "fare_amount"],
    data_context=data_context,
)
domains: list = domain_builder.get_domains(rule_name="my_rule", batch_request=single_batch_batch_request)
assert len(domains) == 1 # 2 columns are part of a single multi-column domain. 

expect_columns_dict: dict = {'column_A': 'fare_amount', 'column_B': 'vendor_id'}

assert domains[0]["domain_kwargs"] == expect_columns_dict

"""
Explanation: ColumnPairDomainBuilder
This DomainBuilder outputs columnpair domains by taking in a column pair list in the include_column_names parameter.
End of explanation
"""


from great_expectations.rule_based_profiler.domain_builder import TableDomainBuilder

domain_builder: DomainBuilder = TableDomainBuilder(
    data_context=data_context,
)
domains: list = domain_builder.get_domains(rule_name="my_rule", batch_request=single_batch_batch_request)
domains

"""
Explanation: TableDomainBuilder
This DomainBuilder outputs table Domains, which is required by Expectations that act on tables, like (expect_table_row_count_to_equal, or expect_table_columns_to_match_set).
End of explanation
"""


from great_expectations.rule_based_profiler.domain_builder import MapMetricColumnDomainBuilder

domain_builder: DomainBuilder = MapMetricColumnDomainBuilder(
    map_metric_name="column_values.nonnull",
    data_context=data_context,
)
domains: list = domain_builder.get_domains(rule_name="my_rule", batch_request=single_batch_batch_request)
len(domains) == 17 # filtered 1 column that was all None

"""
Explanation: MapMetricColumnDomainBuilder
This DomainBuilder allows you to choose columns based on Map Metrics, which give a yes/no answer for individual values or rows. In this example, we use the Map Metrics column_values.nonnull to filter out a column that was all None from taxi_data.
End of explanation
"""


from great_expectations.rule_based_profiler.domain_builder import CategoricalColumnDomainBuilder

domain_builder: DomainBuilder = CategoricalColumnDomainBuilder(
    cardinality_limit_mode="very_few", # VERY_FEW = 10 or less
    data_context=data_context,
)

domains: list = domain_builder.get_domains(rule_name="my_rule", batch_request=single_batch_batch_request)
assert len(domains) == 7

"""
Explanation: CategoricalColumnDomainBuilder
This DomainBuilder allows you to choose columns based on their cardinality (number of unique values).The CategoricalColumnDomainBuilder will take in various cardinality_limit_mode values for cardinality, and in this example we are only interested in columns that have "very_few" (less than 10) unique values. For a full of valid modes, along with the associated values, please refer to the CardinalityLimitMode enum in:
https://github.com/great-expectations/great_expectations/blob/develop/great_expectations/rule_based_profiler/helpers/cardinality_checker.py
End of explanation
"""


from great_expectations.rule_based_profiler.types.domain import Domain
from great_expectations.execution_engine.execution_engine import MetricDomainTypes
from great_expectations.rule_based_profiler.types import ParameterContainer

domain: Domain = Domain(rule_name="my_rule", domain_type=MetricDomainTypes.COLUMN, domain_kwargs = {'column': 'total_amount'})

"""
Explanation: ParameterBuilders
ParameterBuilders work under the hood by populating a ParameterContainer, which can also be shared by multiple ParameterBuilders. It requires a Domain, and metric_name, with domain_kwargs accessible from the DomainBuilder using the fully qualified parameter $domain.domain_kwargs.
For the sake of simplicity, we will define a Domain object directly using the Domain() constructor, and pass in a column name within domain_kwargs.
End of explanation
"""


from great_expectations.rule_based_profiler.parameter_builder import MetricMultiBatchParameterBuilder

numeric_range_parameter_builder: MetricMultiBatchParameterBuilder = (
    MetricMultiBatchParameterBuilder(
        data_context=data_context,
        metric_name="column.min",
        metric_domain_kwargs=domain.domain_kwargs,
        name="my_column_min",
    )
)

parameter_container: ParameterContainer = ParameterContainer(parameter_nodes=None)

parameters = {
    domain.id: parameter_container,
}

numeric_range_parameter_builder.build_parameters(
    domain=domain,
    parameters=parameters,
    batch_request=multi_batch_batch_request,
)
# we check the parameter container
print(parameter_container.parameter_nodes)


min(parameter_container.parameter_nodes["parameter"]["parameter"]["my_column_min"]["value"])

"""
Explanation: MetricMultiBatchParameterBuilder
The MetricMultiBatchParameterBuilder computes a Metric on data from one or more batches. It takes domain_kwargs, value_kwargs, and metric_name as arguments.
End of explanation
"""


from great_expectations.rule_based_profiler.parameter_builder import ValueSetMultiBatchParameterBuilder

domain: Domain = Domain(rule_name="my_rule", domain_type=MetricDomainTypes.COLUMN, domain_kwargs = {'column': 'vendor_id'})

# instantiating a new parameter container, since it can contain the results of more than one ParmeterBuilder. 
parameter_container: ParameterContainer = ParameterContainer(parameter_nodes=None)

parameters[domain.id] = parameter_container

value_set_parameter_builder: ValueSetMultiBatchParameterBuilder = (
    ValueSetMultiBatchParameterBuilder(
        data_context=data_context,
        metric_domain_kwargs=domain.domain_kwargs,
        name="my_value_set",
    )
)

value_set_parameter_builder.build_parameters(
    domain=domain,
    parameters=parameters,
    batch_request=multi_batch_batch_request,
)

print(parameter_container.parameter_nodes)

"""
Explanation: my_column_min[value] now contains a list of 12 values, which are the minimum values the total_amount column for each of the 12 Batches associated with 2018 taxi_data data. If we were to use the values in a ExpectationConfigurationBuilder, it would be accessible through the fully-qualified parameter: $parameter.my_column_min.value.
ValueSetMultiBatchParameterBuilder
The ValueSetMultiBatchParameterBuilder is able to build a value set across multiple Batches (or just one Batch).
End of explanation
"""


from great_expectations.rule_based_profiler.parameter_builder import RegexPatternStringParameterBuilder

domain: Domain = Domain(rule_name="my_rule", domain_type=MetricDomainTypes.COLUMN, domain_kwargs = {'column': 'vendor_id'})

"""
Explanation: my_value_set[value] now contains a list of 3 values, which is a list of all unique vendor_ids across 12 Batches in the 2018 taxi_data dataset.
RegexPatternStringParameterBuilder
The RegexPatternStringParameterBuilder contains a set of default regex patterns and builds a value set of the best-matching patterns. Users are also able to pass in new patterns as a parameter.
End of explanation
"""


parameter_container: ParameterContainer = ParameterContainer(parameter_nodes=None)

parameters[domain.id] = parameter_container

regex_parameter_builder: RegexPatternStringParameterBuilder = (
    RegexPatternStringParameterBuilder(
        data_context=data_context,
        metric_domain_kwargs=domain.domain_kwargs,
        name="my_regex_set",
    )
)

regex_parameter_builder.build_parameters(
    domain=domain,
    parameters=parameters,
    batch_request=single_batch_batch_request,
)

print(parameter_container.parameter_nodes)

"""
Explanation: vendor_id is a single integer. Let's see if our default patterns can match it.
End of explanation
"""


regex_parameter_builder: RegexPatternStringParameterBuilder = (
    RegexPatternStringParameterBuilder(
        data_context=data_context,
        metric_domain_kwargs=domain.domain_kwargs,
        candidate_regexes=["^\\d{1}$"],
        name="my_regex_set",
    )
)

regex_parameter_builder.build_parameters(
    domain=domain,
    parameters=parameters,
    batch_request=single_batch_batch_request,
)

print(parameter_container.parameter_nodes)

"""
Explanation: Looks like my_regex_set[value] is an empty list. This means that none of the evaluated_regexes matched our domain. Let's try the same thing again, but this time with a regex that will match our vendor_id column. ^\\d{1}$ and ^\\d{2}$ which will match 1 or 2 digit integers anchored at the beginning and end of the string.
End of explanation
"""


from great_expectations.rule_based_profiler.parameter_builder import SimpleDateFormatStringParameterBuilder

domain: Domain = Domain(rule_name="my_rule", domain_type=MetricDomainTypes.COLUMN, domain_kwargs = {'column': 'pickup_datetime'})

parameter_container: ParameterContainer = ParameterContainer(parameter_nodes=None)

parameters[domain.id] = parameter_container

simple_date_format_string_parameter_builder: SimpleDateFormatStringParameterBuilder = (
    SimpleDateFormatStringParameterBuilder(
        data_context=data_context,
        metric_domain_kwargs=domain.domain_kwargs,
        name="my_value_set",
    )
)

simple_date_format_string_parameter_builder.build_parameters(
    domain=domain,
    parameters=parameters,
    batch_request=single_batch_batch_request,
)

print(parameter_container.parameter_nodes)

parameter_container.parameter_nodes["parameter"]["parameter"]["my_value_set"]["value"]

"""
Explanation: Now my_regex_set[value] contains ^\\d{1}$.

SimpleDateFormatStringParameterBuilder
The SimpleDateFormatStringParameterBuilder contains a set of default Datetime format patterns and builds a value set of the best-matching patterns. Users are also able to pass in new patterns as a parameter.
End of explanation
"""


from great_expectations.rule_based_profiler.parameter_builder import NumericMetricRangeMultiBatchParameterBuilder

domain: Domain = Domain(rule_name="my_rule", domain_type=MetricDomainTypes.COLUMN, domain_kwargs = {'column': 'total_amount'})

parameter_container: ParameterContainer = ParameterContainer(parameter_nodes=None)

parameters[domain.id] = parameter_container

numeric_metric_range_parameter_builder: NumericMetricRangeMultiBatchParameterBuilder = NumericMetricRangeMultiBatchParameterBuilder(
    name="column_mean_range",
    metric_name="column.mean",
    estimator="bootstrap",
    metric_domain_kwargs=domain.domain_kwargs,
    false_positive_rate=1.0e-2,
    round_decimals=0,
    data_context=data_context,
)

numeric_metric_range_parameter_builder.build_parameters(
    domain=domain,
    parameters=parameters,
    batch_request=multi_batch_batch_request,
)

print(parameter_container.parameter_nodes)

"""
Explanation: The result contains our matching datetime pattern, which is '%Y-%m-%d %H:%M:%S'
NumericMetricRangeMultiBatchParameterBuilder
The NumericMetricRangeMultiBatchParameterBuilder is able to provide range estimations across Batches using sampling methods. For instance, if we expect a table's row_count to change between Batches, we could calculate the min / max values of row_count by using the NumericMetricRangeMultiBatchParameterBuilder. These parameters could then be used by Expectations that take in ranges, like ExpectTableRowCountToBeBetween, or ExpectColumnValuesToBeBetween.
In this example, we will be taking a single Metric, column.mean and calculating it for a single column, total_amount. The parameter we will be building is the column mean-range, which are the min-max values of the total_amount column across random samples of 12 Batches of the 2018 taxi_data dataaset. 
We will also be passing in specifications for estimator, namely bootstrap sampling with a false-positive rate of less than 0.01.
End of explanation
"""


#import shutil
# clean up Expectations directory after running tests
#shutil.rmtree("great_expectations/expectations/tmp")
#os.remove("great_expectations/expectations/.ge_store_backend_id")

"""
Explanation: As we see, the mean value range for the total_amount column is 16.0 to 44.0
Optional: Clean-up Directory
As part of running this notebook, the RuleBasedProfiler will create a number of ExpectationSuite configurations in the great_expectations/expectations/tmp directory. Optionally run the following cell to clean up the directory.
End of explanation
"""


