import pandas as pd
import great_expectations as ge
import great_expectations.jupyter_ux
import json

from datetime import datetime
from sklearn.model_selection import train_test_split

df = ge.read_csv("data/titanic.csv")
train, test = train_test_split(df, test_size=0.3)

"""
Explanation: Why Data Quality

Data driven oraganizations/products
Wrong desicion / Bad product
Its all about trust

Data Quality Effect

Is the data correct ?
If so to what time ?




great to find violations as early as possible


productivity and integrity are the same thing data stuff





Trying to maintain data systems that are untested, undocumented and unstable is nearly impossible
Data Quality characteristics

Accuracy: for whatever data described, it needs to be accurate.
Relevancy: the data should meet the requirements for the intended use.
Completeness: the data should not have missing values or miss data records.
Timeliness: the data should be up to date.
Consistency:the data should have the data format as expected and can be cross reference-able with the same results.

Data Quality tools

Great Expectations
Deequ
Tensorflow Data Validation

Great Expectations Native
End of explanation
"""


train.expect_column_values_to_be_between("Age", 0,80)
train.expect_column_values_to_be_in_set('Survived', [1, 0])
train.expect_column_mean_to_be_between("Age", 20,40)
train.expect_column_values_to_match_regex('Name', '[A-Z][a-z]+(?: \([A-Z][a-z]+\))?, ', mostly=.95)
train.expect_column_values_to_be_in_set("Sex", ["male", "female"])

results = train.validate() 
if train.validate()["SUCCESS"]:
    ...
results

my_expectations = train.get_expectation_suite()
test.validate(expectation_suite=my_expectations)

"""
Explanation: expectations are assertions about data
End of explanation
"""


# ! great_expectations init

context = ge.data_context.DataContext()
context.list_expectation_suite_names()

expectation_suite_name = " "
batch_kwargs = {'path': "https://github.com/plotly/datasets/raw/master/titanic.csv",
                'datasource': "titanic"}
batch = context.get_batch(batch_kwargs, my_expectations)
batch.head()

run_id = {"run_name": "First validation", "run_time": datetime.now()}
results = context.run_validation_operator("action_list_operator", 
                                          assets_to_validate=[batch], 
                                          run_id=run_id)
results

"""
Explanation: Great Expectations Validation in Your pipeline
End of explanation
"""


context.build_data_docs()
context.open_data_docs()

"""
Explanation: Test as data documentations
your docs are your tests and your  tests are your docs
End of explanation
"""


