# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="a36e9e20-001" colab_type="text"
# #1. Install Dependencies
# First install the libraries needed to execute recipes, this only needs to be done once, then click play.
#

# + id="a36e9e20-002" colab_type="code"
# !pip install git+https://github.com/google/starthinker


# + [markdown] id="a36e9e20-003" colab_type="text"
# #2. Get Cloud Project ID
# To run this recipe [requires a Google Cloud Project](https://github.com/google/starthinker/blob/master/tutorials/cloud_project.md), this only needs to be done once, then click play.
#

# + id="a36e9e20-004" colab_type="code"
CLOUD_PROJECT = 'PASTE PROJECT ID HERE'

print("Cloud Project Set To: %s" % CLOUD_PROJECT)


# + [markdown] id="a36e9e20-005" colab_type="text"
# #3. Get Client Credentials
# To read and write to various endpoints requires [downloading client credentials](https://github.com/google/starthinker/blob/master/tutorials/cloud_client_installed.md), this only needs to be done once, then click play.
#

# + id="a36e9e20-006" colab_type="code"
CLIENT_CREDENTIALS = 'PASTE CLIENT CREDENTIALS HERE'

print("Client Credentials Set To: %s" % CLIENT_CREDENTIALS)


# + [markdown] id="a36e9e20-007" colab_type="text"
# #4. Enter CM360 Log Audit Parameters
# Downloads Campaign manager logs and allows audits.
#  1. Wait for <b>BigQuery->->->CM_*</b> to be created.
#  1. Wait for <b>BigQuery->->->Barnacle_*</b> to be created, then copy and connect the following data sources.
#  1. Join the <a href='https://groups.google.com/d/forum/starthinker-assets' target='_blank'>StarThinker Assets Group</a> to access the following assets
#  1. Copy <a href='https://datastudio.google.com/open/1a6K-XdPUzCYRXZp1ZcmeOUOURc9wn2Jj' target='_blank'>Barnacle Profile Advertiser Map</a> and connect.
#  1. Copy <a href='https://datastudio.google.com/open/1NEzrQWWnPjkD90iUwN-ASKbVBzoeBdoT' target='_blank'>Barnacle Profile Campaign Map</a> and connect.
#  1. Copy <a href='https://datastudio.google.com/open/1v_GRaitwPaHHKUzfJZYNBhzotvZ-bR7Y' target='_blank'>Barnacle Profile Site Map</a> and connect.
#  1. Copy <a href='https://datastudio.google.com/open/14tWlh7yiqzxKJIppMFVOw2MoMtQV_ucE' target='_blank'>Barnacle Profiles Connections</a> and connect.
#  1. Copy <a href='https://datastudio.google.com/open/1mavjxvHSEPfJq5aW4FYgCXsBCE5rthZG' target='_blank'>Barnacle Report Delivery Profiles</a> and connect.
#  1. Copy <a href='https://datastudio.google.com/open/1Azk_Nul-auinf4NnDq8T9fDyiKkUWD7A' target='_blank'>Barnacle Roles Duplicates</a> and connect.
#  1. Copy <a href='https://datastudio.google.com/open/1ogoofpKtqkLwcW9qC_Ju_JvJdIajsjNI' target='_blank'>Barnacle Roles Not Used</a> and connect.
#  1. Copy <a href='https://datastudio.google.com/open/1xLgZPjOPDtmPyEqYMiMbWwMI8-WTslfj' target='_blank'>Barnacle Site Contacts Profiles</a> and connect.
#  1. If reports checked, copy <a href='https://datastudio.google.com/open/1-YGDiQPDnk0gD78_QOY5XdTXRlTrLeEq' target='_blank'>Barnacle Profile Report Map</a> and connect.
#  1. Copy <a href='https://datastudio.google.com/open/1gjxHm0jUlQUd0jMuxaOlmrl8gOX1kyKT' target='_blank'>Barnacle Report</a>.
#  1. When prompted choose the new data sources you just created.
#  1. Or give these intructions to the client.
# Modify the values below for your use case, can be done multiple times, then click play.
#

# + id="a36e9e20-008" colab_type="code"
FIELDS = {
  'auth_read': 'user',  # Credentials used for reading data.
  'auth_write': 'service',  # Credentials used for writing data.
  'accounts': [],  # Comma separated CM account ids.
  'days': 7,  # Number of days to backfill the log, works on first run only.
  'recipe_project': '',  # Google BigQuery project to create tables in.
  'recipe_slug': '',  # Google BigQuery dataset to create tables in.
}

print("Parameters Set To: %s" % FIELDS)


# + [markdown] id="a36e9e20-009" colab_type="text"
# #5. Execute CM360 Log Audit
# This does NOT need to be modified unless you are changing the recipe, click play.
#

# + id="a36e9e20-010" colab_type="code"
from starthinker.util.configuration import Configuration
from starthinker.util.configuration import execute
from starthinker.util.recipe import json_set_fields

USER_CREDENTIALS = '/content/user.json'

TASKS = [
  {
    'dataset': {
      'description': 'The dataset will hold log table, Create it exists.',
      'hour': [
        1
      ],
      'auth': 'user',
      'dataset': {'field': {'name': 'recipe_slug','kind': 'string','order': 4,'default': '','description': 'Name of Google BigQuery dataset to create.'}}
    }
  },
  {
    'dcm_log': {
      'description': 'Will create tables with format CM_* to hold each endpoint via a call to the API list function. Exclude reports for its own task.',
      'hour': [
        2
      ],
      'auth': 'user',
      'accounts': {
        'single_cell': True,
        'values': {'field': {'name': 'accounts','kind': 'integer_list','order': 2,'default': [],'description': 'Comma separated CM account ids.'}}
      },
      'days': {'field': {'name': 'days','kind': 'integer','order': 3,'default': 7,'description': 'Number of days to backfill the log, works on first run only.'}},
      'out': {
        'auth': 'user',
        'project': {'field': {'name': 'recipe_project','kind': 'string','order': 4,'default': '','description': 'Google BigQuery project to create tables in.'}},
        'dataset': {'field': {'name': 'recipe_slug','kind': 'string','order': 5,'default': '','description': 'Google BigQuery dataset to create tables in.'}}
      }
    }
  }
]

json_set_fields(TASKS, FIELDS)

execute(Configuration(project=CLOUD_PROJECT, client=CLIENT_CREDENTIALS, user=USER_CREDENTIALS, verbose=True), TASKS, force=True)


StarcoderdataJupyterScriptsDedupFiltered