from load_config import params_to_cli

    llr, emb, pred,evaluation = params_to_cli("CONFIGS/ex1-ml-1m-config.yml", "CONFIGS/ex4-du04d100w10l80n10d30p1q1-1000-081417-params.yml")

llr

evaluation

"""
Explanation: The following code will take the CLI commands produced in 01-JJA-L2V-Configuration-Files notebook


You need to install aws cli  
http://docs.aws.amazon.com/cli/latest/userguide/installing.html


You need to run aws config 


Let's import the functions defined before for loading parameters
End of explanation
"""


def create_steps(llr=None, emb=None, pred=None, evaluation=None, name=''):
    if llr != None:
        Steps=[

        {
            'Name': name + '-LLR',
            'ActionOnFailure': 'CONTINUE',
            'HadoopJarStep': {
                'Jar': 'command-runner.jar',
                'Args': (llr).split(),
            }
        },
        {
            'Name': name + '-EMB',
            'ActionOnFailure': 'CONTINUE',
            'HadoopJarStep': {
                'Jar': 'command-runner.jar',
                'Args': (emb).split(),
            }
        },
        {
            'Name': name + '-PRED',
            'ActionOnFailure': 'CONTINUE',
            'HadoopJarStep': {
                'Jar': 'command-runner.jar',
                'Args': (pred).split(),
            }
        },
            {
            'Name': name + '-EVAL',
            'ActionOnFailure': 'CONTINUE',
            'HadoopJarStep': {
                'Jar': 'command-runner.jar',
                'Args': (evaluation).split(),
            }
        }
    ]
    else:
        Steps=[
        {
            'Name': name + '-EMB',
            'ActionOnFailure': 'CONTINUE',
            'HadoopJarStep': {
                'Jar': 'command-runner.jar',
                'Args': (emb).split(),
            }
        },
        {
            'Name': name + '-PRED',
            'ActionOnFailure': 'CONTINUE',
            'HadoopJarStep': {
                'Jar': 'command-runner.jar',
                'Args': (pred).split(),
            }
        },
            {
            'Name': name + '-EVAL',
            'ActionOnFailure': 'CONTINUE',
            'HadoopJarStep': {
                'Jar': 'command-runner.jar',
                'Args': (evaluation).split(),
            }
        }
    ]
                     
            
    return Steps
    

"""
Explanation: This function will format the AWS CLI commands so we can pass them to the cluster using boto3
End of explanation
"""


# ex2 = create_steps(llr=llr, emb=emb, pred=pred, evaluation=evaluation, name='EXP3')
ex3 = create_steps(llr=llr, emb=emb, pred=pred, evaluation=evaluation, name='EXP3')
# ex4 = create_steps(emb=emb348, pred=pred348, name='EXP4')
# ex5 = create_steps(emb=emb349, pred=pred349, name='EXP5')

"""
Explanation: To load the commands into EMR
Here we create steps based on the three steps in the pipeline
End of explanation
"""


# steps = ex2 + ex3 + ex4 + ex5
steps = ex3

steps

"""
Explanation: If we are adding multiple runs of the pipeline
End of explanation
"""


import boto3

client = boto3.client('emr')

cluster_id = 'j-2JGJ9RIFQ4VRK'

response = client.add_job_flow_steps(
    JobFlowId = cluster_id,
    Steps= steps
)

response

"""
Explanation: To run the steps into EMR using boto3
End of explanation
"""


