#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Feb  6 15:01:16 2017

@author: leo

TODO:
    - Safe file name / not unique per date
    
    - API: List of finished modules for given project / source
    - API: List of loaded sources
    
    - API: Fetch logs
    - API: Move implicit load out of API
    
    - API: Error codes / remove error
    
    - Change metadata to use_public and ref_name to last used or smt. Data to
      use is specified on api call and not read from metadata (unless using last used)
    
    - Protect admin functions

    - General error handling
    
    - Allocate memory by user/ by IP?
    
    - Study impact of training set size on match rate

    - Choose btw add/select/upload and read/load/get
    
    - Catch exceptions. 
    
    - Re-Run inference if selecting more columns

    - Dealing with inference parameters when columns change...

    - integration test
    
    - Error if job failed
    
    - Avoid import in scheduled 
    - fix cancel job

    - https://blog.miguelgrinberg.com/post/restful-authentication-with-flask

DEV GUIDELINES:
    - By default the API will use the file with the same name in the last 
      module that was completed. Otherwise, you can specify the module to use file from
    - Suggestion methods shall be prefixed by infer (ex: infer_load_params, infer_mvs)
    - Suggestion methods shall can be plugged as input as params variable of transformation modules
    - Single file modules shall take as input: (pandas_dataframe, params)
    - Single file modules suggestion modules shall ouput (params, log)
    - Single file modules replacement modules shall ouput (pandas_dataframe, log)
    
    - Multiple file modules shall take as input: (pd_dataframe_1, pd_dataframe_2, params)
    - Multiple file modules suggestion modules shall ouput params, log
    - Multiple file modules merge module shall ouput ???
    
    - run_info should contain fields: has_modifications and modified_columns
    
    - Files generated by modules should be in module directory and have names determined at the project level (not API, nor module)
    
    - Do NOT return files, instead write files which users can fetch file through the API
    - If bad params are passed to modules, exceptions are raised, it is the 
        APIs role to transform these exceptions in messages
    - Functions to check parameters should be named _check_{variable_or_function} (ex: _check_file_role)
    - All securing will be done in the API part
    - Always return {"error": ..., "project_id": ..., "response": ...} ???

    - All methods to load specific configs should raise an error if the config is not coherent
    - For each module, store user input
    
    - Load all configurations to project variables
    
    - Use _init_project when project_type is a variable in path
    
    - Always include project_type as variable or hardcode
    - Put module name before project_type if it exists for all project_type
    - Put module name after project_type if it exists only for this project_type (only with linker)
    - Put in API code modules that are of use only for the API

NOTES:
    - Pay for persistant storage?

# Download metadata
curl -i http://127.0.0.1:5000/metadata/ -X POST -F "request_json=@sample_download_request.json;type=application/json"

USES: /python-memcached
"""
import json
import functools
import hashlib
import logging
import os
import tempfile
import zipfile
    
# Change current path to path of api.py
curdir = os.path.dirname(os.path.realpath(__file__))
os.chdir(curdir)

# Flask imports
import flask
from flask import Flask, jsonify, request, send_file, url_for
from flask_session import Session
from flask_cors import CORS, cross_origin
import werkzeug
from werkzeug.utils import secure_filename

# Redis imports
from rq import cancel_job as rq_cancel_job, Queue, Worker
from rq.job import Job
from worker import conn, VALID_QUEUES

from admin import Admin
from my_json_encoder import MyEncoder
from normalizer import ESNormalizer, MINI_PREFIX
from linker import ESLinker


# =============================================================================
# PARAMETERS
# =============================================================================

SECRET_KEY = open('secret_key.txt').read()
if os.path.isfile('config.json'):
    config = json.load(open('config.json'))
else:
    config = {}

HOST = config.get('HOST', '0.0.0.0')
PORT = int(config.get('PORT', 8080))
REQUIRE_PASSWORD = config.get('REQUIRE_PASSWORD', False)
PROD = config.get('PROD', False)
MAX_CONTENT_LENGTH = config.get('MAX_CONTENT_LENGTH', 10 * 1024 * 1024 * 1024)
ALLOWED_EXTENSIONS = config.get('ALLOWED_EXTENSIONS', ['csv', 'xls', 'xlsx', 'zip'])

#==============================================================================
# INITIATE APPLICATION
#==============================================================================

# Initiate application
app = Flask(__name__)
app.json_encoder = MyEncoder

cors = CORS(app)    
app.config['CORS_HEADERS'] = 'Content-Type'
#app.config['SERVER_NAME'] = '127.0.0.1:5000'
app.config['SESSION_TYPE'] = "memcached"# 'memcached'
Session(app)

app.config['SECRET_KEY'] = SECRET_KEY
app.debug = not PROD
app.config['MAX_CONTENT_LENGTH'] =  MAX_CONTENT_LENGTH
app.config['ALLOWED_EXTENSIONS'] = ALLOWED_EXTENSIONS

# Redis connection
q = dict()
for q_name in VALID_QUEUES:
    q[q_name] = Queue(q_name, connection=conn, default_timeout=7200)


#==============================================================================
# HELPER FUNCTIONS
#==============================================================================
    
def _check_privilege(privilege):
    if privilege not in ['user', 'admin']:
        raise Exception('privilege can be only user or admin')

def _check_project_type(project_type):
    if project_type not in ['normalize', 'link']:
        raise Exception('project type can be only normalize or link')

def _check_file_role(file_role):
    if file_role not in ['ref', 'source']:
        raise Exception('File type should be ref or source')

def _check_request():
    '''Check that input request is valid'''
    pass

def _parse_request():
    '''
    Separates data information from parameters and assures that values in data
    parameters are safe
    '''
    # Parse json request
    data_params = None
    module_params = None
    if request.json:
        req = request.json
        assert isinstance(req, dict)
    
        print(req)
    
        if 'data_params' in req:
            data_params = req['data_params']
            
            # Make paths secure
            for key, value in data_params.items():
                data_params[key] = secure_filename(value)
            
        if 'module_params' in req:
            module_params = req['module_params']
    
    return data_params, module_params
    
def _parse_linking_request():
    data_params = None
    module_params = None
    if request.json:
        params = request.json
        assert isinstance(params, dict)
    
        if 'data_params' in params:
            data_params = params['data_params']
            for file_role in ['ref', 'source']:
                # Make paths secure
                for key, value in data_params[file_role].items():
                    data_params[file_role][key] = secure_filename(value)
                
        if 'module_params' in params:
            module_params = params['module_params']
    
    return data_params, module_params    


def _init_project(project_type, 
                 project_id=None, 
                 create_new=False, 
                 display_name=None, 
                 description=None):
    '''
    Runs the appropriate constructor for Linker or Normalizer projects
    
    DEV NOTE: Use this in api calls that have project_type as a variable
    '''
    _check_project_type(project_type)

    project_id = secure_filename(project_id)

    if project_type == 'link':
        proj = ESLinker(project_id=project_id, 
                          create_new=create_new, 
                          display_name=display_name, 
                          description=description)
    else:
        proj = ESNormalizer(project_id=project_id, 
                              create_new=create_new, 
                              display_name=display_name, 
                              description=description)
    
    return proj

def _init_project_decorator(func):
    """Use this to initialize the appropriate project to be dealt with by API."""
    
    not_found_msg = "Could not load the {} project with Id {} . The most likely" \
                    " reason is that it does not / no longer exists. It could" \
                    " also be corrupt"
    
    @functools.wraps(func)
    def temp(project_type, project_id):
        try:
            proj = _init_project(project_type, project_id)
        except RuntimeError:
            return jsonify(error=True, message=not_found_msg.format(project_type, project_id)), 404
        return func(proj)
    return temp

        
#==============================================================================
# Error handling
#==============================================================================

#@app.errorhandler(404)
#def page_not_found(error):
#    app.logger.error('URL not valid: %s', (error))
#    return jsonify(error=True, message=error.description), 404

#@app.errorhandler(405)
#def method_not_allowed(error):
#    app.logger.error('Method not allowed (POST or GET): %s', (error))
#    return jsonify(error=True, message=error.description), 405

@app.errorhandler(500)
def server_error(error):
    app.logger.error('Server Error:', (error))
    return jsonify(error=True, message=error.__str__()), 500 
    

@app.route('/api/err/')
def err():
    raise Exception('Yolo this an error')


# =============================================================================
# Security
# =============================================================================

def _real_pwd(string):
    md5 = hashlib.md5()
    md5.update(string.encode('utf-8'))
    md5.update(app.config['SECRET_KEY'].encode('utf-8'))
    return md5.hexdigest()

def _protect_project(func):
    """Check that the password passed through POST is valid.    
    """
    if not REQUIRE_PASSWORD:
        return func
    
    @functools.wraps(func)
    def wrapper(*argv, **kwargs):
        
        # Get user token
        auth = request.authorization
        if auth is None:
            user_pwd = None
        else:
            auth.get('password')
        
        # Get real token
        project_id = locals()['kwargs']['project_id']
        real_pwd = _real_pwd(project_id)
        
        if user_pwd is None:
            return jsonify(error=True, message='No credentials were provided.'), 401

        if user_pwd != real_pwd:
            return jsonify(error=True, message='The password provided is incorrect.'), 403
    
        return func(*argv, **kwargs)
        
    return wrapper
    
#==============================================================================
# API
#==============================================================================

# TODO: get_config if module_name is specified specific module, otherwise, entire project
#@app.route('/api/<project_type>/<project_id>/<module_name>/<file_name>/')
#def get_config(project_type, project_id, module_name=None, file_name=None):
#    '''See docs in abstract_project'''
#    proj = _init_project(project_type, project_id)
#    return proj.get_config(module_name, file_name)

@app.route('/api/ping/')
def ping():
    return jsonify(error=False, message="It's alive !!")

@app.route('/api/ping_redis/')
def ping_redis():
    num_workers = len(Worker.all(conn))
    return jsonify(error=not bool(num_workers), num_workers=num_workers)

#==============================================================================
# GENERIC API METHODS (NORMALIZE AND LINK)
#==============================================================================

@app.route('/api/new/<project_type>', methods=['POST'])
@cross_origin()
def new_project(project_type):
    '''
    Create a new project:
        
    GET:
        - project_type: "link" or "normalize"
        
    POST:
        - (description): project description
        - (display_name): name to show to user
        - (public): make project freely available
        - (info): additional data concerning the project
    
    '''
    _check_project_type(project_type)
    
    description = request.json.get('description', '')
    display_name = request.json.get('display_name', '')
    public = request.json.get('public', False)
    info = request.json.get('info', None)
    
    if public and (not description):
        raise Exception('Public projects should have a description')

    if (info is not None) and (not public):
        raise Exception('Only public projects can have additional information')

    if project_type == 'normalize':
        proj = ESNormalizer(create_new=True, description=description, display_name=display_name, public=public, other_info=info)
    else:
        proj = ESLinker(create_new=True, description=description, display_name=display_name, public=public, other_info=info)
        
    return jsonify(error=False, 
                   project_id=proj.project_id)

@app.route('/api/delete/<project_type>/<project_id>', methods=['GET'])
#@_protect_project
def delete_project(project_type, project_id):
    """
    Delete an existing project (including all configuration, data and metadata)
    
    GET:
        - project_type: "link" or "normalize"
        - project_id
    """
    _check_project_type(project_type)
    
    # TODO: replace by _init_project
    if project_type == 'normalize':
        proj = ESNormalizer(project_id=project_id)
    else:
        proj = ESLinker(project_id=project_id)
    proj.delete_project()
    return jsonify(error=False)


@app.route('/api/metadata/<project_type>/<project_id>', methods=['GET'])
@_init_project_decorator
@cross_origin()
def metadata(proj):
    '''
    Fetch metadata for project ID
    
    GET:
        - project_type: "link" or "normalize"
        - project_id
    '''
    # proj = _init_project(project_type, project_id=project_id)
    resp = jsonify(error=False,
                   metadata=proj.metadata, 
                   project_id=proj.project_id)
    return resp

@app.route('/api/set_skip/<project_type>/<project_id>', methods=['POST'])
@_init_project_decorator
@cross_origin()
#@_protect_project
def set_skipped(proj):
    """
    Set skip value for selected module
    
    GET:
        - project_type: "link" or "normalize"
        - project_type
        
    POST:
        data_params:
            - module_name
            - file_name
        module_params:
            - skip_value: (true)
    """
    data_params, module_params = _parse_request()
    proj.set_log_property(data_params['module_name'], data_params['file_name'], 
                          'skipped', module_params.get('skip_value', True))
    return jsonify(error=False)

@app.route('/api/set_log_property/<project_type>/<project_id>', methods=['POST'])
@_init_project_decorator
@cross_origin()
#@_protect_project
def set_log_property(proj):
    """
    Set a log property (skipped or completed) to a given bool value.
    
    GET:
        - project_type: "link" or "normalize"
        - project_type
        
    POST:
        data_params:
            - module_name
            - file_name
        module_params:
            - property: "skipped" or "completed"
            - value: bool (defaults to True) 
    """
    data_params, module_params = _parse_request()
    proj.set_log_property(data_params['module_name'], data_params['file_name'], 
                          module_params['property'], module_params.get('value', True))
    return jsonify(error=False)    


@app.route('/api/last_written/<project_type>/<project_id>', methods=['GET', 'POST'])
@_init_project_decorator
@cross_origin()
def get_last_written(proj):
    """
    Get coordinates (module_name, file_name) of the last file written for a 
    given project.
    
    wrapper around: AbstractDataProject.get_last_written
    
    GET:
        - project_type: "link" or "normalize"
        - project_id
    POST:
        - module_name: if not null, get last file written in chosen module
        - file_name: if not null, get last file written with this given_name
        - before_module: (contains module_name) if not null, get coordinates 
                            for file written before the chosen module (with an 
                            order specified by MODULE_ORDER)
    """
    (module_name, file_name) = proj.get_last_written(request.json.get('module_name'), 
                          request.json.get('file_name'), 
                          request.json.get('before_module'))
    return jsonify(project_type=proj.metadata['project_type'], 
                   project_id=proj.metadata['project_id'], 
                   module_name=module_name, 
                   file_name=file_name)


@app.route('/api/download/<project_type>/<project_id>', methods=['GET', 'POST'])
@_init_project_decorator
@cross_origin()
#@_protect_project
def download(proj):
    '''
    Download specific file from project.
    
    GET:
        - project_type: "link" or "normalize"
        - project_type
        
    POST:
        data_params:
            - module_name: Module from which to fetch the file
            - file_name
        module_params:
            - file_type: ['csv' or 'xls']
            - zip: False (returns a zipped version)
            - thresh: 1 (threshold on __CONFIDENCE for results of linking)
    '''

    data_params, module_params = _parse_request()
    
    if data_params is None:
        data_params = {}
    if module_params is None:
        module_params = {}
    file_type = module_params.get('file_type', 'csv')
    zip_ = module_params.get('zip', False)
    
    print('data_params', data_params)
    print('module_params', module_params)
    
    if file_type != 'csv':
        raise NotImplementedError('file_type can only be csv')
        
    file_role = data_params.get('file_role')
    module_name = data_params.get('module_name')
    file_name = data_params.get('file_name')

    if file_role is not None:
        file_role = secure_filename(file_role)
    if module_name is not None:
        module_name = secure_filename(module_name)
    if file_name is not None:
        file_name = secure_filename(file_name)
    
    if module_params is None:
        file_type = 'csv'
        thresh = 1
    else:
        file_type = module_params.get('file_type', 'csv')
        thresh = module_params.get('thresh', 1)
    
    if file_type not in ['csv', 'xls', 'xlsx']:
        raise ValueError('Download file type should be csv, xls or xlsx')
    
    (module_name, file_name) = proj.get_last_written(module_name, file_name)
    columns = proj._get_header(module_name, file_name)
    
    # Remove "remove each time"
    if proj.metadata['log'][file_name]['upload_es_train'].get('was_modified', True):
        proj._remove(module_name, file_name)
        proj.ES_to_csv(module_name, file_name, 
                       columns=list(filter(lambda x: '__MODIFIED' not in x, columns)),
                       thresh=thresh)
        # TODO: fix this: very dirty
        proj.metadata['log'][file_name]['upload_es_train']['was_modified'] = False
        proj._write_metadata()

    if module_name == 'INIT':
        return jsonify(error=True,
               message='No changes were made since upload. Download is not \
                       permitted. Please do not use this service for storage')
        
    if file_type == 'csv':
        new_file_name = file_name.split('.csv')[0] + '_MMM.csv'
    else:
        new_file_name = proj.to_xls(module_name, file_name)
    
    file_path = proj.path_to(module_name, file_name)
    
    # Zip this file and send the zipped file
    if zip_:
        zip_file_name = new_file_name + '.zip'
        zip_file_path = proj.path_to(module_name, zip_file_name)
        zf = zipfile.ZipFile(zip_file_path, mode='w')
        
        compression = zipfile.ZIP_DEFLATED
        zf.write(file_path, compress_type=compression, arcname=new_file_name)
        zf.close()
        return send_file(zip_file_path, as_attachment=True, attachment_filename=zip_file_name)

    else:
        return send_file(file_path, as_attachment=True, attachment_filename=new_file_name)


# TODO: get this from MODULES ?
API_SAMPLE_NAMES = ['standard', 'sample_mvs', 'sample_types']
#API_SAMPLE_NAMES = NORMALIZE_MODULES['sample'].keys() + LINK_MODULES['sample'].keys()

@app.route('/api/sample/<project_type>/<project_id>', methods=['POST'])
@_init_project_decorator
@cross_origin()
def get_sample(proj):
    '''
    Generate a sample.
    
    GET:
        - project_type
        - project_id
        
    POST:
        - data_params:
            - module_name
            - file_name
        - module_params:
            - sampler_module_name: (ex: 'sample_mvs'). 
                            TODO: explicit
            - module_params: (optional) parameters generated by the associated inference module 
                            (Usually the result of inference. ex: result of infer_mvs)
            - sample_params: (optional) parameters to use for sampling 
                            TODO: standardize and explicit
                            {
                            'restrict_to_selected': True or False (default True),
                            'num_rows': number of rows to return (default 50) (does not apply for non standard samplers)
                            'randomize': (default True) If false, will return first values
                            }
    '''   
    data_params, all_params = _parse_request() # TODO: add size limit on params
    
    print(data_params)
    
    if all_params is None:
        all_params = dict()

    sampler_module_name = all_params.get('sampler_module_name', None)
    if sampler_module_name == 'standard':
        sampler_module_name = None
    
    module_params = all_params.get('module_params', {})
    sample_params = all_params.get('sample_params', {})
    sample_params.setdefault('restrict_to_selected', True)

    # Get sample
    proj.load_data(data_params['module_name'], 
                   data_params['file_name'], 
                   restrict_to_selected=sample_params['restrict_to_selected'])

    sample_params.setdefault('randomize', True)
    sample_params.setdefault('num_rows', 50) # TODO: figure out how to put back min(50, proj.mem_data.shape[0]))

    if (sampler_module_name is not None) and (sampler_module_name not in API_SAMPLE_NAMES):
        raise ValueError('Requested sampler_module_name {0} is not valid. Valid'\
                         + 'modules are: {1}'.format(sampler_module_name, API_SAMPLE_NAMES))
        
    sample = proj.get_sample(sampler_module_name, module_params, sample_params)
    return jsonify(sample=sample)


@app.route('/api/exists/<project_type>/<project_id>', methods=['GET'])
@cross_origin()
def project_exists(project_type, project_id):
    '''
    Check if project exists
    
    GET:
        - project_type: "link" or "normalize"
        - project_id
    '''
    try:
        _init_project(project_type=project_type, project_id=project_id)
        return jsonify(exists=True)
    except Exception as exc: 
        return jsonify(exists=False)

@app.route('/api/download_config/<project_type>/<project_id>/', methods=['POST'])
@_init_project_decorator
@cross_origin()
def read_config(proj):
    """
    Reads content of a config file.
    
    GET:
        - project_type: "link" or "normalize"
        - project_id
        
    POST:
        - data: {
                "module_name": module to fetch from
                "file_name": file to fetch
                }    
    """
    # TODO: do not expose ? 
    data_params, _ = _parse_request() # TODO: add size limit on params
    
    file_name = data_params['file_name']
    
    # Check that the file_name is allowed:
    assert (file_name in ['training.json', 
                          'infered_config.json', 
                          'config.json',
                          'column_matches.json',
                          'learned_settings.json']) \
            or '__run_info.json' in file_name
    
    result = proj.read_config_data(data_params['module_name'], file_name)
    if result:
        return jsonify(result=result)
    else:
        return jsonify(result=result, message="Config is empty"), 404


@app.route('/api/upload_config/<project_type>/<project_id>/', methods=['POST'])
@_init_project_decorator
@cross_origin()
#@_protect_project
def upload_config(proj):
    """
    Writes the content of params
    
    GET:
        - project_type: "link" or "normalize"
        - project_id
        
    POST:
        - data_params: {
                "module_name": module to fetch from
                "file_name": file to fetch
                }
        - module_params: parameters to write
    """
    # TODO: do not expose ?  
    data_params, params = _parse_request() # TODO: add size limit on params
    
    file_name = data_params['file_name']
    
    # Check that the file_name is allowed:
    assert file_name in ['training.json', 'config.json', 'learned_settings.json']
    
    proj.upload_config_data(params, data_params['module_name'], file_name)
    return jsonify(error=False)

#==============================================================================
# NORMALIZE API METHODS (see also SCHEDULER)
#==============================================================================

@app.route('/api/<project_type>/select_columns/<project_id>', methods=['POST'])
@_init_project_decorator
#@_protect_project
def add_selected_columns(proj):
    """
    Select columns to modify in normalization project. 
    
    /!\ If column selection includes new columns 
    
    GET:
        - project_type: must be "normalize"
        - project_id

    POST: 
        - columns: [list of columns]
        
    """
    assert proj.metadata['project_type'] == 'normalize'
    selected_columns = request.json['columns']
    proj.add_selected_columns(selected_columns)    
    return jsonify(error=False)

@app.route('/api/<project_type>/upload/<project_id>', methods=['POST'])
@_init_project_decorator
@cross_origin()
#@_protect_project
def upload(proj):
    '''
    Uploads files to a normalization project. (NB: cannot upload directly to 
    a link type project). 
                                               
    Also creates the mini version of the project
    
    GET:
        - project_type: must be "normalize"
        - project_id: ID of the normalization project
        
    POST:
        
      file: (csv file) A csv to upload to the chosen normalization project
                  NB: the "filename" property will be used to name the file
      json:
        - module_params:
            - make_mini: (default True) Set to False to NOT create a mini version of the file
            - sample_size
            - randomize
    '''
    assert proj.metadata['project_type'] == 'normalize'
    _, module_params = _parse_request()   
    if module_params is None:
        module_params = {}
    make_mini = module_params.get('make_mini', True) # TODO: can remove ?
    
    # Upload data        
    def custom_stream_factory(total_content_length, filename, content_type, content_length=None):
        tmpfile = tempfile.NamedTemporaryFile('wb+', prefix='flaskapp')
        app.logger.info("start receiving file ... filename => " + str(tmpfile.name))
        return tmpfile
    
    _, _, files = werkzeug.formparser.parse_form_data(flask.request.environ, stream_factory=custom_stream_factory)
    
    
    # Upload data
    file_name = files['file'].filename
    stream = files['file'].stream
    
    _, run_info = proj.upload_init_data(stream, file_name)
    
    # Make mini
    if make_mini:
        proj.load_data('INIT', run_info['file_name'])
        proj.make_mini(module_params)
        
        # Write transformations and log # TODO: not clean
        if proj.metadata['has_mini']:
            proj.write_data()
        else:
            proj._write_metadata()
    
    return jsonify(run_info=run_info, project_id=proj.project_id)


#@app.route("/upload/<filename>", methods=["POST", "PUT"])
#def upload_process(filename):
#    filename = secure_filename(filename)
#    fileFullPath = os.path.join(application.config['UPLOAD_FOLDER'], filename)
#    with open(fileFullPath, "wb") as f:
#        chunk_size = 4096
#        while True:
#            chunk = flask.request.stream.read(chunk_size)
#            if len(chunk) == 0:
#                return
#
#            f.write(chunk)
#    return jsonify({'filename': filename})


@app.route('/api/<project_type>/make_mini/<project_id>', methods=['POST'])
@_init_project_decorator
@cross_origin()
#@_protect_project
def make_mini(proj):
    '''
    Create sample version of selected file (call just after upload).
    
    GET:
        - project_type: must be "normalize"
        - project_id
    POST:
        - data_params: 
                        {
                        module_name: 'INIT' (mandatory to be init)
                        file_name: 
                        }
        - module_params: {
                            sample_size: 
                            randomize:
                        }
    '''
    assert proj.metadata['project_type'] == 'normalize'
    data_params, module_params = _parse_request()
    
    proj.load_data(data_params['module_name'], data_params['file_name'])
    proj.make_mini(module_params)
    
    # Write transformations and log
    proj.write_data()


#==============================================================================
# LINK API METHODS (see also SCHEDULER)
#==============================================================================

@app.route('/api/<project_type>/select_file/<project_id>', methods=['POST'])
@_init_project_decorator
@cross_origin()
#@_protect_project
def select_file(proj):
    '''    
    Choose a file to use as source or referential for merging
    send {file_role: "source", project_id: "ABCYOUANDME", public: False}
    
    GET:
        - project_type: must be "link"
        - project_id: ID for the "link" project
        
    POST:
        - file_role: "ref" or "source". Role of the normalized file for linking
        - project_id: ID of the "normalize" project to use for linking
    '''
    assert proj.metadata['project_type'] == 'link'
    params = request.json
    proj.add_selected_project(file_role=params['file_role'], 
                           public=params.get('public', False),
                           project_id=params['project_id'])
    return jsonify(error=False)


@app.route('/api/<project_type>/add_column_matches/<project_id>/', methods=['POST'])
@_init_project_decorator
@cross_origin()
#@_protect_project
def add_column_matches(proj):
    """
    Add pairs of columns to compare for linking.
    
    wrapper around ESLinker.add_col_matches
    
    GET: 
        - project_type: must be "link"
        - project_id: ID for the "link" project
        
    POST:
        - column_matches: [list object] column matches (see doc in original function)
    """
    assert proj.metadata['project_type'] == 'link'
    column_matches = request.json['column_matches']
    if len(column_matches) > 5:
        return jsonify(error=True, message="The API does not allow to select more than 5 column pairs to match"), 500
 
    proj.add_col_matches(column_matches)
    return jsonify(error=False)
    

@app.route('/api/<project_type>/add_column_certain_matches/<project_id>/', methods=['POST'])
@_init_project_decorator
@cross_origin()
#@_protect_project
def add_column_certain_matches(proj):
    '''
    Specify certain column matches (exact match on a subset of columns equivalent 
    to entity identity). This is used to test performances.
    
    wrapper around ESLinker.add_col_certain_matches
    
    GET:
        - project_type: must be "link"
        - project_id: ID for "link" project
        
    POST:
        - column_certain_matches: {dict object}: (see doc in original function)
    
    '''
    assert proj.metadata['project_type'] == 'link'
    column_matches = request.json['column_certain_matches']
    proj.add_col_certain_matches(column_matches)
    return jsonify(error=False)



#@app.route('/api/link/add_columns_to_return/<project_id>/<file_role>/', methods=['POST'])
#@cross_origin()
#def add_columns_to_return(project_id, file_role):
#    '''
#    Specify columns to be included in download version of file. For link project 
#    
#    # TODO: shouldn't this be for normalize also ?
#    
#    wrapper around ESLinker.add_cols_to_return
#    
#    GET:
#        project_id: ID for "link" project
#        file_role: "ref" or "source"
#    '''
#    columns_to_return = request.json
#    proj = ESLinker(project_id=project_id)
#    proj.add_cols_to_return(file_role, columns_to_return)    
#    return jsonify(error=False)


@app.route('/api/link/label_pair/<project_id>/', methods=['POST'])
@cross_origin()
#@_protect_project
def label_pair(project_id):
    '''
    Assign a label to a (source_id, ref_id) pair
    
    GET:
        project_id: ID for "link" project
    
    POST:
        source_id: ID of the source element within source
        ref_id: ID of ref element in elasticsearch
        label: label to assign to pair ('yes', 'no', 'forget')
    '''
    _, module_params = _parse_request()   
    proj = ESLinker(project_id=project_id)    
    
    source_id = module_params['source_id']
    ref_id = module_params['ref_id']
    label = module_params['label']
    
    # TODO: add label_pair
    pass


# =============================================================================
# Labeller methods
# =============================================================================

@app.route('/api/link/labeller/current/<project_id>/', methods=['GET'])
@cross_origin()
def current_state(project_id):
    '''
    Get the current state for an ES labeller.
    
    GET:
        project_id: ID for "link" project
    '''
    # Generate necessary paths and create labeller
    proj = ESLinker(project_id=project_id)
    
    labeller = proj.labeller_from_json()

    encoder = MyEncoder()
    return jsonify(error=False,
                   result=encoder.encode(labeller.to_emit()))


@app.route('/api/link/labeller/update/<project_id>/', methods=['POST'])
@cross_origin()
#@_protect_project
def update_labeller(project_id):
    '''
    Send an user input to the labeller and receive the updated labeller state    
    
    GET:
        project_id: ID for "link" project
    
    POST:
        module_params:
            user_input: #TODO: document
    '''
    _, module_params = _parse_request()
    logging.info(module_params)
    user_input = module_params['user_input']

    proj = ESLinker(project_id=project_id)
    labeller = proj.labeller_from_json()

    if labeller.answer_is_valid(user_input):
        labeller.update(user_input)
    else:
        raise ValueError('Answer received "{0}" is not valid'.format(user_input))
    
    proj.labeller_to_json(labeller)
        
    encoder = MyEncoder()
    return jsonify(error=False,
                   result=encoder.encode(labeller.to_emit()))


@app.route('/api/link/labeller/update_filters/<project_id>/', methods=['POST'])
@cross_origin()
#@_protect_project
def update_filters_labeller(project_id):
    '''
    Update filters for a labeller and receive the updated labeller state. 
    
    GET:
        project_id: ID for "link" project
    
    POST:
        module_params:
            must: #TODO: document
            must_not: #TODO: document
    '''
    _, module_params = _parse_request()
    
    logging.info('update_musts got:', module_params)
    must = module_params['must']    
    must_not = module_params['must_not'] 
    
    proj = ESLinker(project_id=project_id)
    labeller = proj.labeller_from_json()
    
    labeller.update_musts(must, must_not)
    
    proj.labeller_to_json(labeller)
    
    encoder = MyEncoder()
    return jsonify(error=False,
                   result=encoder.encode(labeller.to_emit()))

@app.route('/api/link/labeller/update_targets/<project_id>/', methods=['POST'])
@cross_origin()
#@_protect_project
def update_targets_labeller(project_id):
    '''
    Update filters for a labeller and receive the updated labeller state. 
    
    GET:
        project_id: ID for "link" project
    
    POST:
        module_params:
            target_precision: value between 0 and 1 representing the minimal 
                acceptable value for precision (results with lower values will
                be strongly penalised)
            target_recall:value between 0 and 1 representing the minimal 
                acceptable value for recall (results with lower values will
                be strongly penalised)
    '''
    _, module_params = _parse_request()
    
    logging.info('update_musts got:', module_params)
    t_p = module_params['target_precision']    
    t_r = module_params['target_recall'] 
    
    proj = ESLinker(project_id=project_id)
    labeller = proj.labeller_from_json()
    
    labeller.update_targets(t_p, t_r)
    
    proj.labeller_to_json(labeller)
    
    encoder = MyEncoder()
    return jsonify(error=False,
                   result=encoder.encode(labeller.to_emit()))  

@app.route('/api/link/labeller/complete_training/<project_id>/', methods=['GET'])
@cross_origin()
#@_protect_project
def complete_training(project_id):
    '''
    # TODO: SOON deprecated 
    
    GET:
        project_id: ID for "link" project
    '''
    proj = ESLinker(project_id)

    logging.info('Writing train')
    labeller = proj.labeller_from_json()
    
    learned_settings = labeller.export_best_params()
    
    proj.add_es_learned_settings(learned_settings)
    logging.info('Wrote train')
    
    return jsonify(error=False)
            

@app.route('/api/link/labeller/add_search/<project_id>/', methods=['POST'])
@cross_origin()
#@_protect_project
def add_search(project_id):
    '''
    # Perform search on specific user-specified terms
    
    GET:
        project_id: ID for "link" project
    
    POST:
        module_params:
            search: A dictionnary mapping the column name to query string
                Ex: [{'columns': ['col1', 'col2'], 'values_to_search': ['val1']}, ...]
            max_num_results: (optional) Max number of results for search
    '''
    _, module_params = _parse_request()
    
    proj = ESLinker(project_id)
    
    labeller = proj.labeller_from_json()
    
    # TODO: change this hack
    def temp(cols):
        if isinstance(cols, str):
            return cols
        elif isinstance(cols, list):
            return tuple(cols)
        
    print('Received custom search:', module_params['search'])
    pms = {temp(search['columns']): search['values_to_search'] for search in module_params['search']}
    labeller.add_custom_search(pms, module_params.get('max_num_results', 15))
    
    proj.labeller_to_json(labeller)
        
    encoder = MyEncoder()
    return jsonify(error=False,
                   result=encoder.encode(labeller.to_emit()))

@app.route('/api/link/labeller/clear_search/<project_id>/', methods=['GET'])
@cross_origin()
#@_protect_project
def clear_search(project_id):
    '''
    Remove user search items from the list of next labeller proposals
    '''
    _, module_params = _parse_request()
    
    proj = ESLinker(project_id)
    
    labeller = proj.labeller_from_json()
    
    labeller.clear_custom_search()
    
    proj.labeller_to_json(labeller)
        
    encoder = MyEncoder()
    return jsonify(error=False,
                   result=encoder.encode(labeller.to_emit()))



@app.route('/api/link/update_results/<project_id>/', methods=['POST'])
@cross_origin()
def update_results(project_id):
    '''Update the results in Elasticsearch index.
    
    GET:
        - project_id
        
    POST:
        - module_params:
            - labels: [{'source_id': ABC,  'ref_id': 'XYZ', 'is_match': bool}]
            
    '''
    _, module_params = _parse_request()
    
    proj = ESLinker(project_id)
    
    proj.update_results(module_params['labels'])
    
    return jsonify(error=False)

# =============================================================================
# ES FETCH
# =============================================================================

@app.route('/api/es_fetch_by_id/<project_type>/<project_id>', methods=['GET', 'POST'])
@_init_project_decorator
@cross_origin()
#@_protect_project
def es_fetch_by_id(proj):
    ''' /!\ DEPRECATED USE es_fetch /!\
    
    Fetch result from the existing ES index project_id
    https://www.elastic.co/guide/en/elasticsearch/guide/current/pagination.html
    
    GET:
        - project_type
        - project_id
    POST:
        - data_params: None
        - module_params:
            - (size): size of sample
            - (from): where to start
    '''
    
    _, module_params = _parse_request()
    
    if module_params is None:
        module_params = {}
    size = module_params.get('size', 10)
    from_ = module_params.get('from', 0)
    
    
    res = proj.fetch_by_id(size, from_)   
    return jsonify(res)
    

@app.route('/api/es_fetch/<project_type>/<project_id>', methods=['GET', 'POST'])
@_init_project_decorator
@cross_origin()
#@_protect_project
def es_fetch(proj):
    '''
    Fetch result from the existing ES index project_id
    https://www.elastic.co/guide/en/elasticsearch/guide/current/pagination.html
    
    By default it will paginate based on the document ID order
    
    GET:
        - project_type
        - project_id
    POST:
        - data_params: None
        - module_params:
            - (size): size of sample
            - (from): where to start
            - (field): field to sort on (defaults to "_id", the original file order)
            - (order): "desc" or "asc"
    
    '''
    
    _, module_params = _parse_request()
    
    if module_params is None:
        module_params = {}
    size = module_params.get('size', 10)
    from_ = module_params.get('from', 0)
    field = module_params.get('field', '_id')
    order = module_params.get('order', 'asc')

    # For now only _id and __CONFIDENCE are sortable
    assert field in ['_id', '__CONFIDENCE']
    
    if field == '_id':
        res = proj.fetch_by_id(size, from_, order)
    else:
        res = proj.fetch_by_sort(field, order, size, from_)
    return jsonify(res)


#==============================================================================
# SCHEDULER
#==============================================================================

SCHEDULED_JOBS = {
                    'infer_mvs': {'project_type': 'normalize'}, 
                    'replace_mvs': {'project_type': 'normalize'}, 
                    'infer_types': {'project_type': 'normalize'}, 
                    'recode_types': {'project_type': 'normalize'}, 
                    'concat_with_init': {'project_type': 'normalize'}, 
                    'run_all_transforms': {'project_type': 'normalize'}, 
                    'create_es_index': {'project_type': 'link', 'timeout': 7200},
                    'create_es_labeller': {'project_type': 'link', 
                                        'priority': 'high'}, 
                    'es_linker': {'project_type': 'link', 'timeout': 7200},
                    'infer_restriction': {'project_type': 'link', 
                                          'priority': 'high'}, 
                    'perform_restriction': {'project_type': 'link'},
                    'link_results_analyzer': {'project_type': 'link'}
                    }

def _choose_queue(job_name, project_id, data_params):
    '''
    Priority is low by default. It is high if specified in SCHEDULED_MODULES
    or if performing on a __MINI or file that doesn't have __MINI
    # TODO: MAKE impossible to overwrite metadata
    '''
    project_type = None
    if data_params is not None:
        project_type = data_params.get('project_type')
        
    if project_type is None:
        project_type = SCHEDULED_JOBS[job_name]['project_type']
    proj = _init_project(project_type=project_type, project_id=project_id)  

    if data_params and data_params is not None:
        if (project_type=='normalize') and (
                    (MINI_PREFIX in data_params['file_name']) 
                    or (not proj.metadata['has_mini'])):
            return 'high'
    
    return SCHEDULED_JOBS[job_name].get('priority', 'low')
 
def _choose_timeout(job_name):
    DEFAULT_TIMEOUT = 900
    return SCHEDULED_JOBS[job_name].get('timeout', DEFAULT_TIMEOUT)

@app.route('/api/schedule/<job_name>/<project_id>/', methods=['GET', 'POST'])
@cross_origin()
#@_protect_project
def schedule_job(job_name, project_id):    
    '''
    Schedule module runs
    
    GET:
        - job_name: name of module to run (full list in API_MODULE_NAMES)
        - project_id
    POST:
    
        - data_params: the data to transform (see specific module docs)
        - module_params: how to transform the data (see spectific module docs)
    
    ex: '/api/schedule/infer_mvs/<project_id>/'
    '''
    assert job_name in SCHEDULED_JOBS
    data_params, module_params = _parse_request()

    q_priority = _choose_queue(job_name, project_id, data_params)
    assert q_priority in VALID_QUEUES
    
    job_id = project_id + '_' + job_name
    
    # Delete job if already queued
    if job_id in q[q_priority].job_ids:
        logging.warning('Deleting job: {0} for re-scheduling'.format(job_id))
        q[q_priority].remove(job_id)
    
    #TODO: remove and de-comment under
    job = q[q_priority].enqueue_call(
            func='api_queued_modules._' + job_name,
            args=(project_id, data_params, module_params), 
            ttl=24*3600, # Job can be queued for 24 hours max
            result_ttl=2*3600, # Result can be kept for 2 hours max
            job_id=job_id,
            timeout=_choose_timeout(job_name)
            #depends_on=project_id
    )        
    
    # 
    job_id = job.get_id()
    logging.info('Scheduled job: {0}'.format(job_id))
    return jsonify(job_id=job_id,
                   job_result_api_url=url_for('get_job_result', job_id=job_id))    
    

@app.route('/queue/result/<job_id>', methods=['GET'])
@cross_origin()
def get_job_result(job_id):
    '''
    Fetch the json output of a module run scheduled by schedule_job. Will return 
    a 202 code if job is not yet complete and 404 if job could not be found.
    
    GET:
        - job_id: as returned by schedule_job
    '''    
    try:
        job = Job.fetch(job_id, connection=conn)
    except:
        return jsonify(error=True, message='job_id could not be found', completed=False), 404
        
    if job.status == 'failed':
        return jsonify(error=True, message='Job failed', completed=False), 500
    
    if job.is_finished:
        #return str(job.result), 200
        return jsonify(completed=True, result=job.result)
    else:
        if job.status == 'failed':
            return jsonify(completed=False, error=True, message=job.exc_info), 500
        
        # TODO: Check for success specifically
        return jsonify(completed=False), 202

@app.route('/queue/cancel/<job_id>', methods=['GET'])
@cross_origin()
def cancel_job(job_id):
    '''
    Remove job from queue
    
    # TODO: make this work
    
    GET:
        - job_id: as returned by schedule_job
    '''
    
    try:
        job = Job.fetch(job_id, connection=conn)
        job.cancel()
        return jsonify(job_canceled=True, completed=False)
    except:
        return jsonify(job_canceled=False,
                       error=True, 
                       message='job_id could not be found', 
                       completed=False), 404
    
@app.route('/queue/num_jobs/<job_id>', methods=['GET'])
@cross_origin()
def count_jobs_in_queue_before(job_id):
    '''
    Returns the number of jobs preceding job_id or -1 if job is no longer in queue.
    
    GET:
        - job_id: as returned by schedule_job
    '''
    job_ids = q.job_ids
    if job_id in job_ids:
        return jsonify(num_jobs=job_ids.index(job_id))
    else:
        return jsonify(num_jobs=-1)
    # TODO: check if better to return error
    
@app.route('/queue/num_jobs/', methods=['GET'])
@cross_origin()
def count_jobs_in_queue():
    '''Returns the number of jobs enqueued'''
    # TODO: change for position in queue
    num_jobs = len(q.job_ids)
    return jsonify(num_jobs=num_jobs)

#==============================================================================
    # Admin
#==============================================================================

@app.route('/api/public_project_ids/<project_type>', methods=['GET'])
@cross_origin()
def list_public_project_ids(project_type):
    '''
    
    '''
    admin = Admin()
    list_of_project_ids = admin.list_project_ids(project_type, project_access='public')
    return jsonify(list_of_project_ids)

@app.route('/api/public_projects/<project_type>', methods=['GET'])
@cross_origin()
def list_public_projects(project_type):
    '''
    '''
    admin = Admin()
    list_of_projects = admin.list_projects(project_type, project_access='public')
    return jsonify(list_of_projects)

@app.route('/api/admin/list_projects/<project_type>', methods=['POST'])
@_protect_project
@cross_origin()
def list_projects_by_time(project_type):
    '''List projects with various filters.
    
    GET:
        - project_type
        
    POST:
        - module_params:
            project_access: str (either "all", "public", or "private")
                Whether to list all projects or only those that are public or 
                non-public (private)
            action: str (either "created" or "last_used")
                When using temporal filters, whether to use as variable the 
                creation date or the date of last use.
            when: str (either "before" or "after")
                Whether to select the elements before or after the cutoff date
                (both are inclusive, but proba of equality is very slim).
            hours_from_now: int
                How many hours before current time are we looking at.
    '''
    _, module_params = _parse_request()
    
    admin = Admin()
    res = admin.list_projects_by_time(project_type,
                             project_access=module_params['project_access'],
                             action=module_params['action'], 
                             when=module_params['when'], 
                             hours_from_now=module_params['hours_from_now'])
    return jsonify(*res)
     
@app.route('/api/admin/remove_projects/<project_type>', methods=['POST'])
@_protect_project
@cross_origin()   
def remove_projects(project_type):
    '''Remove multiple projects based on various filters.
    
    GET:
        - project_type
        
    POST:
        - module_params:
            project_access: str (either "all", "public", or "private")
                Whether to list all projects or only those that are public or 
                non-public (private)
            action: str (either "created" or "last_used")
                When using temporal filters, whether to use as variable the 
                creation date or the date of last use.
            when: str (either "before" or "after")
                Whether to select the elements before or after the cutoff date
                (both are inclusive, but proba of equality is very slim).
            hours_from_now: int
                How many hours before current time are we looking at.
    '''
    _, module_params = _parse_request()
    
    admin = Admin()
    res = admin.remove_project_by_time(project_type,
                             project_access=module_params['project_access'],
                             action=module_params['action'], 
                             when=module_params['when'], 
                             hours_from_now=module_params['hours_from_now'])
    return jsonify(*res)
    
@app.route('/api/admin/delete_unused_indices', methods=['GET'])
@_protect_project
@cross_origin()   
def delete_unused_indices():
    '''Delete Elasticsearch indices which are no longer linked to a 
    normalization or link project.
    
    GET:
        - project_type
    '''
    admin = Admin()
    res = admin.delete_unused_indices()
    return jsonify(error=False, indices_deleted=res)

@app.route('/api/admin/delete_loose_links', methods=['GET'])
@_protect_project
@cross_origin()           
def delete_loose_links():
    '''Delete link projects for which one of the normalization projects no 
    longer exists.   
    '''
    admin = Admin()
    res = admin.delete_loose_links()
    return jsonify(error=False, projects_deleted=res)


if __name__ == '__main__':
    
    # =============================================================================
    # RUN APPLICATION        
    # =============================================================================
    app.run(host=HOST, port=PORT, debug=not PROD)
