def main():

    setupLogging()
    args = parseArguments()

    # read the config file
    try:
        with open(args.file, "r") as config_file:
            config = json.load(config_file)
    except __HOLE__:
        print "Error opening file " + args.file
        return


    # default start date is None meaning 'all time'
    start_date = None
    if args.start_date != None:
        start_date = datetime.strptime(args.start_date, "%Y-%m-%d")

    # default end date is today
    end_date = datetime.today()
    if args.end_date != None:
        end_date = datetime.strptime(args.end_date, "%Y-%m-%d")
   

    # The ElasticSearch client
    es_client = Elasticsearch(args.es_host + ":" + str(args.es_port))

    data_type = config['type']
    search_body = json.dumps(config['search'])

    # If no start date find the first logstash index containing our docs
    if start_date == None:        
        start_date = findDateOfFirstIndex(es_client, data_type, search_body)
        if start_date == None:
            print "No documents found with the query " + search_body
            return

    # The REST API client
    engine_client = EngineApiClient(args.api_host, API_BASE_URL, args.api_port)
    (http_status, response) = engine_client.createJob(json.dumps(config['job_config']))
    if http_status != 201:
        print "Error creatting job"
        print http_status, json.dumps(response)
        return


    job_id = response['id']  
    print "Created job with id " + str(job_id)

    doc_count = 0
    for index_name in nextLogStashIndex(start_date, end_date):

        print "Reading from index " + index_name

        skip = 0
        try:
            # Query the documents from ElasticSearch and write to the Engine
            hits = es_client.search(index=index_name, doc_type=data_type, 
                body=search_body, from_=skip, size=MAX_DOC_TAKE)
        except elasticsearch.exceptions.NotFoundError:
            # Index not found try the next one
            continue

        # upload to the API
        content = json.dumps(elasticSearchDocsToDicts(hits['hits']['hits']))        
        (http_status, response) = engine_client.upload(job_id, content)
        if http_status != 202:
            print "Error uploading log content to the Engine"
            print http_status, json.dumps(response)
            continue

        doc_count += len(hits['hits']['hits']) 

        # get any other docs
        hitcount = int(hits['hits']['total'])
        while hitcount > (skip + MAX_DOC_TAKE):    
            skip += MAX_DOC_TAKE
            hits = es_client.search(index=index_name, doc_type=data_type, 
                body=search_body, from_=skip, size=MAX_DOC_TAKE)

            content = json.dumps(elasticSearchDocsToDicts(hits['hits']['hits']))        
            (http_status, response) = engine_client.upload(job_id, content)
            if http_status != 202:
                print json.dumps(response)
                continue

            doc_count += len(hits['hits']['hits']) 


        print "Uploaded {0} records".format(str(doc_count))
        
    (http_status, response) = engine_client.close(job_id)
    if http_status != 202:
        print "Error closing job"
        print http_status, json.dumps(response)
        return
    print "{0} records successfully written to job {1}".format(str(doc_count), job_id)
dataset/ETHPy150Open prelert/engine-python/elk_connector/elk_connector.py/main