import api
import db
from utils import parse_timestamp

import hashlib
import json

def download_logs():
    """ Download mailgun logs and store them in the database """
    logs = []
    skip = 0
    
    # Fetch all unsaved logs and add them to a LIFO queue 
    while True:
        print("fecthing logs starting at {}".format(skip))
        for log in api.get_logs(limit=100, skip=skip)['items']:
            log_data = json.dumps(log)
            log_hash = hashlib.sha256(log_data).hexdigest()
            if db.MailgunLog.objects.filter(log_hash=log_hash).exists():
                break
            else:
                logs[:0] = [(log_hash, log_data, parse_timestamp(log['date_created']]
        else: 
            break
        skip += 100

    # take items from LIFO queue and save to db
    for log_hash, data, timestamp in logs:
        db.MailgunLog(
            log_hash=log_hash,
            data=data,
            timestamp=timestamp
        ).save()

import api
import db
from utils import parse_timestamp
from django.db import transaction

from collections import OrderedDict

import hashlib
import json

@transaction.commit_manually
def download_logs():
    """ Download mailgun logs and store them in the database """
    # use ordered dict to protect against new logs arriving while downloading logs
    logs = OrderedDict()
    skip = 0
    
    # Fetch all unsaved logs and add them to a LIFO queue
    fetch_more = True
    while fetch_more:
        print("fecthing logs skip={}".format(skip))
        logs_tmp = api.get_logs(limit=1000, skip=skip)['items']
        if len(logs_tmp) == 0:
            break
        for log in logs_tmp:
            log_data = json.dumps(log)
            log_hash = hashlib.sha256(log_data).hexdigest()
            if db.MailgunLog.objects.filter(log_hash=log_hash).exists():
                fetch_more = False
                break
            else:
                logs[log_hash] = (log_hash, log_data, parse_timestamp(log['created_at']))
        skip += 1000

    # take items from LIFO queue and save to db
    print("Saving {0} logs to database".format(len(logs)))
    for i, (log_hash, data, timestamp) in enumerate(logs.values()):
        db.MailgunLog(
            log_hash=log_hash,
            data=data,
            timestamp=timestamp
        ).save()
        if (i+1) % 100 == 0:
            transaction.commit()
    transaction.commit()

Handle transactions manually when saving downloaded logs
Handle transactions manually when saving downloaded logs
