import sys
import getopt
import gzip
import logging.config
import random
import signal
from time import sleep
import zlib
import jsonschema
import simplejson
import boto3
import requests
import zmq
from zmq.eventloop import ioloop, zmqstream
from tornado.ioloop import PeriodicCallback
import utils

logger = logging.getLogger(__name__)


class Subscriber(object):
    def __init__(self, zmq_context, timeout, process_callback, cleanup_callback):
        self.context = zmq_context
        self.timeout = timeout
        logger.debug("Using subscriber timeout of {0} seconds".format(timeout))
        self.io_loop = ioloop.IOLoop.instance()
        self.process_callback = process_callback
        self.cleanup_callback = cleanup_callback
        self.shutdown_callback = None
        self.cleanup_handle = None
        self.timeout_handle = None
        self.shutdown_signalled = False
        self.shutdown_detected = False
        return super(Subscriber, self).__init__()

    def start(self):
        signal.signal(signal.SIGINT, lambda signum, frame: self.io_loop.add_callback_from_signal(self.stop, signum, frame))
        signal.signal(signal.SIGTERM, lambda signum, frame: self.io_loop.add_callback_from_signal(self.stop, signum, frame))

        subscriber = self.context.socket(zmq.SUB)
        subscriber.setsockopt(zmq.SUBSCRIBE, b'')
        subscriber.connect('tcp://eddn.edcd.io:9500')
        subscriber_stream = zmqstream.ZMQStream(subscriber)

        # Ensure that we kill the IO loop and re-start (courtesy of caller's while(...)) if
        # too much time has elapsed between messages - this helps with ZeroMQ connections lapsing in the publisher if there's no
        # keep-alive signalling from the publisher to subscribers.
        self.timeout_handle = self.io_loop.call_later(self.timeout, self.io_loop.stop)
        # Need a periodic timer to prompt the IO loop to check callbacks, otherwise it seems ZeroMQ's IO ends up taking things
        # over - without this, the SIGINT callback isn't even noticed until something comes in on the subscribed port...
        # Bit strange, but given that this is relatively low-cost, probably ok for now
        self.shutdown_callback = PeriodicCallback(self.check_exit, 1500)
        self.shutdown_callback.start()

        subscriber_stream.on_recv(self.on_sub_message)
        logger.info('Connected to ZeroMQ, listening')
        self.io_loop.start()
        if self.shutdown_signalled:
            logger.info('Disconnecting for shutdown')
        else:
            logger.info('Disconnecting for reconnect')
        self.shutdown_callback.stop()
        subscriber_stream.close()
        subscriber.close()

    def check_exit(self):
        # Since we have to have a callback for checking anyway, might as well put the actual shutdown code in here
        # rather than inside the signal handler
        if self.shutdown_signalled and not self.shutdown_detected:
            self.shutdown_detected = True
            self.io_loop.add_callback(self.cleanup_callback)
            self.io_loop.add_callback(self.io_loop.stop)

    def stop(self, signum, frame):
        logger.info('Received signal {0}'.format(signum))
        self.shutdown_signalled = True

    def on_sub_message(self, msgs):
        if self.timeout_handle is not None:
            self.io_loop.remove_timeout(self.timeout_handle)
        self.timeout_handle = self.io_loop.call_later(self.timeout, self.io_loop.stop)

        if self.cleanup_handle is not None:
            self.io_loop.remove_timeout(self.cleanup_handle)
        self.cleanup_handle = self.io_loop.call_later(20, self.cleanup_callback)

        for msg in msgs:
            self.process_callback(msg)


class MessageProcessor(object):
    def __init__(self, dynamodb):
        self.dynamodb = dynamodb
        self.error_table = dynamodb.Table('eddn-bad-data')
        self.info_table = None
        self.json_storage = {}
        self.json_lookup = {'commodity/':'c', 'shipyard/':'s', 'outfitting/':'o', 'blackmarket/':'b', 'journal/':'j'}
        self.year = None
        return super(MessageProcessor, self).__init__()

    def process_message(self, msg):
        year = utils.utcnow().year
        if self.info_table is None or self.year != year:
            self.info_table = self.dynamodb.Table('eddn-archive-{0}'.format(year))
            self.year = year

        try:
            raw_json = zlib.decompress(msg).decode(encoding='UTF-8')
            try:
                msg_from_json = simplejson.loads(raw_json)
                logger.debug('Raw json {0}'.format(msg_from_json))

                if msg_from_json['$schemaRef'].endswith('/test'):
                    logger.debug('Data returned is test data, ignoring {0}'.format(msg_from_json))
                    return

                if msg_from_json['$schemaRef'].startswith('http://schemas.elite-markets.net/eddn/'):
                    schema = msg_from_json['$schemaRef'][38:]
                elif msg_from_json['$schemaRef'].startswith('https://eddn.edcd.io/schemas/'):
                    schema = msg_from_json['$schemaRef'][29:]

                type = None
                key = None
                for k,v in self.json_lookup.items():
                    if schema.startswith(k):
                        type = k
                        key = v
                        break

                if type is None:
                    logger.debug('Data returned is not commodity, shipyard, outfitting, blackmarket, journal, ignoring {0}'.format(msg_from_json))
                    return

                logger.debug('Json indicates {0} schema'.format(type))

                timestamp = utils.utcnow()

                if raw_json not in self.json_storage:
                    self.json_storage[raw_json] = (timestamp, key, msg_from_json)

                if len(self.json_storage) >= 3:
                    self.send_messages()

            except Exception as ex:
                logger.exception('Exception encountered in parsing, listening again')
                self.error_table.put_item(Item={
                    'timestamp': utils.date_to_epoch_micro(utils.utcnow()),
                    'badData': raw_json,
                    'errorCause': repr(ex),
                    'source': 'DynamoRawStore'
                    })
                sleep(0.001) # Ensure next error doesn't accidentally overwrite

        except Exception as ex:
            logger.exception('Exception encountered in communications, listening again')
            self.error_table.put_item(Item={
                'timestamp': utils.date_to_epoch_micro(utils.utcnow()),
                'errorCause': repr(ex),
                'source': 'DynamoRawDecompress'
                })
            sleep(0.001) # Ensure next error doesn't accidentally overwrite

    def send_messages(self):
        logger.debug('Sending message batch')
        self.send_messages_to_table()
        self.json_storage = {}

    def send_messages_to_table(self):
        try:
            with self.info_table.batch_writer() as batch:
                for json, meta in self.json_storage.items():
                    self.insert_into_table(batch, json, meta[1], meta[0], meta[2])

        except Exception as ex:
            logger.exception('Exception encountered in sending, sending to bad data for retry')
            with self.error_table.batch_writer() as batch:
                for entry in self.json_storage:
                    batch.put_item(Item={
                        'timestamp': utils.date_to_epoch_micro(utils.utcnow()),
                        'badData': entry,
                        'errorCause': repr(ex),
                        'source': 'DynamoRawSend'
                        })
                    sleep(0.001) # Ensure next error doesn't accidentally overwrite

    def insert_into_table(self, table, raw_json, data_type, timestamp, msg_from_json):
        new_row = dict()
        gatewayTime = utils.parse_to_utc_date(msg_from_json['header']['gatewayTimestamp'])
        new_row['gatewayInfo'] = '{0}_{1}'.format(data_type, gatewayTime.strftime('%Y-%m-%d'))
        new_row['json'] = gzip.compress(bytes(raw_json, 'utf-8'))
        new_row['gatewayEpoch'] = utils.date_to_epoch_micro(gatewayTime)
        new_row['dynamoTimestamp'] = utils.date_to_epoch_micro(timestamp)
        logger.debug('Putting valid JSON {0}'.format(new_row))
        table.put_item(Item=new_row)


def main(runPath, argv):
    try:
        opts, args = getopt.getopt(argv, 'ip', ['iam_role', 'profile_name'])
    except getopt.GetoptError as err:
        print(repr(err))
        print('EDDNDynamoRaw.py -p <profile name> OR -r')
        sys.exit(2)

    profile_name = 'eddntest'
    for opt, arg in opts:
        if opt in ("-i", "--iam_role"):
            profile_name = ''
        if opt in ("-p", "--profile_name"):
            profile_name = arg

    logging.config.dictConfig({
        'version': 1,
        'disable_existing_loggers': False,

        'formatters': {
            'standard': {
                'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
            },
        },

        'handlers': {
            'default': {
                'class': 'logging.StreamHandler',
                'level': 'INFO',
                'formatter': 'standard',
                'stream': 'ext://sys.stdout'
            },
            'file_handler': {
                'class': 'logging.handlers.RotatingFileHandler',
                'formatter': 'standard',
                'level': 'INFO',
                'filename': 'eddn-dynamo-raw.log',
                'maxBytes': 10485760,
                'backupCount': 5,
                'encoding': 'utf8'
            }
        },

        'loggers': {
            '': {
                'handlers': ['default', 'file_handler'],
                'level': 'INFO',
                'propagate': True
            }
        }
    })
    logger.debug('Logging configured')

    if profile_name:
        boto3.setup_default_session(profile_name=profile_name)

    dynamodb = boto3.resource('dynamodb', region_name='eu-west-1')
    logger.info('Connected to Dynamo')

    ioloop.install()
    logger.info('Installed PyZMQ version of Tornado IOLoop')

    context = zmq.Context()
    message_processes = MessageProcessor(dynamodb)
    # Ideally the timeout here would be coordinated with keep-alive timing from EDDN
    subscriber = Subscriber(context, random.randint(1500, 1800), message_processes.process_message, message_processes.send_messages)
    while not subscriber.shutdown_signalled:
        try:
            subscriber.start()
        except Exception as ex:
            logger.exception('Exception encountered in communications, listening again')
            bad_data_table.put_item(Item={
                'timestamp': utils.date_to_epoch_micro(utils.utcnow()),
                'errorCause': repr(ex),
                'source': 'DynamoRaw'
                })
            sleep(0.001)

if __name__ == '__main__':
    main(sys.argv[0], sys.argv[1:])
