import asyncio
from io import BytesIO
import logging
import os
import random
import time
from typing import List
from urllib.parse import urlparse

import aiohttp
from aiohttp import ClientSession, TCPConnector
import requests
from requests import Response
from tqdm import tqdm

from nogi import REQUEST_HEADERS
from nogi.db.nogi_blog_content import NogiBlogContent
from nogi.db.nogi_blog_summary import NogiBlogSummary
from nogi.storages.gcs import GCS
from nogi.utils.parsers import PostParser, generate_post_key

logger = logging.getLogger()

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36'
}


class PostExecutor:

    def __init__(self, member: dict, summary_db: NogiBlogSummary, content_db: NogiBlogContent, gcs_client: GCS, bucket: str, concurrent: int = 4):
        self._waiting_limit = concurrent
        self.member = member

        # DB
        self.summary_db = summary_db
        self.content_db = content_db

        # GCS Storage
        self.bucket = bucket
        self.storage = gcs_client
        self.storage_blog_post_prefix = os.path.join(member['roma_name'], 'post')
        self.storage_blog_image_prefix = os.path.join(member['roma_name'], 'img')

        # Tasks
        self.todos = self.summary_db.get_missing_blog_url(member['id'])

    @staticmethod
    def db_transform(post_url: str, obj: dict, **kwargs) -> dict:
        return dict(
            member_id=kwargs.get('member_id'),
            blog_key=generate_post_key(post_url),
            url=post_url,
            title=obj['title'],
            content=obj['content'],
            image_gcs_paths=kwargs.get('image_gcs_paths'),
            post_gcs_path=kwargs.get('post_gcs_path'),
            blog_created_at=int(obj['created_at'].timestamp()))

    @staticmethod
    def _get_hd_image(url: str) -> BytesIO:
        first_layer_response: Response = requests.get(url, headers=HEADERS)
        logger.debug(first_layer_response.cookies)
        resp = requests.get(
            url=url.replace('http://dcimg.awalker.jp/v/', 'http://dcimg.awalker.jp/i/'),
            cookies=first_layer_response.cookies)
        logger.debug(resp.status_code)
        logger.debug(resp.headers)
        return BytesIO(resp.content) if resp.status_code == 200 else BytesIO(bytes=b'')

    def backup_images(self, image_urls: List[dict]) -> List[str]:
        downloaded_image_urls = list()
        for url in image_urls:
            image_gcs_path = os.path.join(self.storage_blog_image_prefix,
                                          '/'.join(urlparse(url['image_url']).path.split('/')[-5:]))
            if url['high_resolution_url'] != url['image_url']:
                hd_image = self._get_hd_image(url['high_resolution_url'])
                if hd_image:
                    self.storage.upload_stream(
                        bucket=self.bucket,
                        blob_name=image_gcs_path,
                        content=hd_image.read(),
                        content_type='image/jpeg'
                    )
            else:
                image = requests.get(url=url['image_url'])
                if image.status_code != 200:
                    logger.warning('Image Request Fail: %s', url)
                    continue
                self.storage.upload_stream(
                    bucket=self.bucket,
                    blob_name=image_gcs_path,
                    content=image.content,
                    content_type='image/jpeg'
                )
            downloaded_image_urls.append(url)
        return downloaded_image_urls

    async def backup_content(self, session: ClientSession, post_url: str) -> str:
        post_gcs_path = os.path.join(self.storage_blog_post_prefix, '/'.join(urlparse(post_url).path.split('/')[-3:]))
        try:
            async with session.get(url=post_url, headers=REQUEST_HEADERS) as response:
                self.storage.upload_stream(
                    bucket=self.bucket, blob_name=post_gcs_path,
                    content=await response.read(), content_type='text/html')
            return post_gcs_path
        except aiohttp.client_exceptions.InvalidURL:
            print('Invalid URL: %s' % post_url)
        except aiohttp.client_exceptions.ClientConnectorError:
            print('Client Connector Error: %s' % post_url)

    @staticmethod
    def crawl_post(url: str) -> None:
        return PostParser(requests.get(url, headers=REQUEST_HEADERS).text).to_dict()

    async def _run(self, url: str):
        try:
            async with aiohttp.ClientSession(connector=TCPConnector(verify_ssl=False)) as session:
                post_gcs_path = await self.backup_content(session, url)
                post = self.crawl_post(url)
                images_gcs_paths = self.backup_images(post['image_urls'])
                result = self.db_transform(
                    post_url=url, obj=post, member_id=self.member['id'], image_gcs_paths=images_gcs_paths, post_gcs_path=post_gcs_path)
                self.content_db.upsert_crawled_post(result)
                self.summary_db.update_crawled_result(result)
        except aiohttp.client_exceptions.InvalidURL:
            print('Invalid URL: %s' % url)
        except aiohttp.client_exceptions.ClientConnectorError:
            print('Client Connector Error: %s' % url)
        except Exception:
            import traceback
            print('Error URL: %s' % url)
            print(traceback.format_exc())

    def run(self):
        loop = asyncio.get_event_loop()
        if self.todos:
            tasks = []
            for url in tqdm(self.todos, desc='Current Member: {}'.format(self.member['kanji_name']), ncols=120):
                tasks.append(asyncio.ensure_future(self._run(url)))
                if len(tasks) > self._waiting_limit:
                    loop.run_until_complete(asyncio.gather(*tasks))
                    tasks = []
            if tasks:
                loop.run_until_complete(asyncio.gather(*tasks))

            slepp_second = random.randint(1, 15)
            print('Sleep for %s second' % slepp_second)
            time.sleep(slepp_second)

[((22, 9, 22, 28), 'logging.getLogger', 'logging.getLogger', ({}, {}), '()', False, 'import logging\n'), ((42, 40, 42, 81), 'os.path.join', 'os.path.join', ({(42, 53, 42, 72): "member['roma_name']", (42, 74, 42, 80): '"""post"""'}, {}), "(member['roma_name'], 'post')", False, 'import os\n'), ((43, 41, 43, 81), 'os.path.join', 'os.path.join', ({(43, 54, 43, 73): "member['roma_name']", (43, 75, 43, 80): '"""img"""'}, {}), "(member['roma_name'], 'img')", False, 'import os\n'), ((62, 41, 62, 75), 'requests.get', 'requests.get', (), '', False, 'import requests\n'), ((136, 15, 136, 39), 'asyncio.get_event_loop', 'asyncio.get_event_loop', ({}, {}), '()', False, 'import asyncio\n'), ((69, 15, 69, 36), 'io.BytesIO', 'BytesIO', ({(69, 23, 69, 35): 'resp.content'}, {}), '(resp.content)', False, 'from io import BytesIO\n'), ((69, 69, 69, 87), 'io.BytesIO', 'BytesIO', (), '', False, 'from io import BytesIO\n'), ((147, 27, 147, 48), 'random.randint', 'random.randint', ({(147, 42, 147, 43): '1', (147, 45, 147, 47): '15'}, {}), '(1, 15)', False, 'import random\n'), ((149, 12, 149, 36), 'time.sleep', 'time.sleep', ({(149, 23, 149, 35): 'slepp_second'}, {}), '(slepp_second)', False, 'import time\n'), ((52, 21, 52, 48), 'nogi.utils.parsers.generate_post_key', 'generate_post_key', ({(52, 39, 52, 47): 'post_url'}, {}), '(post_url)', False, 'from nogi.utils.parsers import PostParser, generate_post_key\n'), ((86, 24, 86, 58), 'requests.get', 'requests.get', (), '', False, 'import requests\n'), ((133, 18, 133, 40), 'traceback.format_exc', 'traceback.format_exc', ({}, {}), '()', False, 'import traceback\n'), ((145, 40, 145, 62), 'asyncio.gather', 'asyncio.gather', ({(145, 55, 145, 61): '*tasks'}, {}), '(*tasks)', False, 'import asyncio\n'), ((114, 26, 114, 68), 'requests.get', 'requests.get', (), '', False, 'import requests\n'), ((118, 55, 118, 85), 'aiohttp.TCPConnector', 'TCPConnector', (), '', False, 'from aiohttp import ClientSession, TCPConnector\n'), ((142, 44, 142, 66), 'asyncio.gather', 'asyncio.gather', ({(142, 59, 142, 65): '*tasks'}, {}), '(*tasks)', False, 'import asyncio\n'), ((100, 77, 100, 95), 'urllib.parse.urlparse', 'urlparse', ({(100, 86, 100, 94): 'post_url'}, {}), '(post_url)', False, 'from urllib.parse import urlparse\n'), ((75, 51, 75, 77), 'urllib.parse.urlparse', 'urlparse', ({(75, 60, 75, 76): "url['image_url']"}, {}), "(url['image_url'])", False, 'from urllib.parse import urlparse\n')]