text: # encoding: utf-8
  import os
  import re
  import scrapy
  import requests
  import lxml.html
  import sqlite3
  import shutil
  import base64
  
  base_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "")
  EMOJI_STYLES = ['apple', 'twemoji', 'noto', 'blobmoji']
  ICONS_PATH = lambda s: base_dir + 'images/%s/emoji' % s
  DB_PATH = base_dir + 'emoji.sqlite'
  
  def rm_r(path):
      if os.path.isdir(path) and not os.path.islink(path):
          shutil.rmtree(path)
      elif os.path.exists(path):
          os.remove(path)
  
  def cleanup():
      for style in EMOJI_STYLES: rm_r(ICONS_PATH(style))
      rm_r(DB_PATH)
      for style in EMOJI_STYLES: os.makedirs(ICONS_PATH(style))
  
  def setup_db():
      conn = sqlite3.connect(DB_PATH, check_same_thread=False)
      conn.executescript('''
          CREATE TABLE emoji (
              name VARCHAR PRIMARY KEY, 
              code VARCHAR,
              icon_apple VARCHAR, 
              icon_twemoji VARCHAR,
              icon_noto VARCHAR, 
              icon_blobmoji VARCHAR,
              keywords VARCHAR, 
              name_search VARCHAR
          );
          CREATE TABLE skin_tone (
              name VARCHAR, 
              code VARCHAR, 
              tone VARCHAR,
              icon_apple VARCHAR, 
              icon_twemoji VARCHAR,
              icon_noto VARCHAR, 
              icon_blobmoji VARCHAR
          );
          CREATE TABLE shortcode (
              name VARCHAR,
              code VARCHAR
          );
          CREATE INDEX name_idx ON skin_tone (name);
          ''')
      conn.row_factory = sqlite3.Row
  
      return conn
  
  
  def str_to_unicode_emoji(s):
      """
      Converts 'U+FE0E' to u'\U0000FE0E'
      """
      return re.sub(r'U\+([0-9a-fA-F]+)', lambda m: chr(int(m.group(1), 16)), s).replace(' ', '')
  
  
  def codepoint_to_url(codepoint, style):
      """
      Given an emoji's codepoint (e.g. 'U+FE0E') and a non-apple emoji style, 
      returns a url to to the png image of the emoji in that style. 
  
      Only works for style = 'twemoji', 'noto', and 'blobmoji'.
      """
      base = codepoint.replace('U+', '').lower()
      if style == 'twemoji':
          # See discussion in commit 8115b76 for more information about
          # why the base needs to be patched like this.
          patched = re.sub(r'0*([1-9a-f][0-9a-f]*)', lambda m: m.group(1), 
                  base.replace(' ', '-').replace('fe0f-20e3', '20e3').replace('1f441-fe0f-200d-1f5e8-fe0f', '1f441-200d-1f5e8'))
          response = requests.get('https://github.com/twitter/twemoji/raw/gh-pages/v/latest')
          version = response.text if response.ok else None
          if version:
              return 'https://github.com/twitter/twemoji/raw/gh-pages/v/%s/72x72/%s.png' \
                      % (version, patched)
          else:
              return 'https://github.com/twitter/twemoji/raw/master/assets/72x72/%s.png' \
                      % patched
      elif style == 'noto':
          return 'https://github.com/googlefonts/noto-emoji/raw/master/png/128/emoji_u%s.png' \
                  % base.replace(' ', '_')
      elif style == 'blobmoji':
          return 'https://github.com/C1710/blobmoji/raw/master/png/128/emoji_u%s.png' \
                  % base.replace(' ', '_')
  
  def name_to_shortcodes(shortname):
      """
      Given an emoji's CLDR Shortname (e.g. 'grinning face with smiling eyes'), returns a list
      of common shortcodes used for that emoji.
      
      NOTE: These shortcodes will NOT have colons at the beginning and end, even if they normally
            would.
      """
      url = 'https://emojipedia.org/%s/' % re.sub(r'[^a-z0-9 ]', '', shortname.lower()).replace(' ', '-')
      response = requests.get(url, stream=True)
      response.raw.decode_content = True
      html = lxml.html.parse(response.raw) if response.ok else None
      shortcode_nodes = html.xpath('//ul[@class="shortcodes"]/li/span[@class="shortcode"]') if html else []
      return [s.text for s in shortcode_nodes]
  
  cleanup()
  conn = setup_db()
  
  class EmojiSpider(scrapy.Spider):
      name = 'emojispider'
      start_urls = ['http://unicode.org/emoji/charts/emoji-list.html',
                    'http://unicode.org/emoji/charts/full-emoji-modifiers.html']
      # start_urls = ['http://172.17.0.1:8000/list2.html']
  
      def parse(self, response):
          icon = 0
          emoji_nodes = response.xpath('//tr[.//td[@class="code"]]')
          for i in range(0, len(emoji_nodes)):
              # Scrape Data from unicode.org
              tr = emoji_nodes[i]
              code = tr.css('.code a::text').extract_first()
              encoded_code = str_to_unicode_emoji(code)
              name = ''.join(tr.xpath('(.//td[@class="name"])[1]//text()').extract())
              keywords = ''.join(tr.xpath('(.//td[@class="name"])[2]//text()').extract())
              keywords = ' '.join([kw.strip() for kw in keywords.split('|') if 'skin tone' not in kw])
              name = name.replace(u'⊛', '').strip()
              icon_name = name.replace(':', '').replace(' ', '_')
              skin_tone = ''
              found = re.search(r'(?P<skin_tone>[-\w]+) skin tone', name, re.I)
              if found:
                  skin_tone = found.group('skin_tone')
                  name = name.replace(': %s skin tone' % skin_tone, '')
              shortcodes = name_to_shortcodes(name)
  
              # Prepare emoji data to be inserted into DB
              print("Fetching %i/%i: %s %s" % (i+1, len(emoji_nodes), encoded_code, name))
              record = {
                  'name': name,
                  'code': encoded_code,
                  'shortcodes': ' '.join(shortcodes),
                  'keywords': keywords,
                  'tone': skin_tone,
                  'name_search': ' '.join(set(
                      [s[1:-1] for s in shortcodes] + [kw.strip() for kw in ('%s %s' % (name, keywords)).split(' ')]
                  )),
                  # Merge icon styles into record
                  **{ 'icon_%s' % style: '%s/%s.png' \
                          % (ICONS_PATH(style), icon_name) \
                          for style in EMOJI_STYLES \
                  }
              }
  
              # Download Icons for each emoji
              print("🖼  Downloading Icons...")
              supported_styles = []
              for style in EMOJI_STYLES:
                  if style == 'apple':
                      icon_data = tr.css('.andr img').xpath('@src').extract_first().split('base64,')[1]
                      icon_data = base64.decodestring(icon_data.encode('utf-8'))
                  else:
                      link = codepoint_to_url(code, style)
                      resp = requests.get(link)
                      icon_data = resp.content if resp.ok else None
                      print('- %s: %s' % ('✅' if resp.ok else '❎', style))
                  
                  if icon_data:
                      with open(record['icon_%s' % style], 'wb') as f:
                          f.write(icon_data)
                      supported_styles += [style] 
  
              # Prepare emoji insertion query
              supported_styles = ['icon_%s' % style for style in supported_styles]
              if skin_tone:
                  query = '''INSERT INTO skin_tone (name, code, tone, ''' + ', '.join(supported_styles) + ''')
                             VALUES (:name, :code, :tone, ''' + ', '.join([':%s' % s for s in supported_styles]) + ''')'''
              else:
                  query = '''INSERT INTO emoji (name, code, ''' + ', '.join(supported_styles) + ''', 
                                                keywords, name_search)
                             VALUES (:name, :code, ''' + ', '.join([':%s' % s for s in supported_styles]) + ''',
                                     :keywords, :name_search)'''
              
              # Insert emoji & associated shortcodes into DB
              conn.execute(query, record)
              for sc in shortcodes:
                  squery = '''INSERT INTO shortcode (name, code)
                              VALUES (:name, "%s")''' % sc
                  conn.execute(squery, record)
  
          conn.commit()
  