text: # Copyright (C) 2019 The Raphielscape Company LLC.
  #
  # Licensed under the Raphielscape Public License, Version 1.c (the "License");
  # you may not use this file except in compliance with the License.
  #
  """ Userbot module containing various scrapers. """
  
  import os
  import time
  import asyncio
  import shutil
  from bs4 import BeautifulSoup
  import re
  import asyncurban
  from html import unescape
  from re import findall
  from urllib.parse import quote_plus
  from wikipedia import summary
  from wikipedia.exceptions import DisambiguationError, PageError
  from requests import get
  from search_engine_parser import GoogleSearch
  from googleapiclient.discovery import build
  from googleapiclient.errors import HttpError
  from googletrans import LANGUAGES, Translator
  from gtts import gTTS
  from gtts.lang import tts_langs
  from emoji import get_emoji_regexp
  from youtube_dl import YoutubeDL
  from youtube_dl.utils import (DownloadError, ContentTooShortError,
                                ExtractorError, GeoRestrictedError,
                                MaxDownloadsReached, PostProcessingError,
                                UnavailableVideoError, XAttrMetadataError)
  from asyncio import sleep
  from userbot import (CMD_HELP, BOTLOG, BOTLOG_CHATID, YOUTUBE_API_KEY,
                       TEMP_DOWNLOAD_DIRECTORY)
  from userbot.events import register
  from telethon.tl.types import DocumentAttributeAudio
  from userbot.utils import progress, chrome, googleimagesdownload
  
  CARBONLANG = "auto"
  TTS_LANG = "en"
  TRT_LANG = "en"
  TEMP_DOWNLOAD_DIRECTORY = "/root/userbot/.bin"
  
  
  @register(outgoing=True, pattern=r"^\.crblang (.*)")
  async def setlang(prog):
      global CARBONLANG
      CARBONLANG = prog.pattern_match.group(1)
      await prog.edit(f"Language for carbon.now.sh set to {CARBONLANG}")
  
  
  @register(outgoing=True, pattern=r"^\.carbon")
  async def carbon_api(e):
      """ A Wrapper for carbon.now.sh """
      await e.edit("`Processing...`")
      CARBON = 'https://carbon.now.sh/?l={lang}&code={code}'
      global CARBONLANG
      textx = await e.get_reply_message()
      pcode = e.text
      if pcode[8:]:
          pcode = str(pcode[8:])
      elif textx:
          pcode = str(textx.message)  # Importing message to module
      code = quote_plus(pcode)  # Converting to urlencoded
      await e.edit("`Processing...\n25%`")
      file_path = TEMP_DOWNLOAD_DIRECTORY + "carbon.png"
      if os.path.isfile(file_path):
          os.remove(file_path)
      url = CARBON.format(code=code, lang=CARBONLANG)
      driver = await chrome()
      driver.get(url)
      await e.edit("`Processing..\n50%`")
      download_path = './'
      driver.command_executor._commands["send_command"] = (
          "POST", '/session/$sessionId/chromium/send_command')
      params = {
          'cmd': 'Page.setDownloadBehavior',
          'params': {
              'behavior': 'allow',
              'downloadPath': download_path
          }
      }
      driver.execute("send_command", params)
      driver.find_element_by_xpath("//button[@id='export-menu']").click()
      driver.find_element_by_xpath("//button[contains(text(),'4x')]").click()
      driver.find_element_by_xpath("//button[contains(text(),'PNG')]").click()
      await e.edit("`Processing...\n75%`")
      # Waiting for downloading
      while not os.path.isfile(file_path):
          await sleep(0.5)
      await e.edit("`Processing...\n100%`")
      await e.edit("`Uploading...`")
      await e.client.send_file(
          e.chat_id,
          file_path,
          caption=("Made using [Carbon](https://carbon.now.sh/about/),"
                   "\na project by [Dawn Labs](https://dawnlabs.io/)"),
          force_document=True,
          reply_to=e.message.reply_to_msg_id,
      )
  
      os.remove(file_path)
      driver.quit()
      # Removing carbon.png after uploading
      await e.delete()  # Deleting msg
  
  
  @register(outgoing=True, pattern=r"^\.img (.*)")
  async def img_sampler(event):
      """ For .img command, search and return images matching the query. """
      await event.edit("`Processing...`")
      counter = int(event.pattern_match.group(1).split(' ', 1)[0])
      query = str(event.pattern_match.group(1).split(' ', 1)[1])
      if counter > 10:
          counter = 10
      if counter < 1:
          counter = 1
      lim = findall(r"lim=\d+", query)
      try:
          lim = lim[0]
          lim = lim.replace("lim=r", "")
          query = query.replace("lim=r" + lim[0], "")
      except IndexError:
          lim = counter
      response = googleimagesdownload()
  
      # creating list of arguments
      arguments = {
          "keywords": query,
          "limit": lim,
          "format": "jpg",
          "no_directory": "no_directory"
      }
  
      # passing the arguments to the function
      paths = response.download(arguments)
      lst = paths[0][query]
      await event.client.send_file(
          await event.client.get_input_entity(event.chat_id), lst)
      shutil.rmtree(os.path.dirname(os.path.abspath(lst[0])))
      await event.delete()
  
  
  @register(outgoing=True, pattern=r"^\.currency (.*)")
  async def moni(event):
      input_str = event.pattern_match.group(1)
      input_sgra = input_str.split(" ")
      if len(input_sgra) == 3:
          try:
              number = float(input_sgra[0])
              currency_from = input_sgra[1].upper()
              currency_to = input_sgra[2].upper()
              request_url = "https://api.exchangeratesapi.io/latest?base={}".format(
                  currency_from)
              current_response = get(request_url).json()
              if currency_to in current_response["rates"]:
                  current_rate = float(current_response["rates"][currency_to])
                  rebmun = round(number * current_rate, 2)
                  await event.edit("{} {} = {} {}".format(
                      number, currency_from, rebmun, currency_to))
              else:
                  await event.edit(
                      "`This seems to be some alien currency, which I can't convert right now.`"
                  )
          except Exception as e:
              await event.edit(str(e))
      else:
          return await event.edit("`Invalid syntax.`")
  
  
  @register(outgoing=True, pattern=r"^\.google (.*)")
  async def gsearch(q_event):
      """ For .google command, do a Google search. """
      counter = int(q_event.pattern_match.group(1).split(' ', 1)[0])
      match = str(q_event.pattern_match.group(1).split(' ', 1)[1])
      if counter > 10:
          counter = 10
      if counter < 0:
          counter = 1
      page = findall(r"page=\d+", match)
      try:
          page = page[0]
          page = page.replace("page=r", "")
          match = match.replace("page=r" + page[0], "")
      except IndexError:
          page = 1
      search_args = (str(match), int(page))
      gsearch = GoogleSearch()
      gresults = await gsearch.async_search(*search_args)
      msg = ""
      for i in range(counter):
          try:
              title = gresults["titles"][i]
              link = gresults["links"][i]
              desc = gresults["descriptions"][i]
              msg += f"[{title}]({link})\n`{desc}`\n\n"
          except IndexError:
              break
      await q_event.edit("**Search Query:**\n`" + match + "`\n\n**Results:**\n" +
                         msg,
                         link_preview=False)
  
      if BOTLOG:
          await q_event.client.send_message(
              BOTLOG_CHATID,
              "Google Search query `" + match + "` was executed successfully",
          )
  
  
  @register(outgoing=True, pattern=r"^\.wiki (.*)")
  async def wiki(wiki_q):
      """ For .wiki command, fetch content from Wikipedia. """
      match = wiki_q.pattern_match.group(1)
      try:
          summary(match)
      except DisambiguationError as error:
          return await wiki_q.edit(f"Disambiguated page found.\n\n{error}")
      except PageError as pageerror:
          return await wiki_q.edit(f"Page not found.\n\n{pageerror}")
      result = summary(match)
      if len(result) >= 4096:
          file = open("output.txt", "w+")
          file.write(result)
          file.close()
          await wiki_q.client.send_file(
              wiki_q.chat_id,
              "output.txt",
              reply_to=wiki_q.id,
              caption=r"`Output too large, sending as file`",
          )
          if os.path.exists("output.txt"):
              return os.remove("output.txt")
      await wiki_q.edit("**Search:**\n`" + match + "`\n\n**Result:**\n" + result)
      if BOTLOG:
          await wiki_q.client.send_message(
              BOTLOG_CHATID, f"Wiki query `{match}` was executed successfully")
  
  
  @register(outgoing=True, pattern=r"^\.ud (.*)")
  async def urban_dict(event):
      """Output the definition of a word from Urban Dictionary"""
      ud = asyncurban.UrbanDictionary()
      await event.edit("Processing...")
      query = event.pattern_match.group(1)
  
      if not query:
          return await event.edit("`Error: Provide a word!`")
      template = "`Query: `{}\n\n`Definition: `{}\n\n`Example:\n`{}"
  
      try:
          definition = await ud.get_word(query)
      except asyncurban.WordNotFoundError:
          return await event.edit("`Error: No definition available.`")
  
      result = template.format(
          definition.word, definition.definition, definition.example)
  
      if len(result) >= 4096:
          await event.edit("`Output too large, sending as file...`")
          with open("output.txt", "w+") as file:
              file.write("Query: " + definition.word + "\n\nMeaning: " +
                         definition.definition + "Example: \n" + definition.example)
          await event.client.send_file(
              event.chat_id,
              "output.txt",
              caption=f"Urban Dictionary's definition of {query}")
          if os.path.exists("output.txt"):
              os.remove("output.txt")
          return await event.delete()
      else:
          return await event.edit(result)
  
  
  @register(outgoing=True, pattern=r"^\.tts(?: |$)([\s\S]*)")
  async def text_to_speech(query):
      """ For .tts command, a wrapper for Google Text-to-Speech. """
      textx = await query.get_reply_message()
      message = query.pattern_match.group(1)
      if message:
          pass
      elif textx:
          message = textx.text
      else:
          return await query.edit(
              "`Give a text or reply to a message for Text-to-Speech!`")
  
      try:
          gTTS(message, lang=TTS_LANG)
      except AssertionError:
          return await query.edit(
              'The text is empty.\n'
              'Nothing left to speak after pre-precessing, tokenizing and cleaning.'
          )
      except ValueError:
          return await query.edit('Language is not supported.')
      except RuntimeError:
          return await query.edit('Error loading the languages dictionary.')
      tts = gTTS(message, lang=TTS_LANG)
      tts.save("k.mp3")
      with open("k.mp3", "rb") as audio:
          linelist = list(audio)
          linecount = len(linelist)
      if linecount == 1:
          tts = gTTS(message, lang=TTS_LANG)
          tts.save("k.mp3")
      with open("k.mp3", "r"):
          await query.client.send_file(query.chat_id, "k.mp3", voice_note=True)
          os.remove("k.mp3")
          if BOTLOG:
              await query.client.send_message(
                  BOTLOG_CHATID, "Text to Speech executed successfully !")
          await query.delete()
  
  
  # kanged from Blank-x ;---;
  @register(outgoing=True, pattern=r"^\.imdb (.*)")
  async def imdb(e):
      try:
          movie_name = e.pattern_match.group(1)
          remove_space = movie_name.split(' ')
          final_name = '+'.join(remove_space)
          page = get("https://www.imdb.com/find?ref_=nv_sr_fn&q=r" + final_name +
                     "&s=all")
          soup = BeautifulSoup(page.content, 'lxml')
          odds = soup.findAll("tr", "odd")
          mov_title = odds[0].findNext('td').findNext('td').text
          mov_link = "http://www.imdb.com/" + \
              odds[0].findNext('td').findNext('td').a['href']
          page1 = get(mov_link)
          soup = BeautifulSoup(page1.content, 'lxml')
          if soup.find('div', 'poster'):
              poster = soup.find('div', 'poster').img['src']
          else:
              poster = ''
          if soup.find('div', 'title_wrapper'):
              pg = soup.find('div', 'title_wrapper').findNext('div').text
              mov_details = re.sub(r'\s+', ' ', pg)
          else:
              mov_details = ''
          credits = soup.findAll('div', 'credit_summary_item')
          director = credits[0].a.text
          if len(credits) == 1:
              writer = 'Not available'
              stars = 'Not available'
          elif len(credits) > 2:
              writer = credits[1].a.text
              actors = []
              for x in credits[2].findAll('a'):
                  actors.append(x.text)
              actors.pop()
              stars = actors[0] + ',' + actors[1] + ',' + actors[2]
          else:
              writer = 'Not available'
              actors = []
              for x in credits[1].findAll('a'):
                  actors.append(x.text)
              actors.pop()
              stars = actors[0] + ',' + actors[1] + ',' + actors[2]
          if soup.find('div', "inline canwrap"):
              story_line = soup.find('div',
                                     "inline canwrap").findAll('p')[0].text
          else:
              story_line = 'Not available'
          info = soup.findAll('div', "txt-block")
          if info:
              mov_country = []
              mov_language = []
              for node in info:
                  a = node.findAll('a')
                  for i in a:
                      if "country_of_origin" in i['href']:
                          mov_country.append(i.text)
                      elif "primary_language" in i['href']:
                          mov_language.append(i.text)
          if soup.findAll('div', "ratingValue"):
              for r in soup.findAll('div', "ratingValue"):
                  mov_rating = r.strong['title']
          else:
              mov_rating = 'Not available'
          await e.edit('<a href=' + poster + '>&#8203;</a>'
                       '<b>Title : </b><code>' + mov_title + '</code>\n<code>' +
                       mov_details + '</code>\n<b>Rating : </b><code>' +
                       mov_rating + '</code>\n<b>Country : </b><code>' +
                       mov_country[0] + '</code>\n<b>Language : </b><code>' +
                       mov_language[0] + '</code>\n<b>Director : </b><code>' +
                       director + '</code>\n<b>Writer : </b><code>' + writer +
                       '</code>\n<b>Stars : </b><code>' + stars +
                       '</code>\n<b>IMDB Url : </b>' + mov_link +
                       '\n<b>Story Line : </b>' + story_line,
                       link_preview=True,
                       parse_mode='HTML')
      except IndexError:
          await e.edit("Plox enter **Valid movie name** kthx")
  
  
  @register(outgoing=True, pattern=r"^\.trt(?: |$)([\s\S]*)")
  async def translateme(trans):
      """ For .trt command, translate the given text using Google Translate. """
      translator = Translator()
      textx = await trans.get_reply_message()
      message = trans.pattern_match.group(1)
      if message:
          pass
      elif textx:
          message = textx.text
      else:
          return await trans.edit("`Give a text or reply to a message to translate!`")
  
      try:
          reply_text = translator.translate(deEmojify(message), dest=TRT_LANG)
      except ValueError:
          return await trans.edit("Invalid destination language.")
  
      source_lan = LANGUAGES[f'{reply_text.src.lower()}']
      transl_lan = LANGUAGES[f'{reply_text.dest.lower()}']
      reply_text = f"From **{source_lan.title()}**\nTo **{transl_lan.title()}:**\n\n{reply_text.text}"
  
      await trans.edit(reply_text)
      if BOTLOG:
          await trans.client.send_message(
              BOTLOG_CHATID,
              f"Translated some {source_lan.title()} stuff to {transl_lan.title()} just now.",
          )
  
  
  @register(pattern=r"\.lang (trt|tts) (.*)", outgoing=True)
  async def lang(value):
      """ For .lang command, change the default langauge of userbot scrapers. """
      util = value.pattern_match.group(1).lower()
      if util == "trt":
          scraper = "Translator"
          global TRT_LANG
          arg = value.pattern_match.group(2).lower()
          if arg in LANGUAGES:
              TRT_LANG = arg
              LANG = LANGUAGES[arg]
          else:
              return await value.edit(
                  f"`Invalid Language code !!`\n`Available language codes for TRT`:\n\n`{LANGUAGES}`"
              )
      elif util == "tts":
          scraper = "Text to Speech"
          global TTS_LANG
          arg = value.pattern_match.group(2).lower()
          if arg in tts_langs():
              TTS_LANG = arg
              LANG = tts_langs()[arg]
          else:
              return await value.edit(
                  f"`Invalid Language code !!`\n`Available language codes for TTS`:\n\n`{tts_langs()}`"
              )
      await value.edit(f"`Language for {scraper} changed to {LANG.title()}.`")
      if BOTLOG:
          await value.client.send_message(
              BOTLOG_CHATID,
              f"`Language for {scraper} changed to {LANG.title()}.`")
  
  
  @register(outgoing=True, pattern=r"^\.yt (.*)")
  async def yt_search(video_q):
      """ For .yt command, do a YouTube search from Telegram. """
      query = video_q.pattern_match.group(1)
      result = ''
  
      if not YOUTUBE_API_KEY:
          return await video_q.edit(
              "`Error: YouTube API key missing! Add it to environment vars or config.env.`"
          )
  
      await video_q.edit("```Processing...```")
  
      full_response = await youtube_search(query)
      videos_json = full_response[1]
  
      for video in videos_json:
          title = f"{unescape(video['snippet']['title'])}"
          link = f"https://youtu.be/{video['id']['videoId']}"
          result += f"{title}\n{link}\n\n"
  
      reply_text = f"**Search Query:**\n`{query}`\n\n**Results:**\n\n{result}"
  
      await video_q.edit(reply_text)
  
  
  async def youtube_search(query,
                           order=r"relevance",
                           token=None,
                           location=None,
                           location_radius=None):
      """ Do a YouTube search. """
      youtube = build('youtube',
                      'v3',
                      developerKey=YOUTUBE_API_KEY,
                      cache_discovery=False)
      search_response = youtube.search().list(
          q=query,
          type=r"video",
          pageToken=token,
          order=order,
          part=r"id,snippet",
          maxResults=10,
          location=location,
          locationRadius=location_radius).execute()
  
      videos = []
  
      for search_result in search_response.get("items", []):
          if search_result["id"]["kind"] == "youtube#video":
              videos.append(search_result)
      try:
          nexttok = search_response["nextPageToken"]
          return (nexttok, videos)
      except HttpError:
          nexttok = "last_page"
          return (nexttok, videos)
      except KeyError:
          nexttok = "KeyError, try again."
          return (nexttok, videos)
  
  
  @register(outgoing=True, pattern=r"^\.r(a|v) (.*)")
  async def download_video(v_url):
      """ For media downloader command, download media from YouTube and many other sites. """
      url = v_url.pattern_match.group(2)
      type = v_url.pattern_match.group(1).lower()
  
      await v_url.edit("`Preparing to download...`")
  
      if type == "a":
          opts = {
              'format':
              'bestaudio',
              'addmetadata':
              True,
              'key':
              'FFmpegMetadata',
              'writethumbnail':
              True,
              'prefer_ffmpeg':
              True,
              'geo_bypass':
              True,
              'nocheckcertificate':
              True,
              'postprocessors': [{
                  'key': 'FFmpegExtractAudio',
                  'preferredcodec': 'mp3',
                  'preferredquality': '320',
              }],
              'outtmpl':
              '%(id)s.mp3',
              'quiet':
              True,
              'logtostderr':
              False
          }
          video = False
          song = True
  
      elif type == "v":
          opts = {
              'format':
              'best',
              'addmetadata':
              True,
              'key':
              'FFmpegMetadata',
              'prefer_ffmpeg':
              True,
              'geo_bypass':
              True,
              'nocheckcertificate':
              True,
              'postprocessors': [{
                  'key': 'FFmpegVideoConvertor',
                  'preferedformat': 'mp4'
              }],
              'outtmpl':
              '%(id)s.mp4',
              'logtostderr':
              False,
              'quiet':
              True
          }
          song = False
          video = True
  
      try:
          await v_url.edit("`Fetching data, please wait..`")
          with YoutubeDL(opts) as rip:
              rip_data = rip.extract_info(url)
      except DownloadError as DE:
          return await v_url.edit(f"`{str(DE)}`")
      except ContentTooShortError:
          return await v_url.edit("`The download content was too short.`")
      except GeoRestrictedError:
          return await v_url.edit(
              "`Video is not available from your geographic location "
              "due to geographic restrictions imposed by a website.`"
          )
      except MaxDownloadsReached:
          return await v_url.edit("`Max-downloads limit has been reached.`")
      except PostProcessingError:
          return await v_url.edit("`There was an error during post processing.`")
      except UnavailableVideoError:
          return await v_url.edit("`Media is not available in the requested format.`")
      except XAttrMetadataError as XAME:
          return await v_url.edit(f"`{XAME.code}: {XAME.msg}\n{XAME.reason}`")
      except ExtractorError:
          return await v_url.edit("`There was an error during info extraction.`")
      except Exception as e:
          return await v_url.edit(f"{str(type(e)): {str(e)}}")
      c_time = time.time()
      if song:
          await v_url.edit(
              f"`Preparing to upload song:`\n**{rip_data['title']}**")
          await v_url.client.send_file(
              v_url.chat_id,
              f"{rip_data['id']}.mp3",
              supports_streaming=True,
              attributes=[
                  DocumentAttributeAudio(duration=int(rip_data['duration']),
                                         title=str(rip_data['title']),
                                         performer=str(rip_data['uploader']))
              ],
              progress_callback=lambda d, t: asyncio.get_event_loop(
              ).create_task(
                  progress(d, t, v_url, c_time, "Uploading..",
                           f"{rip_data['title']}.mp3")))
          os.remove(f"{rip_data['id']}.mp3")
          await v_url.delete()
      elif video:
          await v_url.edit(
              f"`Preparing to upload video:`\n**{rip_data['title']}**")
          await v_url.client.send_file(
              v_url.chat_id,
              f"{rip_data['id']}.mp4",
              supports_streaming=True,
              caption=rip_data['title'],
              progress_callback=lambda d, t: asyncio.get_event_loop(
              ).create_task(
                  progress(d, t, v_url, c_time, "Uploading..",
                           f"{rip_data['title']}.mp4")))
          os.remove(f"{rip_data['id']}.mp4")
          await v_url.delete()
  
  
  def deEmojify(inputString):
      """ Remove emojis and other non-safe characters from string """
      return get_emoji_regexp().sub(u'', inputString)
  
  
  CMD_HELP.update({
      "img":
      ">`.img <counter> <search_query>`"
      "\nUsage: Does an image search on Google and shows images.",
      "currency":
      ">`.currency <amount> <from> <to>`"
      "\nUsage: Converts various currencies for you.",
      "carbon":
      ">`.carbon <text> [or reply]`"
      "\nUsage: Beautify your code using carbon.now.sh\n"
      "Use .crblang <text> to set language for your code.",
      "google":
      ">`.google <counter> <query>`"
      "\nUsage: Does a search on Google.",
      "wiki":
      ">`.wiki <query>`"
      "\nUsage: Does a search on Wikipedia.",
      "ud":
      ">`.ud <query>`"
      "\nUsage: Does a search on Urban Dictionary.",
      "tts":
      ">`.tts <text> [or reply]`"
      "\nUsage: Translates text to speech for the language which is set."
      "\nUse >`.lang tts <language code>` to set language for tts. (Default is English.)",
      "trt":
      ">`.trt <text> [or reply]`"
      "\nUsage: Translates text to the language which is set."
      "\nUse >`.lang trt <language code>` to set language for trt. (Default is English)",
      "yt":
      ">`.yt <text>`"
      "\nUsage: Does a YouTube search.",
      "imdb":
      ">`.imdb <movie-name>`"
      "\nUsage: Shows movie info and other stuff.",
      "rip":
      ">`.ra <url> or .rv <url>`"
      "\nUsage: Download videos and songs from YouTube "
      "(and [many other sites](https://ytdl-org.github.io/youtube-dl/supportedsites.html))."
  })
  