text: import sys
  import os
  from pathlib import Path
  p = Path(os.path.abspath('__file__')).parents
  sys.path.append(str(p[1]))
  import django
  os.environ.setdefault("DJANGO_SETTINGS_MODULE", 'forums.settings')
  django.setup()
  from os.path import join
  import shutil
  import subprocess
  import threading
  import re
  import datetime
  import time
  from datetime import timedelta
  from dateutil.relativedelta import relativedelta
  import selenium
  from selenium import webdriver
  from selenium.webdriver.common.keys import Keys
  from selenium.webdriver.support.ui import Select
  from selenium.webdriver.common.by import By
  from selenium.webdriver.support.ui import WebDriverWait
  from selenium.webdriver.support import expected_conditions as EC
  from selenium.webdriver.firefox.options import Options
  from selenium.webdriver.chrome.options import Options
  from bs4 import BeautifulSoup
  
  from yahoo.models import Post
  from yahoo.models import Reply
  
  
  # 设置代理
  chrome_options = webdriver.ChromeOptions()
  # chrome_options.add_argument('--headless')
  # chrome_options.add_argument('--disable-gpu')
  chrome_options.add_argument('--proxy-server=socks5://localhost:1080')
  
  
  class Tools(object):
      '''处理一些表情符号和字符'''
      def remove_blank(self,s):
          '''去除空白字符'''
          blank_pattern = re.compile('\s')
          return blank_pattern.sub(r'', s)
  
      def remove_emoji(self,s):
          '''去除表情'''
          emoji_pattern = re.compile('[\U00010000-\U0010ffff]', flags=re.UNICODE)
          return emoji_pattern.sub(r'', str(s))
  
      def remove_comma(self,s):
          '''剔除数字中的逗号'''
          p = re.compile(",")
          return p.sub('', s)
  
  
  class DatetimeConversion(object):
      '''将日期和时间转换为标准的时间戳格式'''
      def strings_to_datetime(self, strdt):
          '''時、天、秒等向标准时间的转换'''
          try:
              if '秒' in strdt:
                  n = int(re.search(r'\d+', strdt).group())
                  strdt = datetime.datetime.now() - timedelta(days=n)
              elif '鐘' in strdt:
                  # 处理分钟
                  n = int(re.search(r'\d+', strdt).group())
                  strdt = datetime.datetime.now() - timedelta(minutes=n)
              elif '時' in strdt:
                  n = int(re.search(r'\d+', strdt).group())
                  strdt = datetime.datetime.now() - timedelta(hours=n)
              elif '天' in strdt:
                  n = int(re.search(r'\d+', strdt).group())
                  strdt = datetime.datetime.now() - timedelta(days=n)
              elif '星期' or '週' in strdt:
                  n = int(re.search(r'\d+', strdt).group())
                  strdt = datetime.datetime.now() - timedelta(weeks=n)
              elif '月' in strdt:
                  n = int(re.search(r'\d+', strdt).group())
                  strdt = datetime.datetime.now() - relativedelta(months=n)
              elif '年' in strdt:
                  n = int(re.search(r'\d+', strdt).group())
                  strdt = datetime.datetime.now() - relativedelta(years=n)
              elif '剛剛' in strdt:
                  # 处理刚刚
                  strdt = datetime.datetime.now()
              elif '昨天' in strdt:
                  # 处理昨天
                  strdt = datetime.datetime.now() - timedelta(days=1)
              elif '前天' in strdt:
                  # 处理前天
                  strdt = datetime.datetime.now() - timedelta(days=2)
              else:
                  print("请处理这种时间格式", strdt)
              return strdt
          except Exception as e:
              print("时间转换失败",e)
  
      def str_to_time(self, strt):
          '''将带上午下午的格式的时间转为标准格式'''
          strs = strt.split(" ")
          time_ = strs[1]
          morn_after = strs[0]
          morn_after = str(morn_after)
          if "上" == morn_after:
              try:
                  i = int(time_[0:2])
              except ValueError:
                  i = int(time_[0:1])
              if  i == 12:
                  hour = "00"
                  strt = hour + time_[2:]
              else:
                  strt = time_
  
          elif "下" == morn_after:
              try:
                  h = int(time_[0:2])
              except ValueError:
                  h = int(time_[0:1])
              if h < 12:
                  hour = str(h + 12)
                  if h >= 10:
                      strt = hour + time_[2:]
                  else:
                      strt = hour + time_[1:]
              else:
                  hour = "00"
                  strt = hour + time_[2:]
          else:
              print("格式不对")
          # 返回的是字符格式
          return strt
  
  
  class FecthLinkAnswer(object):
      '''抓取回答类型的链接'''
      def __init__(self,deep):
          self.deep = deep
          self.pages = [] # 存储新链接
          self.dbpages = set() # 数据库中的链接
          self.updatepag = [] # 存储旧链接
      
      def fecth_link(self,browser):
          '''获取整个页面的话题链接'''
          elem = WebDriverWait(browser, 15).until(
              EC.presence_of_element_located((By.XPATH, '//*[@id="ya-answer-tab"]')))
          link = 'https://tw.answers.yahoo.com'
          soup = BeautifulSoup(elem.get_attribute('innerHTML'), 'lxml')
          lis = soup.find_all('li', class_="qTile P-14 Mih-60 Bdbx-1g Bgc-w")
          for li in lis:
              # 将新连接和旧链接分开存储
              post_url = link + li.find('a', class_='Clr-b').get('href')
              post_datetime = li.find('div',class_='Fz-12 Clr-888').text
              post_datetime = post_datetime.split('·')[-1]
              # print("post时间", post_datetime)
              if not post_url in self.dbpages:
                  self.pages.append([post_url,post_datetime])
              else:
                  self.updatepag.append([post_url,post_datetime])
              if self.deep <= len(self.pages) + len(self.updatepag):
                  break
      
      def fecth_update(self,browser):
          '''根据deep判断是否下拉加载数据'''
          # 从数据库中提取链接
          for i in Post.objects.all():
              self.dbpages.add(i.post_url)
  
          elem = WebDriverWait(browser, 15).until(
              EC.presence_of_element_located((By.XPATH, '//*[@id="ya-answer-tab"]')))
          soup = BeautifulSoup(elem.get_attribute('innerHTML'), 'lxml')
          lis = soup.find_all('li', class_="qTile P-14 Mih-60 Bdbx-1g Bgc-w")
          while len(lis) < self.deep:
              lens = len(lis)
              # 下拉加载数据数据
              browser.execute_script("window.scrollTo(0,document.body.scrollHeight)")
              # 等待一会
              time.sleep(5)
              elem = WebDriverWait(browser, 15).until(
                  EC.presence_of_element_located((By.XPATH, '//*[@id="ya-answer-tab"]')))
              soup = BeautifulSoup(elem.get_attribute('innerHTML'), 'lxml')
              lis = soup.find_all('li', class_="qTile P-14 Mih-60 Bdbx-1g Bgc-w")
              # 下拉没有数据加载
              if len(lis) == lens:
                  break
          # 开始获取链接数和时间
          self.fecth_link(browser)
          return self.pages, self.updatepag
  
  
  class FecthLinkDiscover(object):
      '''抓取热门类型帖子的链接'''
      def __init__(self,deep):
          self.deep = deep
          self.pages = [] # 存储新链接
          self.dbpages = set() # 数据库中的链接
          self.updatepag = [] # 存储旧链接
  
      def fecth_link(self,browser):
          '''获取整个页面的话题链接'''
          elem = WebDriverWait(browser, 15).until(
              EC.presence_of_element_located((By.XPATH, '//*[@id="ya-discover-tab"]')))
          link = 'https://tw.answers.yahoo.com'
          soup = BeautifulSoup(elem.get_attribute('innerHTML'), 'lxml')
          lis = soup.find_all('li', class_="ya-discover-tile ya-discover-tile-qn Bfc P-14 Bdbx-1g Bgc-w Pos-r")
          for li in lis:
              # 将新连接和旧链接分开存储
              post_url = link + li.find('a', class_='Fz-14 Fw-b Clr-b Wow-bw title').get('href')
              post_datetime = li.find('div',class_='Clr-888 Fz-12 Lh-18').text
              post_datetime = post_datetime.split('·')[-1]
              # print("post时间",(post_datetime))
              if not post_url in self.dbpages:
                  self.pages.append([post_url,post_datetime])
              else:
                  self.updatepag.append([post_url,post_datetime])
              if self.deep <= len(self.pages) + len(self.updatepag):
                  break
  
      def fecth_update(self,browser):
          '''根据deep判断是否下拉加载数据'''
          # 从数据库中提取链接
          for i in Post.objects.all():
              self.dbpages.add(i.post_url)
  
          elem = WebDriverWait(browser, 15).until(
              EC.presence_of_element_located((By.XPATH, '//*[@id="ya-discover-tab"]')))
          soup = BeautifulSoup(elem.get_attribute('innerHTML'), 'lxml')
          lis = soup.find_all('li', class_="ya-discover-tile ya-discover-tile-qn Bfc P-14 Bdbx-1g Bgc-w Pos-r")
          while len(lis) < self.deep:
              lens = len(lis)
              # 下拉加载数据数据
              browser.execute_script("window.scrollTo(0,document.body.scrollHeight)")
              # 等待一会
              time.sleep(5)
              elem = WebDriverWait(browser, 15).until(
                  EC.presence_of_element_located((By.XPATH, '//*[@id="ya-discover-tab"]')))
              soup = BeautifulSoup(elem.get_attribute('innerHTML'), 'lxml')
              lis = soup.find_all('li', class_="ya-discover-tile ya-discover-tile-qn Bfc P-14 Bdbx-1g Bgc-w Pos-r")
              # 下拉没有数据加载
              if len(lis) == lens:
                  break
          # 开始获取链接数和时间
          self.fecth_link(browser)
          return self.pages, self.updatepag
  
  
  class FecthData(DatetimeConversion):
      '''获取post、reply数据'''
      def fecth_post(self,post_url,strdt,browser):
          browser.get(post_url)
          # 加载一下数据
          time.sleep(3)
          # 采集发文时间
          xpath = '//*[@id="Stencil"]/head/meta[10]'
          post_datetime_ = browser.find_element_by_xpath(xpath).get_attribute("content")
          t = re.sub(r'T',' ',post_datetime_)
          temp = re.sub(r'Z','',t)
          post_datetime = datetime.datetime.strptime(temp, "%Y-%m-%d %H:%M:%S")
          print("时间是", post_datetime)
  
          # 采集发文者姓名
          try:
              x = '//*[@id="yq-question-detail-profile-img"]'
              name = browser.find_element_by_xpath(x)
              name_ = BeautifulSoup(name.get_attribute('innerHTML'), 'lxml')
              poster_name = name_.find("img").get("alt")
              print("发文者名字", poster_name)
              post_id = name_.find("img").get("data-id")
              print('发文者id', post_id)
          except AttributeError:
              poster_name = "**发文者为匿名**"
              print(poster_name)
              post_id = "**发文者为匿名,无id**"
              print("发文者id",post_id)
          except Exception as e:
              print('抓取发文者账号时遇到异常', e)
              return
          category = browser.find_element_by_id('brdCrb').text
          category = re.sub(" ", ',', category)
          print("类别是",category)
          title = Tools().remove_emoji(browser.find_element_by_tag_name('h1').text)
          print("标题是",title)
  
          try:
              content = WebDriverWait(browser, 5).until(EC.presence_of_element_located((By.CLASS_NAME,'ya-q-text'))).text
              if len(content) == 0:
                  content = "**发文者的内容为空**"
                  print(content)
              print("发文内容是", content)
          except Exception as e:
              # 没有内容时
              print("抓取发文内容时遇到异常",e)
  
  
          print("发文连接是",post_url)
          post = Post(title=title,post_url=post_url,category=category,
                      content=content,post_datetime=post_datetime,entrance_id=0, poster_name=poster_name,post_id=post_id)
          post.save()
          return  post
  
      def fecth_reply(self,post,browser):
          # 抓取不要过快或者在不抓取post的情况下加载一下数据
          time.sleep(1)
          browser.execute_script("window.scrollTo(0,document.body.scrollHeight)")
          # 最优答案的情况
          try:
              bestanswer = browser.find_element_by_class_name('Py-12')
              soup = BeautifulSoup(bestanswer.get_attribute('innerHTML'), 'lxml')
              l = soup.find('div',attrs={'class':'Pt-15',}).text
              reply_poster = Tools().remove_blank(l.split('·')[0])
              print("回复者是",reply_poster)
              # 回复者的id
              try:
                  id_ =  soup.find("a",class_="Clr-b").get('href')
                  t = re.search(r"=\w+&",id_).group()
                  temp = re.sub(r"=",'',t)
                  reply_id = re.sub(r'&','',temp)
                  print('回复者id是', reply_id)
              except AttributeError:
                  reply_id = "**回复者为匿名,无回复者id**"
                  print(reply_id)
              except Exception as e:
                  print('在获取回复者id时遇到错误', e)
              reply_datetime = soup.find(
                  'div',attrs={'class':'Pt-15',}).find("span",attrs={"class":"Clr-88 ya-localtime"}).get('title')
              # 处理时间
              l = reply_datetime.split(' ')
              date_ = l[0]
              d = date_.split('/')
              date_ = d[0] + "-" + d[1] + '-' + d[2]
              print("日期是", date_)
              strlist = l[1].split("午")
              strt = strlist[0] + " " + strlist[1]
              print("字符串", strt)
              time_ = self.str_to_time(strt)
              print('时间', time_)
              reply_datetime = date_ + ' '  + time_
              reply_datetime = str(datetime.datetime.strptime(reply_datetime, "%Y-%m-%d %H:%M:%S"))
  
              print("回复时间", reply_datetime)
              reply_content =Tools().remove_emoji(soup.find('span',attrs={'itemprop':'text'}).text)
              print("回复内容是",reply_content)
              reply_positive = int(soup.find("div",attrs={'itemprop':'upvoteCount'}).text)
              print("赞数是",reply_positive)
              reply_negative = int(soup.find("div",attrs={'class':'D-ib Mstart-23 count'}).text)
              print("嘘数",reply_negative)
              reply = Reply(reply_poster=reply_poster,
                            reply_content=reply_content,reply_datetime=reply_datetime,
                            reply_positive=reply_positive,reply_negative=reply_negative,
                            reply_id=reply_id, post=post)
              reply.save()
          except AttributeError:
              # 没有最优答案
              pass
          except Exception as e:
              print('在进行抓取最优答案时遇到错误',e)
  
          try:
              answer = browser.find_element_by_id('ya-qn-answers')
              soup = BeautifulSoup(answer.get_attribute('innerHTML'), 'lxml')
              # print(soup)
              divs = soup.find_all("div",attrs={'class':'Mstart-75 Pos-r'})
              print("divs的长度",len(divs))
              # 获取reply
              for div in divs:
                  l = div.find('div', attrs={'class': 'Pt-15', }).text
                  reply_poster = Tools().remove_blank(l.split('·')[0])
                  print("回复者是", reply_poster)
                  # 回复者的id
                  try:
                      id_ = div.find("a", class_="uname Clr-b").get('href')
                      t = re.search(r"=\w+&", id_).group()
                      temp = re.sub(r"=", '', t)
                      reply_id = re.sub(r'&', '', temp)
                      print('回复者id是', reply_id)
                  except AttributeError:
                      reply_id = "**回复者为匿名,无回复者id**"
                      print(reply_id)
                  except Exception as e:
                      print('在获取回复者id时遇到错误',e)
                  reply_content = Tools().remove_emoji(div.find('span', attrs={'itemprop': 'text'}).text)
                  print("回复内容是", reply_content)
                  reply_datetime = div.find('div', attrs={'class': 'Pt-15', }).find("span", attrs={
                      "class": "Clr-88 ya-localtime"}).get('title')
                  print("回复时间是",reply_datetime)
                  l = reply_datetime.split(' ')
                  date_ = l[0]
                  d = date_.split('/')
                  date_ = d[0] + "-" + d[1] + '-' + d[2]
                  print("日期是", date_)
                  strlist = l[1].split("午")
                  strt = strlist[0] + " " + strlist[1]
                  print("字符串", strt)
                  time_ = self.str_to_time(strt)
                  print('时间', time_)
                  reply_datetime = date_ + ' ' + time_
                  reply_datetime = str(datetime.datetime.strptime(reply_datetime, "%Y-%m-%d %H:%M:%S"))
  
                  print("{}的回复时间是{}".format(reply_poster, reply_datetime))
                  reply_positive = int(div.find("div", attrs={'itemprop': 'upvoteCount'}).text)
                  print("赞数是", reply_positive)
                  reply_negative = int(div.find("div", attrs={'class': 'D-ib Mstart-23 count'}).text)
                  print("嘘数", reply_negative)
                  reply = Reply(reply_poster=reply_poster,
                                reply_content=reply_content, reply_datetime=reply_datetime,
                                reply_positive=reply_positive, reply_negative=reply_negative,
                                reply_id=reply_id, post=post)
                  reply.save()
          except selenium.common.exceptions.NoSuchElementException:
              # 没有回复信息
              return
          except Exception as e:
              print('在抓取回复者数据时遇到错误，错误信息是',e)
  
      def next_reply(self,post,browser):
          '''翻页,点击翻页按钮回去下一页的reply'''
          while True:
              try:
                  self.fecth_reply(post,browser)
                  ck =browser.find_element_by_link_text('下一頁')
                  browser.execute_script("arguments[0].scrollIntoView(false);", ck)
                  browser.execute_script('arguments[0].click()', ck)
                  # 加载一会，不要太快
                  time.sleep(2.5)
              except selenium.common.exceptions.NoSuchElementException:
                  # 没有下一页
                  break
  
  
  class UpdateData(FecthData):
      '''继承类FecthData使用方法next_reply和fecth_reply'''
      def update_reply(self,link,browser):
          browser.get(link)
          time.sleep(1)
          post_id = Post.objects.get(post_url=link).id
          post = Post.objects.get(post_url=link)
          Reply.objects.filter(post_id=post_id).delete()
          self.next_reply(post,browser)
  
  
  def job(deep, genre="https://tw.answers.yahoo.com/dir/index/discover",
          browser=webdriver.Chrome(options=chrome_options)):
      '''整个的项目的外界入口'''
      browser.maximize_window()
      url = genre
      browser.get(url)
      time.sleep(2)
      print("程序开始运行...")
      # 对回答和热门进行区分
      # 回答的数据
      if '/index/answer' in url:
          feup = FecthLinkAnswer(deep)
          pages, updatepag = feup.fecth_update(browser)
  
          # 抓取新话题数据
          print("总共要抓取{}条数据".format(len(pages)))
          n = 1
          for link,strdt in pages:
              print("正在抓取第{}条数据...".format(n))
              post = FecthData()
              post_ = post.fecth_post(link, strdt, browser)
              post.next_reply(post_, browser)
              n += 1
          print("抓取完毕\n")
  
          #更新数据
          print("共更新{}条数据".format(len(updatepag)))
          n = 1
          for link in updatepag:
              print("正在更新第{}条数据...".format(n))
              reply = UpdateData()
              reply.update_reply(link[0],browser)
              n += 1
          print("更新完毕")
          browser.quit()
      # 热门的数据
      elif '/index/discover' in url:
          feup = FecthLinkDiscover(deep)
          pages, updatepag = feup.fecth_update(browser)
          # 抓取新话题数据
          print("总共要抓取{}条数据".format(len(pages)))
          n = 1
          for link, strdt in pages:
              print("正在抓取第{}条数据...".format(n))
              post = FecthData()
              post_ = post.fecth_post(link, strdt, browser)
              post.next_reply(post_, browser)
              # 不要爬太快
              time.sleep(2)
              n += 1
          print("抓取完毕\n")
  
          # 更新数据
          print("共更新{}条数据".format(len(updatepag)))
          n = 1
          for link in updatepag:
              print("正在更新第{}条数据...".format(n))
              reply = UpdateData()
              reply.update_reply(link[0], browser)
              n += 1
          print("更新完毕")
          browser.quit()
      else:
          print("输出这个链接，这个链接里没有answer和discover")
  
  
  if __name__ == '__main__':
      job(100)
  