text: import requests
  from lxml import etree
  import re
  import pdfkit
  from PyPDF2 import PdfFileMerger
  import os
  # 敏感词过滤类，AC自动机
  import Ac_auto
  
  # pdfkit配置
  confg = pdfkit.configuration(wkhtmltopdf=r'C:\Program Files\wkhtmltopdf\bin\wkhtmltopdf.exe')
  
  # 伪装http请求头部
  headers = {
      'User-Agent':
          'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;'
  }
  
  
  def get_url_info(url_list):
      # 新闻数累加器
      sum_i = 0
  
      # 获取新闻栏目名
      news_heading = 'Test1'
  
      # 创建文件夹
      # 先判断文件夹是否存在，不存在则创建文件夹
      # now_dir = os.getcwd()
      new_dir = 'D:\\PycharmProjects\\zzu_spider' + '\\' + news_heading
      dir_judge = os.path.exists(new_dir)
      if not dir_judge:
          os.mkdir(new_dir)
          # print(new_dir)
  
      # 合并pdf
      merger = PdfFileMerger()
      # 对每页的每个新闻做处理
      for i, url in enumerate(url_list):
          # for j in range(0, 50):
          # 将新闻标题+内容整合，保存为字典
          j = 0
          r = requests.get(url, headers=headers)
          r.encoding = 'UTF-8'
          tips = '获取{}栏目下第{}页第{}条新闻，总第{}条新闻......'.format(news_heading, i + 1, j + 1, sum_i + 1)
          print(tips)
          # 引入tips, 查找爬虫出错未爬取到的空的新闻内容
          try:
              raw_html = r.text
              print(raw_html)
              html_filter = sensitive_word_filter(raw_html)
              pdfkit.from_string(raw_html, new_dir + '\\' + tips[2:-6] + '.pdf', configuration=confg)
              # 合并pdf
              pdf_file = new_dir + '\\' + tips[2:-6] + '.pdf'
              merger.append(open(pdf_file, 'rb'))
              print(merger)
              sum_i += 1
          except:
              continue
  
          with open('test1111' + '.html', 'w+') as f1:
              f1.write(raw_html)
      # 合并pdf
      merger.write(new_dir + '\\' + '合并test.pdf')
      print('{}栏目pdf合并完成'.format(news_heading))
  
  
  # 获取具体一条新闻的内容
  # def get_url_content(news_url, tips):
  #     r = requests.get(news_url, headers=headers)
  #     r.encoding = 'UTF-8'
  #     sub_html = etree.HTML(r.text)
  #     # 对内容做处理，删除空格换行转义等等字符，并进行关键词校验屏蔽
  #     # 关键字的校验屏蔽。（关键字：指的是反动言论，不文明词汇）
  #     content = sub_html.xpath('//*[@id="bok_0"]/div[@class="zzj_5"]//text()')
  #     content = ''.join(content)
  #     content = re.sub(r'\s', '', content)
  #
  #     # print(content)
  #     content = sensitive_word_filter(content)
  #
  #     # 如果出现空的内容，输出具体出错的新闻位置并生成txt
  #     if content == '':
  #         with open('C:/Users/mcgra/Desktop/spider_error.txt', 'a+') as f1:
  #             f1.write(tips)
  #             f1.write('\n')
  #
  #     return content
  
  
  # 敏感词过滤
  def sensitive_word_filter(content):
      ah = Ac_auto.ac_automation()
      path = 'sensitive_words.txt'
      ah.parse(path)
      content = ah.words_replace(content)
      # text1 = "新疆骚乱苹果新品发布会"
      # text2 = ah.words_replace(text1)
      # print(text1)
      # print(text2)
  
      return content
  
  
  def main():
      # 郑大新闻网所有的栏目链接
      # all_urls = all_urls_list()
      # for url in all_urls:
      #     url_list = get_url_list(url)
      url = ['http://news.zzu.edu.cn/', 'http://www16.zzu.edu.cn/msgs/vmsgisapi.dll/vmsglist?mtype=x&lan=203']
      get_url_info(url)
  
  
  if __name__ == '__main__':
      main()