text: import requests
  from bs4 import BeautifulSoup
  import codecs
  import os
  import glob
  import shutil
  import zipfile
  import re
  import logging
  import configparser
  from lib.json2sqlite3 import json2tuple_dict
  from lib.utils import time_delta,path
  from lib.sqlite3_operate import SQLite,cve_exists,cve_exists_where,sqlite_insert
  from lib.exp_label import exp_table
  
  def json_download(fname,url,retry=3,ret=False):
      """
      下载cve json格式的zip
      """
      while retry>0:
          try:
              print("[+] DOWNLOAD %s to %s %s" %(fname,url,retry))
              retry=retry-1
              r=requests.get(url,stream=True,timeout=100) # stream=True
              with open(path('../data/json',fname),'wb') as f:
                  shutil.copyfileobj(r.raw,f) 
              with zipfile.ZipFile(path('../data/json',fname)) as zf:
                  print("[+] UNZIP %s" %fname)
                  retry=0
                  zf.extractall(path=path('../data/json'))
                  ret=True
          except Exception as e:
              ret=False
              print("[DOWNLOAD ERROR] %s error:%s" %(url,repr(e)))
  
      return ret
  
  def day_modified_exp(data,mod_time=None,pub_time=None,key=None):
      """
      从data中根据更新时间等关键字取CVE id
      :param data:tuple_dict, e.g. {1:('xx','xx'),2:('xx','xx')}
      :param time:lastModifiedDate, e.g. "2020-06-17T00:15Z"
      :param key:key words, e.g. "Exploit"
      :return cve_exp:tuple_dict, e.g. 
      """
      cve_exp=dict()
      url='https://nvd.nist.gov/vuln/detail/'
      if mod_time and key:
          for ix,te in data.items():
              if mod_time in te[63] and key in te[12]: # todo 优化
                  cve_exp[te[3]]=(te[3],te[14],te[62],te[63],url+te[3])
      if pub_time and not key:
          for ix,te in data.items():
              if pub_time in te[62]:
                  cve_exp[te[3]]=(te[3],te[14],te[62],te[63],url+te[3])
      
      return cve_exp
  
  def cve_monitor(monitor_init=False):
      """
      存量离线解析，增量在线更新
      :return ret:boolean, if parsed json to sqlite3 or not
      :return cve_day_add:tuple_dict, 当天当次新增cve
      :return exp_day_add:tuple_dict, 当天新增exp
      """
      json_list=[]
      # 取配置文件中控制存量解析的参数,得到待解析入库文件列表
      if monitor_init=='True':
          # 清空已有的CVE数据
          so=SQLite('data/cve.db')
          so.execute('delete from nvd')
  
          zips=glob.glob('data/json/nvdcve-1.1-*.json.zip')
          for z in zips:
              with zipfile.ZipFile(z) as zf:
                  print("[+] UNZIP %s" %z)
                  zf.extractall(path=path('data/json'))
  
          jsons_stock=glob.glob('data/json/nvdcve-1.1-*.json')
          jsons_stock=[i for i in jsons_stock]
          json_list.extend(jsons_stock)
      else:
          # 下载最新的modified数据 
          modified_zip,modified_link=('nvdcve-1.1-modified.json.zip','https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.zip')
          json_download(modified_zip,modified_link)
          json_list.extend([path('../data/json','nvdcve-1.1-modified.json')])
  
      # 统一解析存量和增量cve数据 
      for j in json_list:
          # 插入前处理逻辑：从modified中获取cve exp added
          exp_day_add=dict()
          cve_day_add=dict()
          if "modified" in j:
              sql,modified_data=json2tuple_dict(j)
              modified_time=time_delta(delta=0,format="%Y-%m-%d")
              cve_exps=day_modified_exp(modified_data,mod_time=modified_time,key='Exploit')
              cve_list=list(cve_exps.values())
              cve_add=day_modified_exp(modified_data,pub_time=modified_time)
              cve_add_list=list(cve_add.values())
              for cve in cve_list:
                  so,exid=cve_exists_where(db='cve',table='nvd',key=['CVE_Items_cve_CVE_data_meta_ID'],where='CVE_Items_cve_CVE_data_meta_ID="{d}"'.format(d=cve[0]))
                  so1,eid=cve_exists_where(db='cve',table='nvd',key=['CVE_Items_cve_CVE_data_meta_ID'],where='CVE_Items_cve_CVE_data_meta_ID="{d}" and CVE_Items_cve_references_reference_data_tags not like "%Exploit%"'.format(d=cve[0]))
                  if exid:
                      if eid:
                          print('[EXP] %s occurs to pre cve.db with no exploit,so added' %cve[0])
                          exp_day_add[cve[0]]=cve
                      else:
                          print('[EXP] %s occurs to prev cve.db with exploit,so give up' %cve[0])
                  else:
                      print('[EXP] %s never occurs to prev cve.db with exploit,so added' %cve[0])
                      exp_day_add[cve[0]]=cve
              for cve in cve_add_list:
                  so,exid=cve_exists_where(db='cve',table='nvd',key=['CVE_Items_cve_CVE_data_meta_ID'],where='CVE_Items_cve_CVE_data_meta_ID="{d}"'.format(d=cve[0]))
                  if exid:
                      print('[CVE] %s occurs to prev cve.db,so give up' %cve[0])
                  else:
                      print('[CVE] %s never occurs to prev cve.db,so added' %cve[0])
                      cve_day_add[cve[0]]=cve
  
          # 将nvd日新增的exp插入exp.db
          first_part_exp_add=list(exp_day_add.values())
          exp_table(first_part_exp_add)
          # 开始正式插入cve.db
          sql,cve_data=json2tuple_dict(j)
          ret=sqlite_insert(sql,cve_data,dir_name='data/cve.db')
          print("[+] Parsed %s to sqlite3" %j)
  
      return ret,cve_day_add,exp_day_add
  
  if __name__=="__main__":
      pass
      
  