Python selenium 爬取cnvd(国家信息安全漏洞共享平台)

#coding = utf-8
#@author :今夕
#@Time :2021.08.06 16:09
#@file :mian.py
#@software :PyCharm
import time
from selenium import webdriver
from bs4 import BeautifulSoup
import re
import pymysql
import random
#应用漏洞获取
list=[27,28,29,30,31,32,33,34,35,38]
page=[13700,79200,30800,2500,9400,2600,1000,300,0,100]
def main():
  driver = webdriver.Chrome()
  driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
    "source": """
      Object.defineProperty(navigator, 'webdriver', {
        get: () => undefined
      })
    """
  })
  i=0
  while i<len(list):
    leng=0
    while leng<=page[i]:
      url = "https://www.cnvd.org.cn/flaw/typeResult?typeId=%d&max=100&offset=%d" % (list[i],leng)
      leng=leng+100
      driver.get(url)
      driver.refresh()  # 刷新页面
      html = driver.page_source
      datalist = parseHtml(html,list[i])
      save(datalist)
      driver.refresh()
      print("当前i=%d,leng=%d"%(list[i],leng))
  driver.close()
def parseHtml(html,type):
  datlist=[]#存一页漏洞基本信息
  soup = BeautifulSoup(html, "html.parser")  # 指定Beautiful的解析器为“html.parser”
  findLevel = re.compile(r'</span>(.*?)</td>', re.S)#危害级别匹配规则
  findDate = re.compile(r'<td width="13%">(.*?)</td>', re.S)#时间匹配规则
  k=1
  items=soup.find_all('tr')
  if type==27:
    type="操作系统"
  elif type==28:
    type="应用程序"
  elif type==29:
    type="WEB应用"
  elif type==30:
    type="数据库"
  elif type==31:
    type="网络设备（交换机，路由器等网络终端设备）"
  elif type==32:
    type="安全产品"
  elif type==33:
    type="智能设备（物联网终端设备）"
  elif type==34:
    type="区块链公链"
  elif type==35:
    type="区块链联盟链"
  elif type==38:
    type="工业控制系统"
  while k<len(items):
    dat = []  # 存一个漏洞基本信息
    item=items[k]
    link="https://www.cnvd.org.cn"+item.a["href"]#获取漏洞详细信息链接
    #print(link)
    title=item.a["title"]#获取漏洞标题
    #print(title)
    item=str(item)#转换为字符串
    level=re.findall(findLevel,item)[0]
    #print(level)
    date=re.findall(findDate,item)[0]
    #print(date)
    #print("___________________________")
    dat.append(title)
    dat.append(level)
    dat.append(date)
    dat.append(link)
    dat.append(type)
    datlist.append(dat)
    #print(dat)
    k+=1
  return datlist
def save(datlist):
  #print(datlist[1])
  db = pymysql.connect(user='root', password='123456', host='localhost', database='cnvd')
  cursor = db.cursor()
  for dat in datlist:
    data="'"+dat[0]+"'"+","+"'"+dat[1].replace(" ","").replace("\n","").replace("\t","")+"'"+","+"'"+dat[2]+"'"+","+"'"+dat[3]+"'"+","+"'"+dat[4]+"'"
    print(data)
    sql = "insert into cnvd (title,level,date,link,type) values(%s)"%data
    #print(sql)
    cursor.execute(sql)  # 提交数据库操作
    db.commit()  # 提交事务
  cursor.close()
  db.close()
  t = random.randint(1,6)  # 取随机数
  print("等待%d秒" % t)
  time.sleep(t)  # 休眠t秒

#输入selenium 搜索
# element = driver.find_element_by_name('a')
# ActionChains(driver).move_to_element(element).perform()
#单击，弹出的Ajax元素，根据链接节点的文本内容查找

if __name__ == '__main__':
    main()
    print("爬取完成")