import requests
import pymongo
import bs4

class Rapid7:
    def __init__(self):
        self.client = pymongo.MongoClient('mongodb://localhost:27017/')
        self.db = self.client['THT']
        self.col = self.db['rapid7']
        self.page = 2

    def selectArray(self,array):
        for x in array:
            txt= x.text.strip()
            return txt

    def Script(self):
        for page_n in range(1, self.page + 1):
            resp = requests.get(
                'https://www.rapid7.com/db/?q=&type=&page=' + str(page_n),
                headers={'User-Agent': 'Mozilla/5.0'}
            ).text

            parser = bs4.BeautifulSoup(resp, 'html.parser')
            urls = parser.findAll("a", {"class": "vulndb__result resultblock"},href=True)

            for url in urls:
                exploit_link= "https://www.rapid7.com/"+url['href']
                
                if len(list(self.col.find({'exploit_link': exploit_link}))) == 0:
                    resp = requests.get(
                            exploit_link,
                            headers={'User-Agent': 'Mozilla/5.0'}
                    )

                    source = bs4.BeautifulSoup(resp.content,"lxml")
                    title = source.select("body > div.off-canvas-wrapper > div > div.off-canvas-content > div.pageContent > div > div > h3")
                    severity = source.select("body > div.off-canvas-wrapper > div > div.off-canvas-content > div.pageContent > div > div > section.tableblock > div:nth-child(1) > div")
                    cvss = source.select("body > div.off-canvas-wrapper > div > div.off-canvas-content > div.pageContent > div > div > section.tableblock > div:nth-child(2) > div")
                    published = source.select("body > div.off-canvas-wrapper > div > div.off-canvas-content > div.pageContent > div > div > section.tableblock > div:nth-child(3) > div")
                    created = source.select("body > div.off-canvas-wrapper > div > div.off-canvas-content > div.pageContent > div > div > section.tableblock > div:nth-child(4) > div")
                    added = source.select("body > div.off-canvas-wrapper > div > div.off-canvas-content > div.pageContent > div > div > section.tableblock > div:nth-child(4) > div")
                    modified = source.select("body > div.off-canvas-wrapper > div > div.off-canvas-content > div.pageContent > div > div > section.tableblock > div:nth-child(4) > div")
                    description = source.select("body > div.off-canvas-wrapper > div > div.off-canvas-content > div.pageContent > div > div > section.vulndb__detail-wrapper > div > p:nth-child(3)")
                    
                    data = {
                            'title': self.selectArray(title),
                            'severity': self.selectArray(severity),
                            'cvss': self.selectArray(cvss),
                            'published': self.selectArray(published),
                            'created': self.selectArray(created),
                            'added': self.selectArray(added),
                            'modified': self.selectArray(modified),
                            'description': str(self.selectArray(description)),
                            'exploit_link': exploit_link
                        }

                    self.col.insert_one(data)

if __name__ == '__main__':
    
    App = Rapid7()
    print("Rapi7 Exploit Bilgileri Çekiliyor.")
    App.Script()
    print("İşlem Tamamlandı.")