fbscraper/scraper.py
def _getComments(self,
                     postURL):
        """routine to extract post comments recursively
        -----------------------------------------------
        Input:
        :param postURL: URL of the post
        """
        self._browser.get(postURL)
        delay()
        soup = bs(self._browser.page_source, "lxml")
        commentClass = getDivClass(soup)
        divs = getFilteredDivs(soup.findAll("div", class_=commentClass))
        batch = list()
        for div in divs:
            comment = parseComment(div, self._data["postID"])
            batch.append(comment)
        nCommentsRequired = self._data["nComments"] - len(self._data["post"]["comments"])
        nCommentsRequired = min(nCommentsRequired, len(batch))
        if nCommentsRequired == 0:
            return
        self._data["post"]["comments"].extend(batch[:nCommentsRequired])
        self._data["progressBar"].update(nCommentsRequired)
        nextLink = getMoreCommentsLink(soup, self._data["postID"])
        if nextLink is not None:
            try:
                self._getComments(nextLink)
            except:
                pass
routine to extract post comments recursively
        -----------------------------------------------
        Input:
        :param postURL: URL of the post
        
routine to extract post comments recursively
Input.
routine to extract post comments recursively Input .
def _getComments(self,
                     postURL):
        self._browser.get(postURL)
        delay()
        soup = bs(self._browser.page_source, "lxml")
        commentClass = getDivClass(soup)
        divs = getFilteredDivs(soup.findAll("div", class_=commentClass))
        batch = list()
        for div in divs:
            comment = parseComment(div, self._data["postID"])
            batch.append(comment)
        nCommentsRequired = self._data["nComments"] - len(self._data["post"]["comments"])
        nCommentsRequired = min(nCommentsRequired, len(batch))
        if nCommentsRequired == 0:
            return
        self._data["post"]["comments"].extend(batch[:nCommentsRequired])
        self._data["progressBar"].update(nCommentsRequired)
        nextLink = getMoreCommentsLink(soup, self._data["postID"])
        if nextLink is not None:
            try:
                self._getComments(nextLink)
            except:
                pass
def _getComments ( self , postURL ) : self . _browser . get ( postURL ) delay ( ) soup = bs ( self . _browser . page_source , "lxml" ) commentClass = getDivClass ( soup ) divs = getFilteredDivs ( soup . findAll ( "div" , class_ = commentClass ) ) batch = list ( ) for div in divs : comment = parseComment ( div , self . _data [ "postID" ] ) batch . append ( comment ) nCommentsRequired = self . _data [ "nComments" ] - len ( self . _data [ "post" ] [ "comments" ] ) nCommentsRequired = min ( nCommentsRequired , len ( batch ) ) if nCommentsRequired == 0 : return self . _data [ "post" ] [ "comments" ] . extend ( batch [ : nCommentsRequired ] ) self . _data [ "progressBar" ] . update ( nCommentsRequired ) nextLink = getMoreCommentsLink ( soup , self . _data [ "postID" ] ) if nextLink is not None : try : self . _getComments ( nextLink ) except : pass
routine to extract post comments recursively
Input:
routine to extract post comments recursively Input :
"""routine to extract post comments recursively
        -----------------------------------------------
        Input:
        :param postURL: URL of the post
        """
self
postURL
URL of the post
URL of the post