articles/models.py
def extract_article(self):
        """
        Extracts article out of a downloaded URL as text
        """
        # Get an article
        html = self.download_page()
        soup = BeautifulSoup(html, "html5lib")
        if not soup:
            raise ValueError("Sorry, I could not parse that page properly")

        # Get author, title and name
        author_tag = soup.find(rel="author")
        self.author = author_tag and author_tag.get_text() or "Unknown Author"

        title_tag = soup.find("h1", {'class' : 'content__headline'}) or soup.find("title")
        self.title = title_tag and title_tag.get_text().split("|")[0].strip() or "Unknown Title"

        # Check to see if CIF, either from URL or if the section is labelled as such in the metadata
        section_tag = soup.find("meta", property="article:section")
        section = section_tag and section_tag['content'] or "Unknown Section"
        self.is_cif = self.url.find('commentisfree') > -1 or section.lower() == "comment is free"

        # Desktop and mobile search for the relevant div
        div = soup.find("div", class_="content__article-body")
        if not div:
            raise ValueError("Sorry, I could not find an article in that page")

        # Get rid of blockquotes so the count is fair(er)
        for quote in div.find_all("blockquote"):
            quote.decompose()
        for aside in div.find_all("aside"):
            aside.decompose()
        for figure in div.find_all("figure"):
            figure.decompose()

        # Cleanup whitespace, convert all to spaces and then return
        text = div.get_text(" ", strip=True) or ""
        text = re.sub("\s+", " ", text)
        return text

        Extracts article out of a downloaded URL as text
        
Extracts article out of a downloaded URL as text
Extracts article out of a downloaded URL as text
def extract_article(self):
        html = self.download_page()
        soup = BeautifulSoup(html, "html5lib")
        if not soup:
            raise ValueError("Sorry, I could not parse that page properly")
        author_tag = soup.find(rel="author")
        self.author = author_tag and author_tag.get_text() or "Unknown Author"
        title_tag = soup.find("h1", {'class' : 'content__headline'}) or soup.find("title")
        self.title = title_tag and title_tag.get_text().split("|")[0].strip() or "Unknown Title"
        section_tag = soup.find("meta", property="article:section")
        section = section_tag and section_tag['content'] or "Unknown Section"
        self.is_cif = self.url.find('commentisfree') > -1 or section.lower() == "comment is free"
        div = soup.find("div", class_="content__article-body")
        if not div:
            raise ValueError("Sorry, I could not find an article in that page")
        for quote in div.find_all("blockquote"):
            quote.decompose()
        for aside in div.find_all("aside"):
            aside.decompose()
        for figure in div.find_all("figure"):
            figure.decompose()
        text = div.get_text(" ", strip=True) or ""
        text = re.sub("\s+", " ", text)
        return text
def extract_article ( self ) : html = self . download_page ( ) soup = BeautifulSoup ( html , "html5lib" ) if not soup : raise ValueError ( "Sorry, I could not parse that page properly" ) author_tag = soup . find ( rel = "author" ) self . author = author_tag and author_tag . get_text ( ) or "Unknown Author" title_tag = soup . find ( "h1" , { 'class' : 'content__headline' } ) or soup . find ( "title" ) self . title = title_tag and title_tag . get_text ( ) . split ( "|" ) [ 0 ] . strip ( ) or "Unknown Title" section_tag = soup . find ( "meta" , property = "article:section" ) section = section_tag and section_tag [ 'content' ] or "Unknown Section" self . is_cif = self . url . find ( 'commentisfree' ) > - 1 or section . lower ( ) == "comment is free" div = soup . find ( "div" , class_ = "content__article-body" ) if not div : raise ValueError ( "Sorry, I could not find an article in that page" ) for quote in div . find_all ( "blockquote" ) : quote . decompose ( ) for aside in div . find_all ( "aside" ) : aside . decompose ( ) for figure in div . find_all ( "figure" ) : figure . decompose ( ) text = div . get_text ( " " , strip = True ) or "" text = re . sub ( "\s+" , " " , text ) return text
Extracts article out of a downloaded URL as text
Extracts article out of a downloaded URL as text
"""
        Extracts article out of a downloaded URL as text
        """ # Get an article # Get author, title and name # Check to see if CIF, either from URL or if the section is labelled as such in the metadata # Desktop and mobile search for the relevant div # Get rid of blockquotes so the count is fair(er) # Cleanup whitespace, convert all to spaces and then return
self