zeeguu/core/model/article.py
def find_or_create(
        cls,
        session,
        _url: str,
        language=None,
        sleep_a_bit=False,
        htmlContent=None,
        title=None,
        authors: str = "",
    ):
        """

            If article for url found, return ID

            If not found,

                - if htmlContent is present, create article for that
                - if not, download and create article then return

        :param url:
        :return:
        """
        from zeeguu.core.model import Url, Article, Language

        url = Url.extract_canonical_url(_url)

        try:
            found = cls.find(url)
            if found:
                return found

            if htmlContent:
                text = re.sub(HTML_TAG_CLEANR, "", htmlContent)

                # replace many newlines with max two; in some
                # cases many newlines are left after stripping the html tags
                text = re.sub(MULTIPLE_NEWLINES, "\n\n", text)
                text = text.strip()

                summary = text[0:MAX_CHAR_COUNT_IN_SUMMARY]
                lang = detect(text)
            else:
                title, text, summary, lang, author_list = download_url(url, sleep_a_bit)
                authors = ", ".join(author_list or [])

            language = Language.find(lang)

            # Create new article and save it to DB
            url_object = Url.find_or_create(session, url)

            new_article = Article(
                url_object,
                title,
                authors,
                text,  # any article longer than this will be truncated...
                summary,
                datetime.now(),
                None,
                language,
                htmlContent,
            )
            session.add(new_article)
            session.commit()

            return new_article
        except sqlalchemy.exc.IntegrityError or sqlalchemy.exc.DatabaseError:
            for i in range(10):
                try:
                    session.rollback()
                    u = cls.find(url)
                    print("Found article by url after recovering from race")
                    return u
                except:
                    print("Exception of second degree in article..." + str(i))
                    time.sleep(0.3)
                    continue
                break


            If article for url found, return ID

            If not found,

                - if htmlContent is present, create article for that
                - if not, download and create article then return

        :param url:
        :return:
        
If article for url found, return ID
If not found.

if htmlContent is present, create article for that
if not, download and create article then return
If article for url found return ID If not found . if htmlContent is present create article for that if not download and create article then return
def find_or_create(
        cls,
        session,
        _url: str,
        language=None,
        sleep_a_bit=False,
        htmlContent=None,
        title=None,
        authors: str = "",
    ):
        from zeeguu.core.model import Url, Article, Language
        url = Url.extract_canonical_url(_url)
        try:
            found = cls.find(url)
            if found:
                return found
            if htmlContent:
                text = re.sub(HTML_TAG_CLEANR, "", htmlContent)
                text = re.sub(MULTIPLE_NEWLINES, "\n\n", text)
                text = text.strip()
                summary = text[0:MAX_CHAR_COUNT_IN_SUMMARY]
                lang = detect(text)
            else:
                title, text, summary, lang, author_list = download_url(url, sleep_a_bit)
                authors = ", ".join(author_list or [])
            language = Language.find(lang)
            url_object = Url.find_or_create(session, url)
            new_article = Article(
                url_object,
                title,
                authors,
                text,  
                summary,
                datetime.now(),
                None,
                language,
                htmlContent,
            )
            session.add(new_article)
            session.commit()
            return new_article
        except sqlalchemy.exc.IntegrityError or sqlalchemy.exc.DatabaseError:
            for i in range(10):
                try:
                    session.rollback()
                    u = cls.find(url)
                    print("Found article by url after recovering from race")
                    return u
                except:
                    print("Exception of second degree in article..." + str(i))
                    time.sleep(0.3)
                    continue
                break
def find_or_create ( cls , session , _url : str , language = None , sleep_a_bit = False , htmlContent = None , title = None , authors : str = "" , ) : from zeeguu . core . model import Url , Article , Language url = Url . extract_canonical_url ( _url ) try : found = cls . find ( url ) if found : return found if htmlContent : text = re . sub ( HTML_TAG_CLEANR , "" , htmlContent ) text = re . sub ( MULTIPLE_NEWLINES , "\n\n" , text ) text = text . strip ( ) summary = text [ 0 : MAX_CHAR_COUNT_IN_SUMMARY ] lang = detect ( text ) else : title , text , summary , lang , author_list = download_url ( url , sleep_a_bit ) authors = ", " . join ( author_list or [ ] ) language = Language . find ( lang ) url_object = Url . find_or_create ( session , url ) new_article = Article ( url_object , title , authors , text , summary , datetime . now ( ) , None , language , htmlContent , ) session . add ( new_article ) session . commit ( ) return new_article except sqlalchemy . exc . IntegrityError or sqlalchemy . exc . DatabaseError : for i in range ( 10 ) : try : session . rollback ( ) u = cls . find ( url ) print ( "Found article by url after recovering from race" ) return u except : print ( "Exception of second degree in article..." + str ( i ) ) time . sleep ( 0.3 ) continue break
If article for url found, return ID
If not found,
If article for url found return ID If not found
"""

            If article for url found, return ID

            If not found,

                - if htmlContent is present, create article for that
                - if not, download and create article then return

        :param url:
        :return:
        """ # replace many newlines with max two; in some # cases many newlines are left after stripping the html tags # Create new article and save it to DB # any article longer than this will be truncated...
None
cls
session
_url
str
language
sleep_a_bit
htmlContent
title
authors
str
url
None