tap_feed/streams.py
def parse_response(self, response: requests.Response) -> Iterable[dict]:
        """Parse the response and return an iterator of result rows."""
        feed = feedparser.parse(response.text)
        rep_key = FeedStream.replication_key[6:]
        entries = sorted(
            feed["entries"], key=lambda d: datetime_parser.parse(d[rep_key])
        )
        feed_url = response.url
        bookmark_timestamp = self.get_starting_timestamp({"feed_url": feed_url})
        for entry in entries:
            published_str = entry.get(rep_key)
            if published_str is None:
                continue
            published_date = datetime_parser.parse(published_str)
            if bookmark_timestamp is None or published_date > bookmark_timestamp:
                record = {"feed_url": feed_url}
                record.update(
                    {
                        f"feed_{name}": feed["feed"].get(name)
                        for name in self.config["feed_fields"]
                    }
                )
                record.update(
                    {
                        "entry_id": entry["id"],
                        FeedStream.replication_key: str(published_date),
                    }
                )
                record.update(
                    {
                        f"entry_{name}": entry.get(name)
                        for name in self.config["feed_entry_fields"]
                        if name not in ["id", "published", "updated"]
                    }
                )
                yield record
Parse the response and return an iterator of result rows.
Parse the response and return an iterator of result rows.
Parse the response and return an iterator of result rows .
def parse_response(self, response: requests.Response) -> Iterable[dict]:
        feed = feedparser.parse(response.text)
        rep_key = FeedStream.replication_key[6:]
        entries = sorted(
            feed["entries"], key=lambda d: datetime_parser.parse(d[rep_key])
        )
        feed_url = response.url
        bookmark_timestamp = self.get_starting_timestamp({"feed_url": feed_url})
        for entry in entries:
            published_str = entry.get(rep_key)
            if published_str is None:
                continue
            published_date = datetime_parser.parse(published_str)
            if bookmark_timestamp is None or published_date > bookmark_timestamp:
                record = {"feed_url": feed_url}
                record.update(
                    {
                        f"feed_{name}": feed["feed"].get(name)
                        for name in self.config["feed_fields"]
                    }
                )
                record.update(
                    {
                        "entry_id": entry["id"],
                        FeedStream.replication_key: str(published_date),
                    }
                )
                record.update(
                    {
                        f"entry_{name}": entry.get(name)
                        for name in self.config["feed_entry_fields"]
                        if name not in ["id", "published", "updated"]
                    }
                )
                yield record
def parse_response ( self , response : requests . Response ) -> Iterable [ dict ] : feed = feedparser . parse ( response . text ) rep_key = FeedStream . replication_key [ 6 : ] entries = sorted ( feed [ "entries" ] , key = lambda d : datetime_parser . parse ( d [ rep_key ] ) ) feed_url = response . url bookmark_timestamp = self . get_starting_timestamp ( { "feed_url" : feed_url } ) for entry in entries : published_str = entry . get ( rep_key ) if published_str is None : continue published_date = datetime_parser . parse ( published_str ) if bookmark_timestamp is None or published_date > bookmark_timestamp : record = { "feed_url" : feed_url } record . update ( { f"feed_{name}" : feed [ "feed" ] . get ( name ) for name in self . config [ "feed_fields" ] } ) record . update ( { "entry_id" : entry [ "id" ] , FeedStream . replication_key : str ( published_date ) , } ) record . update ( { f"entry_{name}" : entry . get ( name ) for name in self . config [ "feed_entry_fields" ] if name not in [ "id" , "published" , "updated" ] } ) yield record
Parse the response and return an iterator of result rows.
Parse the response and return an iterator of result rows .
"""Parse the response and return an iterator of result rows."""
self
response
requests.Response