Fetch StackOverflow questions and save them to a markdown table I'm trying to write a script that would (a) fetch StackOverflow questions and (b) update a markdown table with new entries.
With github actions I automated this script to run daily.
Idea borrowed from simonw
Code:
import re
import requests
from pathlib import Path
from datetime import datetime


URL = &quot;https://api.stackexchange.com/2.2/questions&quot;
DATE = datetime.utcnow().date()
ROOT = Path(__file__).parent.resolve()


def get_epochs(date):
    &quot;&quot;&quot; Get epoch dates for the start and end of the (current) day. &quot;&quot;&quot;

    start = datetime(
        year=date.year, month=date.month, day=date.day,
        hour=0, minute=0, second=0
    )
    end = datetime(
        year=date.year, month=date.month, day=date.day,
        hour=23, minute=59, second=59
    )
    return int(start.timestamp()), int(end.timestamp())


def fetch_questions(start, end, tag, site=&quot;stackoverflow&quot;):
    &quot;&quot;&quot; Fetch questions from stackoverflowAPI. &quot;&quot;&quot;

    _params = {
        &quot;fromdate&quot;: start,
        &quot;todate&quot;: end,
        &quot;order&quot;: &quot;desc&quot;,
        &quot;sort&quot;: &quot;votes&quot;,
        &quot;tagged&quot;: tag,
        &quot;site&quot;: site,
    }
    return requests.get(URL, params=_params).json()


def build_table(*args, **kwargs):
    &quot;&quot;&quot; Build a markdown table from a list of entries. &quot;&quot;&quot;

    columns = [
        &quot;\n&quot;.join(
            &quot;* [{title}]({url}) - {score} votes&quot;.format(
                # to prevent code from breaking if special characters are present
                title=re.sub(r'[^\w\s]', '', item[&quot;title&quot;]),
                url=item[&quot;link&quot;],
                score=item[&quot;score&quot;]
            )
            for item in chunk[&quot;items&quot;][:8]
        )
        for chunk in args
    ]
    return columns


def replace_chunk(content, marker, chunk, inline=False):
    &quot;&quot;&quot; Replace chunks of README.md &quot;&quot;&quot;

    r = re.compile(
        r&quot;&lt;!\-\- {} starts \-\-&gt;.*&lt;!\-\- {} ends \-\-&gt;&quot;.format(marker, marker),
        re.DOTALL,
    )
    if not inline:
        chunk = &quot;\n{}\n&quot;.format(chunk)
    chunk = &quot;&lt;!-- {} starts --&gt;{}&lt;!-- {} ends --&gt;&quot;.format(marker, chunk, marker)
    return r.sub(chunk, content)


if __name__ == &quot;__main__&quot;:
    
    readme = ROOT / &quot;README.md&quot;
    start, end = get_epochs(DATE)

    pandas, beautifulsoup, code_review = build_table(
        fetch_questions(start, end, tag=&quot;pandas&quot;),
        fetch_questions(start, end, tag=&quot;beautifulsoup&quot;),
        fetch_questions(start, end, tag=&quot;python&quot;, site=&quot;codereview&quot;)
    )

    readme_contents = readme.open().read()
    rewritten = replace_chunk(readme_contents, &quot;date&quot;, DATE.strftime(&quot;%Y-%m-%d&quot;), inline=True)
    rewritten = replace_chunk(rewritten, &quot;pandas&quot;, pandas)
    rewritten = replace_chunk(rewritten, &quot;bs&quot;, beautifulsoup)
    rewritten = replace_chunk(rewritten, &quot;code_review&quot;, code_review)

    with open(readme, &quot;w&quot;) as output:
        output.write(rewritten)

Markdown file:
# Stackoverflow daily top questions: &lt;!-- date starts --&gt; date &lt;!-- date ends --&gt;

&lt;table&gt;&lt;tr&gt;&lt;td valign=&quot;top&quot; width=&quot;33%&quot;&gt;

### Pandas
&lt;!-- pandas starts --&gt;
pandas content
&lt;!-- pandas ends --&gt;
&lt;/td&gt;&lt;td valign=&quot;top&quot; width=&quot;34%&quot;&gt;


### BeautifulSoup
&lt;!-- bs starts --&gt;
bs4 content
&lt;!-- bs ends --&gt;
&lt;/td&gt;&lt;td valign=&quot;top&quot; width=&quot;34%&quot;&gt;


### Python code review submissions
&lt;!-- code_review starts --&gt;
code review content
&lt;!-- code_review ends --&gt;
&lt;/td&gt;&lt;td valign=&quot;top&quot; width=&quot;34%&quot;&gt;


Sample output:
Things look good and well organised at first glance. However, there are a few things you can make use of.

Use f-string over using str.format. The former is more performant.
You can group the columns and iterate over them, instead of having multiple lines for each section. This way, you can add/remove to this iterable without having to change much of the code itself.
When using raw strings, you do not need to escape the - character, unless they are being used inside a character set (for regex).
The build_table can be split to another separate function.
Use type hinting.
If you're not using kwargs, no need to declare them.


Rewritten snippets (you might need to fit them in your code accordingly):
sections = (
    {'tag': &quot;pandas&quot;, &quot;marker&quot;: &quot;pandas&quot;},
    {'tag': &quot;beautifulsoup&quot;, &quot;marker&quot;: &quot;bs&quot;},
    {'tag': &quot;python&quot;, &quot;site&quot;: &quot;codereview&quot;, &quot;marker&quot;: &quot;code_review&quot;},
)
for section in sections:
    questions_list = fetch_questions(start, end, **section)
    # Using `**section` might throw an error for the unknown kwarg: `marker`.
    # But it is trivial to handle that.
    .
    ...


def replace_chunk(content: str, marker: str, chunk: str, inline: bool = False):
    &quot;&quot;&quot; Replace chunks of README.md &quot;&quot;&quot;
    r = re.compile(
        rf&quot;&lt;!-- {marker} starts --&gt;.*&lt;!-- {marker} ends --&gt;&quot;,
        re.DOTALL,
    )
    if not inline:
        chunk = &quot;\n{}\n&quot;.format(chunk)
    chunk = f&quot;&lt;!-- {marker} starts --&gt;{chunk}&lt;!-- {marker} ends --&gt;&quot;
    return r.sub(chunk, content)


def get_item_string(item):
    # to prevent code from breaking if special characters are present
    title = re.sub(r'[^\w\s]', '', item[&quot;title&quot;])
    return f&quot;* [{title}]({item['link']}) - {item['score']} votes&quot;

def build_table(*args):
    &quot;&quot;&quot; Build a markdown table from a list of entries. &quot;&quot;&quot;
    columns = [
        &quot;\n&quot;.join(
            map(get_item_string, chunk[&quot;items&quot;][:8])
        )
        for chunk in args
    ]
    return columns