Scraping columns from SEDE results I use the following script to download the result of a SEDE query and scrape a specific column from it using BeautifulSoup:

import json


def extract_column(soup, colname):
    """
    Returns a generator of cell values in selected column.

    For simple columns like timestamp, a cell value can be simple,
    for example: 1414433013197

    For more complex columns like Post Link, a cell value can be an object,
    for example:

      {
        "id": 68102,
        "title": "Bash Script - File Comment out &amp; Notate"
      }

    :param soup: a bs4 (BeautifulSoup) object
    :param colname: name of the SEDE column to extract
    :return: generator of cell values in selected column
    """
    def get_column_index():
        for index, info in enumerate(columns):
            if info['name'] == colname:
                return index
        return -1

    for script in soup.findAll('script'):
        result_sets_col = 'resultSets'
        if result_sets_col in script.text:
            start = script.text.rindex('{', 0, script.text.index(result_sets_col))
            end = script.text.index('}', script.text.index('querySetId')) + 1
            data = json.loads(script.text[start:end])

            results = data[result_sets_col][0]
            columns = results['columns']
            rows = results['rows']

            column_index = get_column_index()
            if column_index &gt; -1:
                for row in rows:
                    yield row[column_index]


Unit tests:

import os
import unittest

from bs4 import BeautifulSoup
from robosanta.stackexchange.sede import extract_column

BASE_DIR = os.path.dirname(__file__)

SEDE_OUTPUT_HTML = os.path.join(BASE_DIR, 'sede-output.html')
POST_ID_COLUMN = 'Post Link'
DATE_COLUMN = 'CreationDate'
ROW_COUNT = 49


def new_soup():
    with open(SEDE_OUTPUT_HTML) as fh:
        return BeautifulSoup(fh)


class TestGetColumn(unittest.TestCase):
    def extract_column(self, colname):
        return extract_column(new_soup(), colname)

    def extract_post_link(self):
        return self.extract_column(POST_ID_COLUMN)

    def extract_date(self):
        return self.extract_column(DATE_COLUMN)

    def test_row_count_of_post_link(self):
        self.assertEqual(ROW_COUNT, len(list(self.extract_post_link())))

    def test_first_post_link(self):
        expected = {
            'title': 'TinyMVC Model / Plugin how to implement?',
            'id': 12144,
        }
        self.assertEqual(expected, next(self.extract_post_link()))

    def test_row_count_of_date(self):
        self.assertEqual(ROW_COUNT, len(list(self.extract_date())))

    def test_first_date(self):
        self.assertEqual(1338304381360, next(self.extract_date()))

    def test_nonexistent_column(self):
        with self.assertRaises(StopIteration):
            next(self.extract_column('nonexistent'))


if __name__ == '__main__':
    unittest.main()


You can find the content of sede-output.html (the test data) here.

I'm looking for any and all suggestions, especially concerning:


Improving the main implementation (the scraping)
Improving the way to test this
I'd test if not result_sets_col in script.text and continue in that case, to remove a level of nesting:

for script in soup.findAll('script'):
    result_sets_col = 'resultSets'
    if not result_sets_col in script.text:
        continue

    start = script.text.rindex('{', 0, script.text.index(result_sets_col))