# -*- encoding: utf-8 -*-
import gzip
import io
import glob
from concurrent import futures


def find_robots(filename):
    '''
    Find all of the hosts that access robots.txt in a single log file

    '''
    robots = set()
    with gzip.open(filename) as f:
        for line in io.TextIOWrapper(f, encoding='ascii'):
            fields = line.split()
            if fields[6] == '/robots.txt':
                robots.add(fields[0])
    return robots


def find_all_robots(logdir):
    '''
    Find all hosts across and entire sequence of files
    '''
    files = glob.glob(logdir + '/*.log.gz')
    all_robots = set()
    with futures.ProcessPoolExecutor() as pool:
        for robots in pool.map(find_robots, files):
            all_robots.update(robots)
    return all_robots


if __name__ == '__main__':
    robots = find_all_robots('logs')
    for ipaddr in robots:
        print(ipaddr)

[((26, 12, 26, 43), 'glob.glob', 'glob.glob', ({(26, 22, 26, 42): "logdir + '/*.log.gz'"}, {}), "(logdir + '/*.log.gz')", False, 'import glob\n'), ((14, 9, 14, 28), 'gzip.open', 'gzip.open', ({(14, 19, 14, 27): 'filename'}, {}), '(filename)', False, 'import gzip\n'), ((15, 20, 15, 57), 'io.TextIOWrapper', 'io.TextIOWrapper', (), '', False, 'import io\n'), ((28, 9, 28, 38), 'concurrent.futures.ProcessPoolExecutor', 'futures.ProcessPoolExecutor', ({}, {}), '()', False, 'from concurrent import futures\n')]