This code is meant to download images from an image board to create a local backup. Images are grouped by tags selected by the user and then downloaded using multiprocessing. Normal use of this program would be daily or weekly, resulting in a typical download queue of no more than 100. However, if the user opts to download images from all time which match their query, the queue can grow up to 100,000. The way I have it written now uses tuples. This is not a problem in some cases, but it is extremely slow parse and can cause crashes on machines with smaller memory pools. In addition to that major flaw, it is one of my first python programs, so I expect it to be messy and/or hacky in a few other places.It is a fairly large project, which has only grown larger and messier as I think of new features for it, but I trust the stack exchange community a lot when it comes to issues I have. It might even be a ton larger than necessary for what it tries to accomplish.Main#!/usr/bin/env pythonimport loggingimport osimport sysfrom multiprocessing import freeze_supportfrom collections import namedtuplefrom lib import constants, support, api, downloaderif __name__ == '__main__':    freeze_support()    logging.basicConfig(level = support.get_verbosity(), format = constants.LOGGER_FORMAT,        stream = sys.stderr)    LOG = logging.getLogger('program')    LOG.info('Running program version ' + constants.VERSION + '.')    CONFIG = support.get_config('config.ini')    early_terminate = False    early_terminate |= not downloader.internet_connected()    early_terminate |= support.validate_tags(CONFIG)    if early_terminate:        LOG.info('Error(s) occurred during initialization, see above for more information.')        sys.exit(-1)    GROUP = namedtuple('Group', 'tags directory')    blacklist = []    tag_groups = []    LOG.info('Parsing config for blacklist and settings.')    for section in CONFIG.sections():        if section == 'Settings':            pass        elif section == 'Blacklist':            for tag in CONFIG.get('Blacklist', 'tags').replace(',', '').split():                blacklist.append(api.get_alias(tag))        else:            for option, value in CONFIG.items(section):                if option == 'tags':                    tag_groups.append(GROUP(value.replace(',', ''), section))    LOG.info('program will look for new posts since ' +        CONFIG.get('Settings', 'last_run') + '.')    print ''    download_list = []    for group in tag_groups:        LOG.info('Checking for new posts tagged: \' + group.tags.replace(' ', ', ') + '\.')        accumulating = True        current_page = 1        links_missing_tags = 0        links_blacklisted = 0        links_on_disk = 0        will_download = 0        post_list = []        tag_overflow = []        separated_tags = group.tags.split()        if len(separated_tags) > 5:            search_tags = ' '.join(separated_tags[0:5])            for tag in separated_tags:                if tag not in search_tags.split():                    tag_overflow.append(api.get_alias(tag))        else:            search_tags = group.tags        while accumulating:            links_found = api.get_posts(search_tags, CONFIG.get('Settings', 'last_run'),            current_page, constants.MAX_RESULTS)            if not links_found:                accumulating = False            else:                post_list += links_found                accumulating = len(links_found) == constants.MAX_RESULTS                current_page += 1        if len(post_list) > 0:            for i, post in enumerate(post_list):                LOG.debug('Item ' + str(i) + '\'s id is \' + str(post.id) + '\.')                filename = support.make_filename(group.directory, post)                current_tags = post.tags.split()                if len(separated_tags) > 5 and not list(set(tag_overflow) & set(current_tags)):                    links_missing_tags += 1                    LOG.debug('Item ' + str(i) + ' was skipped. Missing a requested tag.')                elif list(set(blacklist) & set(current_tags)):                    links_blacklisted += 1                    LOG.debug('Item ' + str(i) + ' was skipped. Contains a blacklisted tag.')                elif os.path.isfile(filename):                    links_on_disk += 1                    LOG.debug('Item ' + str(i) + ' was skipped. Already downloaded previously.')                else:                    LOG.debug('Item ' + str(i) + ' will be downloaded.')                    download_list.append((post.url, filename))                    will_download += 1            LOG.info(str(will_download) + ' new files. (' + str(len(post_list)) + ' found, ' +            str(links_missing_tags) + ' missing tags, ' + str(links_blacklisted) +            ' blacklisted, ' + str(links_on_disk) + ' duplicate.)')            print ''        else:            LOG.info('0 new files.')            print ''    if download_list:        LOG.info('Starting download of ' + str(len(download_list)) + ' files.')        downloader.multi_download(download_list, CONFIG.getint('Settings',            'parallel_downloads'))        print ''        LOG.info('Successfully downloaded ' + str(len(download_list)) + ' files.')    else:        LOG.info('Nothing to download.')    CONFIG.set('Settings', 'last_run', constants.YESTERDAY.strftime(constants.DATE_FORMAT))    CONFIG.write(open('config.ini', 'w'))    sys.exit(0)Constants#!/usr/bin/env pythonimport datetimeDATE_FORMAT = %Y-%m-%dYESTERDAY = datetime.date.fromordinal(datetime.date.today().toordinal() - 1)LOGGER_FORMAT = %(name)-11s %(levelname)-8s %(message)sMAX_RESULTS = 100VERSION = '3.0.2 -- Forked from 2.4.6'DEFAULT_CONFIG_TEXT = ''';;;;;;;;;;;;;;;;;;;;; MAIN SETTINGS ;;;;;;;;;;;;;;;;;;;;;[Settings]last_run = ''' + YESTERDAY.strftime(DATE_FORMAT) + '''parallel_downloads = 8[Blacklist]tags =;;;;;;;;;;;;;;;;;; TAG GROUPS ;;;;;;;;;;;;;;;;;;; New tag groups can be created by writing the following:; [Directory Name]; tags = tag1, tag2, tag3, ...;; Example:; [Cute Cats]; tags = cat, cute'''Support#!/usr/bin/env pythonimport argparseimport loggingimport osimport ConfigParserfrom urllib import FancyURLopenerimport constantsclass SpoofOpen(FancyURLopener):    version = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) ' + \        'Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12'def get_verbosity():    parser = argparse.ArgumentParser(prog = 'program', description = 'An automated image downloader.')    verbosity = parser.add_mutually_exclusive_group(required = False)    verbosity.add_argument('-v', '--verbose', action = 'store_true', help = 'Display full debug \        information while running.')    verbosity.add_argument('-q', '--quiet', action = 'store_true', help = 'Display no output while \        running, except for errors.')    args = parser.parse_args()    if args.quiet:        return logging.ERROR    elif args.verbose:        return logging.DEBUG    else:        return logging.INFOdef make_config(filename):    LOG = logging.getLogger('config')    with open(filename, 'w') as outfile:        outfile.write(constants.DEFAULT_CONFIG_TEXT)        LOG.info('New default file created: \' + filename + '\.')def get_config(filename):    LOG = logging.getLogger('config')    config = ConfigParser.ConfigParser()    if not os.path.isfile(filename):        LOG.error('No config file found.')        make_config(filename)    with open(filename, 'r') as infile:        config.readfp(infile)        return configdef validate_tags(config):    LOG = logging.getLogger('tags')    sections = 0    for _ in config.sections():        sections += 1    if sections < 3:        LOG.error('Please add at least one tag group to \config.ini\.')        return True    else:        return Falsedef substitute_illegals(char):    illegals = ['\\', '/', ':', '*', '?', '\', '<', '>', '|', ' ']    return '_' if char in illegals else chardef make_filename(directory_name, post):    safe_directory = ''.join([substitute_illegals(char) for char in directory_name]).lower()    name = str(getattr(post, 'id'))    if not os.path.isdir('downloads/' + safe_directory.decode('utf-8')):        os.makedirs('downloads/' + safe_directory)    filename = 'downloads/' + safe_directory + '/' + name + '.' + post.ext    return filenameAPI#!/usr/bin/env pythonimport loggingfrom json import loadsfrom collections import namedtuplefrom support import SpoofOpenPost = namedtuple('Post', 'url id ext tags')UserTag = namedtuple('UserTag', 'alias_id name')AliasedTag = namedtuple('AliasedTag', 'name')LOG = logging.getLogger('api')def get_posts(search_string, uploaded_after, page_number, max_results):    request = 'https://x.json?' + \        'tags=' + search_string + \        ' date:>' + str(uploaded_after) + \        '&page=' + str(page_number) + \        '&limit=' + str(max_results)    LOG.debug('Post request URL: \' + request + '\.')    results = loads(SpoofOpen().open(request).read().decode())    posts = []    for post in results:        posts.append(Post(post['file_url'], post['id'], post['file_ext'], post['tags']))    return postsdef download_post(url, filename):    with open(filename, 'wb') as outfile:        outfile.write(SpoofOpen().open(url).read())def get_alias(tag):    request = 'https://x.json?query=' + tag    LOG.debug('Tag alias request URL: \' + request + '\.')    results = loads(SpoofOpen().open(request).read().decode())    user_tags = []    for user_tag in results:        user_tags.append(UserTag(user_tag['alias_id'], user_tag['name']))    if not user_tags:        LOG.error('The tag \' + tag + '\ does not exist, please remove it from your tags ' +            'file or blacklist.')        return ''    if tag == user_tags[0].name:        request = 'https://x.json?id=' + str(user_tags[0].alias_id)        LOG.debug('Tag official request URL: \' + request + '\.')        results = loads('[' + SpoofOpen().open(request).read() + ']'.decode())        aliased_tags = []        for aliased_tag in results:            aliased_tags.append(AliasedTag(aliased_tag['name']))        LOG.debug('Tag \' + tag + '\ aliased to \' + aliased_tags[0].name + '\.')        return aliased_tags[0].name    else:        return tagDownloader#!/usr/bin/env pythonimport loggingimport urllib2from itertools import repeatfrom multiprocessing import Pool, Manager, Processfrom time import sleepfrom support import SpoofOpendef internet_connected():    try:        urllib2.urlopen('http://www.msftncsi.com/ncsi.txt', timeout = 5)        return True    except urllib2.URLError:        pass    LOG = logging.getLogger('internet')    LOG.info('No internet connection detected.')    return Falsedef update_progress(downloaded, total):    progress = float(downloaded) / float(total)    BAR_LENGTH = 36    status = ''    if isinstance(progress, int):        progress = float(progress)    if progress < 0:        progress = 0.0        status = ' -- Stopped.\n'    if progress >= 1:        progress = 1.0        status = ' -- Done.\n'    completed = int(round(BAR_LENGTH * progress))    progress_bar = '\rDownloading          [{}] {:6.2f}% {} {}'.format('>' * completed +        ' ' * (BAR_LENGTH - completed), progress * 100, '(' + str(downloaded) + ' / ' + str(total) +        ')', status)    print progress_bar,def download_monitor(managed_list, total_items):    while True:        update_progress(len(managed_list), total_items)        if total_items == len(managed_list):            return        sleep(0.2)def single_download(zipped_args):    url_name_list, managed_list = zipped_args    url, filename = url_name_list    spoof = SpoofOpen()    try:        with open(filename, 'wb') as dest:            source = spoof.open(url)            dest.write(source.read())        LOG = logging.getLogger('single_dl')        LOG.debug('Downloading \' + filename + '\.')        managed_list.append(filename)    except KeyboardInterrupt:        passdef multi_download(url_name_list, num_threads):    manager = Manager()    managed_list = manager.list()    LOG = logging.getLogger('multi_dl')    LOG.debug('Staring download pool of ' + str(num_threads) + ' workers.')    monitor = Process(target = download_monitor, args = (managed_list, len(url_name_list)))    monitor.start()    workers = Pool(processes = num_threads)    work = workers.map_async(single_download, zip(url_name_list, repeat(managed_list)))    try:        work.get(0xFFFF)        monitor.join()    except KeyboardInterrupt:        exit()
Download images from an image board based on tags and blacklists
python;performance;python 2.7;json;network file transfer