def filterbots(fh, ip_ua_re, bots_ua, bots_ips, 
               parser=None, format=None, ip_ua_fields=None, 
               reverse=False, debug=False, **kwargs):
    """Filter bots from a log stream using
    ip/useragent blacklists"""
    bots_ua_dict, bots_ua_prefix_dict, bots_ua_suffix_dict, bots_ua_re = \
                parse_bots_ua(bots_ua)
    bots_ips = dict.fromkeys([l.strip() for l in bots_ips \
                              if not l.startswith("#")])
    is_bot_ua_func = partial(is_bot_ua, bots_ua_dict=bots_ua_dict, 
                         bots_ua_prefix_dict=bots_ua_prefix_dict, 
                         bots_ua_suffix_dict=bots_ua_suffix_dict, 
                         bots_ua_re=bots_ua_re)    
    
    _is_bot_func=None    
    if not parser:
        # Regular expression-based matching
        ua_ip_re = re.compile(ip_ua_re)
        
        def _is_bot_func(line):
            match = ua_ip_re.match(line)
            if not match:
                raise ValueError("No match for line: %s" % line)
            logging.debug("Regular expression matched line: %s", match)
    
            matchgroups = match.groupdict()
            is_bot = False
    
            ua = matchgroups.get('ua', None)
            is_bot = is_bot_ua_func(ua)        
    
            if not is_bot and matchgroups.get('ip', None) in bots_ips:
                # IP Is blacklisted
                is_bot = True
                
            return is_bot
                
    else:
        # Custom parser specified, use field-based matching
        parser = eval(parser, vars(logtools.parsers), {})()
        try:
            fields_map = dict([tuple(k.split(':')) for k in ip_ua_fields.split(',')])
        except __HOLE__:
            raise ValueError("Invalid format for --field parameter. Use --help for usage instructions.")
        is_indices = reduce(and_, (k.isdigit() for k in fields_map.values()), True)
        if is_indices:
            # Field index based matching
            def _is_bot_func(line):
                parsed_line = parser(line)
                is_bot = False
                if 'ua' in fields_map and parsed_line:
                    is_bot = is_bot_ua_func(parsed_line.by_index(fields_map['ua']))
                if not is_bot and 'ip' in fields_map:
                    is_bot = parsed_line.by_index(fields_map['ip']) in bots_ips
                return is_bot
        else:
            # Named field based matching
            def _is_bot_func(line):
                parsed_line = parser(line)
                is_bot = False
                if 'ua' in fields_map and parsed_line:
                    is_bot = is_bot_ua_func(parsed_line[fields_map['ua']])
                if not is_bot and 'ip' in fields_map:
                    is_bot = parsed_line[fields_map['ip']] in bots_ips
                return is_bot            
        
    num_lines=0
    num_filtered=0
    num_nomatch=0
    for line in imap(lambda x: x.strip(), fh):
        try:
            is_bot = _is_bot_func(line)
        except (KeyError, ValueError):
            # Parsing error
            logging.warn("No match for line: %s", line)
            num_nomatch +=1
            continue

        if is_bot ^ reverse:
            logging.debug("Filtering line: %s", line)
            num_filtered+=1
            continue

        num_lines+=1
        yield line

    logging.info("Number of lines after bot filtering: %s", num_lines)
    logging.info("Number of lines (bots) filtered: %s", num_filtered)        
    if num_nomatch:
        logging.info("Number of lines could not match on: %s", num_nomatch)

    return
dataset/ETHPy150Open adamhadani/logtools/logtools/_filterbots.py/filterbots