lookyloo/lookyloo.py
def build_ua_file(self) -> None:
        '''Build a file in a format compatible with the capture page'''
        yesterday = (date.today() - timedelta(days=1))
        self_generated_ua_file_path = get_homedir() / 'own_user_agents' / str(yesterday.year) / f'{yesterday.month:02}'
        safe_create_dir(self_generated_ua_file_path)
        self_generated_ua_file = self_generated_ua_file_path / f'{yesterday.isoformat()}.json'
        if self_generated_ua_file.exists():
            return
        entries = self.redis.zrevrange(f'user_agents|{yesterday.isoformat()}', 0, -1)
        if not entries:
            return

        to_store: Dict[str, Any] = {'by_frequency': []}
        uas = Counter([entry.split('|', 1)[1] for entry in entries])
        for ua, _ in uas.most_common():
            parsed_ua = UserAgent(ua)
            if not parsed_ua.platform or not parsed_ua.browser:
                continue
            if parsed_ua.platform not in to_store:
                to_store[parsed_ua.platform] = {}
            if f'{parsed_ua.browser} {parsed_ua.version}' not in to_store[parsed_ua.platform]:
                to_store[parsed_ua.platform][f'{parsed_ua.browser} {parsed_ua.version}'] = []
            to_store[parsed_ua.platform][f'{parsed_ua.browser} {parsed_ua.version}'].append(parsed_ua.string)
            to_store['by_frequency'].append({'os': parsed_ua.platform,
                                             'browser': f'{parsed_ua.browser} {parsed_ua.version}',
                                             'useragent': parsed_ua.string})
        with self_generated_ua_file.open('w') as f:
            json.dump(to_store, f, indent=2)

        # Remove the UA / IP mapping.
        self.redis.delete(f'user_agents|{yesterday.isoformat()}')
Build a file in a format compatible with the capture page
Build a file in a format compatible with the capture page
Build a file in a format compatible with the capture page
def build_ua_file(self) -> None:
        yesterday = (date.today() - timedelta(days=1))
        self_generated_ua_file_path = get_homedir() / 'own_user_agents' / str(yesterday.year) / f'{yesterday.month:02}'
        safe_create_dir(self_generated_ua_file_path)
        self_generated_ua_file = self_generated_ua_file_path / f'{yesterday.isoformat()}.json'
        if self_generated_ua_file.exists():
            return
        entries = self.redis.zrevrange(f'user_agents|{yesterday.isoformat()}', 0, -1)
        if not entries:
            return
        to_store: Dict[str, Any] = {'by_frequency': []}
        uas = Counter([entry.split('|', 1)[1] for entry in entries])
        for ua, _ in uas.most_common():
            parsed_ua = UserAgent(ua)
            if not parsed_ua.platform or not parsed_ua.browser:
                continue
            if parsed_ua.platform not in to_store:
                to_store[parsed_ua.platform] = {}
            if f'{parsed_ua.browser} {parsed_ua.version}' not in to_store[parsed_ua.platform]:
                to_store[parsed_ua.platform][f'{parsed_ua.browser} {parsed_ua.version}'] = []
            to_store[parsed_ua.platform][f'{parsed_ua.browser} {parsed_ua.version}'].append(parsed_ua.string)
            to_store['by_frequency'].append({'os': parsed_ua.platform,
                                             'browser': f'{parsed_ua.browser} {parsed_ua.version}',
                                             'useragent': parsed_ua.string})
        with self_generated_ua_file.open('w') as f:
            json.dump(to_store, f, indent=2)
        self.redis.delete(f'user_agents|{yesterday.isoformat()}')
def build_ua_file ( self ) -> None : yesterday = ( date . today ( ) - timedelta ( days = 1 ) ) self_generated_ua_file_path = get_homedir ( ) / 'own_user_agents' / str ( yesterday . year ) / f'{yesterday.month:02}' safe_create_dir ( self_generated_ua_file_path ) self_generated_ua_file = self_generated_ua_file_path / f'{yesterday.isoformat()}.json' if self_generated_ua_file . exists ( ) : return entries = self . redis . zrevrange ( f'user_agents|{yesterday.isoformat()}' , 0 , - 1 ) if not entries : return to_store : Dict [ str , Any ] = { 'by_frequency' : [ ] } uas = Counter ( [ entry . split ( '|' , 1 ) [ 1 ] for entry in entries ] ) for ua , _ in uas . most_common ( ) : parsed_ua = UserAgent ( ua ) if not parsed_ua . platform or not parsed_ua . browser : continue if parsed_ua . platform not in to_store : to_store [ parsed_ua . platform ] = { } if f'{parsed_ua.browser} {parsed_ua.version}' not in to_store [ parsed_ua . platform ] : to_store [ parsed_ua . platform ] [ f'{parsed_ua.browser} {parsed_ua.version}' ] = [ ] to_store [ parsed_ua . platform ] [ f'{parsed_ua.browser} {parsed_ua.version}' ] . append ( parsed_ua . string ) to_store [ 'by_frequency' ] . append ( { 'os' : parsed_ua . platform , 'browser' : f'{parsed_ua.browser} {parsed_ua.version}' , 'useragent' : parsed_ua . string } ) with self_generated_ua_file . open ( 'w' ) as f : json . dump ( to_store , f , indent = 2 ) self . redis . delete ( f'user_agents|{yesterday.isoformat()}' )
Build a file in a format compatible with the capture page
Build a file in a format compatible with the capture page
'''Build a file in a format compatible with the capture page''' # Remove the UA / IP mapping.
self