lookyloo/lookyloo.py
def process_capture_queue(self) -> Union[bool, None]:
        '''Process a query from the capture queue'''
        if not self.redis.exists('to_capture'):
            return None

        status, message = self.splash_status()
        if not status:
            self.logger.critical(f'Splash is not running, unable to process the capture queue: {message}')
            return None

        value = self.redis.zpopmax('to_capture')
        if not value or not value[0]:
            return None
        uuid, score = value[0]
        queue: str = self.redis.get(f'{uuid}_mgmt')  # type: ignore
        self.redis.sadd('ongoing', uuid)

        lazy_cleanup = self.redis.pipeline()
        lazy_cleanup.delete(f'{uuid}_mgmt')
        lazy_cleanup.zincrby('queues', -1, queue)

        to_capture: Dict[str, Union[str, int, float]] = self.redis.hgetall(uuid)
        to_capture['perma_uuid'] = uuid
        if 'cookies' in to_capture:
            to_capture['cookies_pseudofile'] = to_capture.pop('cookies')

        status = self._capture(**to_capture)  # type: ignore
        lazy_cleanup.srem('ongoing', uuid)
        lazy_cleanup.delete(uuid)
        # make sure to expire the key if nothing was process for a while (= queues empty)
        lazy_cleanup.expire('queues', 600)
        lazy_cleanup.execute()
        if status:
            self.logger.info(f'Processed {to_capture["url"]}')
            return True
        self.logger.warning(f'Unable to capture {to_capture["url"]}')
        return False
Process a query from the capture queue
Process a query from the capture queue
Process a query from the capture queue
def process_capture_queue(self) -> Union[bool, None]:
        if not self.redis.exists('to_capture'):
            return None
        status, message = self.splash_status()
        if not status:
            self.logger.critical(f'Splash is not running, unable to process the capture queue: {message}')
            return None
        value = self.redis.zpopmax('to_capture')
        if not value or not value[0]:
            return None
        uuid, score = value[0]
        queue: str = self.redis.get(f'{uuid}_mgmt')  
        self.redis.sadd('ongoing', uuid)
        lazy_cleanup = self.redis.pipeline()
        lazy_cleanup.delete(f'{uuid}_mgmt')
        lazy_cleanup.zincrby('queues', -1, queue)
        to_capture: Dict[str, Union[str, int, float]] = self.redis.hgetall(uuid)
        to_capture['perma_uuid'] = uuid
        if 'cookies' in to_capture:
            to_capture['cookies_pseudofile'] = to_capture.pop('cookies')
        status = self._capture(**to_capture)  
        lazy_cleanup.srem('ongoing', uuid)
        lazy_cleanup.delete(uuid)
        lazy_cleanup.expire('queues', 600)
        lazy_cleanup.execute()
        if status:
            self.logger.info(f'Processed {to_capture["url"]}')
            return True
        self.logger.warning(f'Unable to capture {to_capture["url"]}')
        return False
def process_capture_queue ( self ) -> Union [ bool , None ] : if not self . redis . exists ( 'to_capture' ) : return None status , message = self . splash_status ( ) if not status : self . logger . critical ( f'Splash is not running, unable to process the capture queue: {message}' ) return None value = self . redis . zpopmax ( 'to_capture' ) if not value or not value [ 0 ] : return None uuid , score = value [ 0 ] queue : str = self . redis . get ( f'{uuid}_mgmt' ) self . redis . sadd ( 'ongoing' , uuid ) lazy_cleanup = self . redis . pipeline ( ) lazy_cleanup . delete ( f'{uuid}_mgmt' ) lazy_cleanup . zincrby ( 'queues' , - 1 , queue ) to_capture : Dict [ str , Union [ str , int , float ] ] = self . redis . hgetall ( uuid ) to_capture [ 'perma_uuid' ] = uuid if 'cookies' in to_capture : to_capture [ 'cookies_pseudofile' ] = to_capture . pop ( 'cookies' ) status = self . _capture ( ** to_capture ) lazy_cleanup . srem ( 'ongoing' , uuid ) lazy_cleanup . delete ( uuid ) lazy_cleanup . expire ( 'queues' , 600 ) lazy_cleanup . execute ( ) if status : self . logger . info ( f'Processed {to_capture["url"]}' ) return True self . logger . warning ( f'Unable to capture {to_capture["url"]}' ) return False
Process a query from the capture queue
Process a query from the capture queue
'''Process a query from the capture queue''' # type: ignore # type: ignore # make sure to expire the key if nothing was process for a while (= queues empty)
self