lookyloo/lookyloo.py
def _cache_capture(self, capture_uuid: str, /) -> CrawledTree:
        '''Generate the pickle, set the cache, add capture in the indexes'''
        capture_dir = self._get_capture_dir(capture_uuid)
        har_files = sorted(capture_dir.glob('*.har'))
        lock_file = capture_dir / 'lock'
        pickle_file = capture_dir / 'tree.pickle'

        if try_make_file(lock_file):
            # Lock created, we can process
            with lock_file.open('w') as f:
                f.write(datetime.now().isoformat())
        else:
            # The pickle is being created somewhere else, wait until it's done.
            while lock_file.exists():
                time.sleep(5)
            keep_going = 5
            while (ct := load_pickle_tree(capture_dir)) is None:
                keep_going -= 1
                if not keep_going:
                    raise LookylooException(f'Unable to get tree for {capture_uuid}')
                time.sleep(5)
            return ct

        # NOTE: We only index the public captures
        index = True
        try:
            ct = CrawledTree(har_files, capture_uuid)
            self._ensure_meta(capture_dir, ct)
            self._resolve_dns(ct)
            self.context.contextualize_tree(ct)
            # Force update cache of the capture (takes care of the incomplete redirect key)
            self._set_capture_cache(capture_dir, force=True)
            cache = self.capture_cache(capture_uuid)
            if not cache:
                raise LookylooException(f'Broken cache for {capture_dir}')
            if self.is_public_instance:
                if cache.no_index:
                    index = False
            if index:
                self.indexing.index_cookies_capture(ct)
                self.indexing.index_body_hashes_capture(ct)
                self.indexing.index_url_capture(ct)
                categories = list(self.categories_capture(capture_uuid).keys())
                self.indexing.index_categories_capture(capture_uuid, categories)
        except Har2TreeError as e:
            raise NoValidHarFile(e)
        except RecursionError as e:
            raise NoValidHarFile(f'Tree too deep, probably a recursive refresh: {e}.\n Append /export to the URL to get the files.')
        else:
            with pickle_file.open('wb') as _p:
                # Some pickles require a pretty high recursion limit, this kindof fixes it.
                # If the capture is really broken (generally a refresh to self), the capture
                # is discarded in the RecursionError above.
                default_recursion_limit = sys.getrecursionlimit()
                sys.setrecursionlimit(int(default_recursion_limit * 1.1))
                pickle.dump(ct, _p)
                sys.setrecursionlimit(default_recursion_limit)
        finally:
            lock_file.unlink(missing_ok=True)
        return ct
Generate the pickle, set the cache, add capture in the indexes
Generate the pickle, set the cache, add capture in the indexes
Generate the pickle set the cache add capture in the indexes
def _cache_capture(self, capture_uuid: str, /) -> CrawledTree:
        capture_dir = self._get_capture_dir(capture_uuid)
        har_files = sorted(capture_dir.glob('*.har'))
        lock_file = capture_dir / 'lock'
        pickle_file = capture_dir / 'tree.pickle'
        if try_make_file(lock_file):
            with lock_file.open('w') as f:
                f.write(datetime.now().isoformat())
        else:
            while lock_file.exists():
                time.sleep(5)
            keep_going = 5
            while (ct := load_pickle_tree(capture_dir)) is None:
                keep_going -= 1
                if not keep_going:
                    raise LookylooException(f'Unable to get tree for {capture_uuid}')
                time.sleep(5)
            return ct
        index = True
        try:
            ct = CrawledTree(har_files, capture_uuid)
            self._ensure_meta(capture_dir, ct)
            self._resolve_dns(ct)
            self.context.contextualize_tree(ct)
            self._set_capture_cache(capture_dir, force=True)
            cache = self.capture_cache(capture_uuid)
            if not cache:
                raise LookylooException(f'Broken cache for {capture_dir}')
            if self.is_public_instance:
                if cache.no_index:
                    index = False
            if index:
                self.indexing.index_cookies_capture(ct)
                self.indexing.index_body_hashes_capture(ct)
                self.indexing.index_url_capture(ct)
                categories = list(self.categories_capture(capture_uuid).keys())
                self.indexing.index_categories_capture(capture_uuid, categories)
        except Har2TreeError as e:
            raise NoValidHarFile(e)
        except RecursionError as e:
            raise NoValidHarFile(f'Tree too deep, probably a recursive refresh: {e}.\n Append /export to the URL to get the files.')
        else:
            with pickle_file.open('wb') as _p:
                default_recursion_limit = sys.getrecursionlimit()
                sys.setrecursionlimit(int(default_recursion_limit * 1.1))
                pickle.dump(ct, _p)
                sys.setrecursionlimit(default_recursion_limit)
        finally:
            lock_file.unlink(missing_ok=True)
        return ct
def _cache_capture ( self , capture_uuid : str , / ) -> CrawledTree : capture_dir = self . _get_capture_dir ( capture_uuid ) har_files = sorted ( capture_dir . glob ( '*.har' ) ) lock_file = capture_dir / 'lock' pickle_file = capture_dir / 'tree.pickle' if try_make_file ( lock_file ) : with lock_file . open ( 'w' ) as f : f . write ( datetime . now ( ) . isoformat ( ) ) else : while lock_file . exists ( ) : time . sleep ( 5 ) keep_going = 5 while ( ct := load_pickle_tree ( capture_dir ) ) is None : keep_going -= 1 if not keep_going : raise LookylooException ( f'Unable to get tree for {capture_uuid}' ) time . sleep ( 5 ) return ct index = True try : ct = CrawledTree ( har_files , capture_uuid ) self . _ensure_meta ( capture_dir , ct ) self . _resolve_dns ( ct ) self . context . contextualize_tree ( ct ) self . _set_capture_cache ( capture_dir , force = True ) cache = self . capture_cache ( capture_uuid ) if not cache : raise LookylooException ( f'Broken cache for {capture_dir}' ) if self . is_public_instance : if cache . no_index : index = False if index : self . indexing . index_cookies_capture ( ct ) self . indexing . index_body_hashes_capture ( ct ) self . indexing . index_url_capture ( ct ) categories = list ( self . categories_capture ( capture_uuid ) . keys ( ) ) self . indexing . index_categories_capture ( capture_uuid , categories ) except Har2TreeError as e : raise NoValidHarFile ( e ) except RecursionError as e : raise NoValidHarFile ( f'Tree too deep, probably a recursive refresh: {e}.\n Append /export to the URL to get the files.' ) else : with pickle_file . open ( 'wb' ) as _p : default_recursion_limit = sys . getrecursionlimit ( ) sys . setrecursionlimit ( int ( default_recursion_limit * 1.1 ) ) pickle . dump ( ct , _p ) sys . setrecursionlimit ( default_recursion_limit ) finally : lock_file . unlink ( missing_ok = True ) return ct
Generate the pickle, set the cache, add capture in the indexes
Generate the pickle set the cache add capture in the indexes
'''Generate the pickle, set the cache, add capture in the indexes''' # Lock created, we can process # The pickle is being created somewhere else, wait until it's done. # NOTE: We only index the public captures # Force update cache of the capture (takes care of the incomplete redirect key) # Some pickles require a pretty high recursion limit, this kindof fixes it. # If the capture is really broken (generally a refresh to self), the capture # is discarded in the RecursionError above.
self
capture_uuid
str