theano/gof/cmodule.py
def refresh(self, age_thresh_use=None, delete_if_problem=False,
                cleanup=True):
        """
        Update cache data by walking the cache directory structure.

        Load key.pkl files that have not been loaded yet.
        Remove entries which have been removed from the filesystem.
        Also, remove malformed cache directories.

        Parameters
        ----------
        age_thresh_use
            Do not use modules other than this. Defaults to self.age_thresh_use.
        delete_if_problem : bool
            If True, cache entries that meet one of those two conditions are
            deleted:
            - Those for which unpickling the KeyData file fails with
              an unknown exception.
            - Duplicated modules, regardless of their age.
        cleanup : bool
            Do a cleanup of the cache removing expired and broken modules.

        Returns
        -------
        list
            A list of modules of age higher than age_thresh_use.

        """
        if age_thresh_use is None:
            age_thresh_use = self.age_thresh_use
        start_time = time.time()
        too_old_to_use = []

        to_delete = []
        to_delete_empty = []

        def rmtree(*args, **kwargs):
            if cleanup:
                to_delete.append((args, kwargs))

        def rmtree_empty(*args, **kwargs):
            if cleanup:
                to_delete_empty.append((args, kwargs))

        # add entries that are not in the entry_from_key dictionary
        time_now = time.time()
        # Go through directories in alphabetical order to ensure consistent
        # behavior.
        try:
            subdirs = sorted(os.listdir(self.dirname))
        except OSError:
            # This can happen if the dir don't exist.
            subdirs = []
        files, root = None, None  # To make sure the "del" below works
        for subdirs_elem in subdirs:
            # Never clean/remove lock_dir
            if subdirs_elem == 'lock_dir':
                continue
            root = os.path.join(self.dirname, subdirs_elem)
            # Don't delete the gpuarray kernel cache
            if root == config.gpuarray.cache_path:
                continue
            key_pkl = os.path.join(root, 'key.pkl')
            if key_pkl in self.loaded_key_pkl:
                continue
            if not os.path.isdir(root):
                continue
            files = os.listdir(root)
            if not files:
                rmtree_empty(root, ignore_nocleanup=True,
                             msg="empty dir")
                continue
            if 'delete.me' in files:
                rmtree(root, ignore_nocleanup=True,
                       msg="delete.me found in dir")
                continue
            elif 'key.pkl' in files:
                try:
                    entry = module_name_from_dir(root, files=files)
                except ValueError:  # there is a key but no dll!
                    if not root.startswith("/tmp"):
                        # Under /tmp, file are removed periodically by the
                        # os. So it is normal that this happens from time
                        # to time.
                        _logger.warning("ModuleCache.refresh() Found key "
                                        "without dll in cache, deleting it. %s",
                                        key_pkl)
                    rmtree(root, ignore_nocleanup=True,
                           msg="missing module file", level=logging.INFO)
                    continue
                if (time_now - last_access_time(entry)) < age_thresh_use:
                    _logger.debug('refresh adding %s', key_pkl)

                    def unpickle_failure():
                        _logger.info("ModuleCache.refresh() Failed to "
                                     "unpickle cache file %s", key_pkl)

                    try:
                        with open(key_pkl, 'rb') as f:
                            key_data = pickle.load(f)
                    except EOFError:
                        # Happened once... not sure why (would be worth
                        # investigating if it ever happens again).
                        unpickle_failure()
                        rmtree(root, ignore_nocleanup=True,
                               msg='broken cache directory [EOF]',
                               level=logging.WARNING)
                        continue
                    except Exception:
                        unpickle_failure()
                        if delete_if_problem:
                            rmtree(root, ignore_nocleanup=True,
                                   msg='broken cache directory',
                                   level=logging.INFO)
                        else:
                            # This exception is often triggered by keys
                            # that contain references to classes that have
                            # not yet been imported (e.g. when running two
                            # different Theano-based scripts). They are not
                            # necessarily broken, but we cannot load them
                            # now. They will be loaded later if needed.
                            pass
                        continue

                    if not isinstance(key_data, KeyData):
                        # This is some old cache data, that does not fit
                        # the new cache format. It would be possible to
                        # update it, but it is not entirely safe since we
                        # do not know the config options that were used.
                        # As a result, we delete it instead (which is also
                        # simpler to implement).
                        rmtree(root, ignore_nocleanup=True,
                               msg=(
                                   'invalid cache entry format -- this '
                                   'should not happen unless your cache '
                                   'was really old'),
                               level=logging.WARN)
                        continue

                    # Check the path to the module stored in the KeyData
                    # object matches the path to `entry`. There may be
                    # a mismatch e.g. due to symlinks, or some directory
                    # being renamed since last time cache was created.
                    kd_entry = key_data.get_entry()
                    if kd_entry != entry:
                        if is_same_entry(entry, kd_entry):
                            # Update KeyData object. Note that we also need
                            # to update the key_pkl field, because it is
                            # likely to be incorrect if the entry itself
                            # was wrong.
                            key_data.entry = entry
                            key_data.key_pkl = key_pkl
                        else:
                            # This is suspicious. Better get rid of it.
                            rmtree(root, ignore_nocleanup=True,
                                   msg='module file path mismatch',
                                   level=logging.INFO)
                            continue

                    # Find unversioned keys from other processes.
                    # TODO: check if this can happen at all
                    to_del = [key for key in key_data.keys if not key[0]]
                    if to_del:
                        _logger.warning(
                            "ModuleCache.refresh() Found unversioned "
                            "key in cache, removing it. %s", key_pkl)
                        # Since the version is in the module hash, all
                        # keys should be unversioned.
                        if len(to_del) != len(key_data.keys):
                            _logger.warning(
                                'Found a mix of unversioned and '
                                'versioned keys for the same '
                                'module %s', key_pkl)
                        rmtree(root, ignore_nocleanup=True,
                               msg="unversioned key(s) in cache",
                               level=logging.INFO)
                        continue

                    mod_hash = key_data.module_hash
                    if mod_hash in self.module_hash_to_key_data:
                        # This may happen when two processes running
                        # simultaneously compiled the same module, one
                        # after the other. We delete one once it is old
                        # enough (to be confident there is no other process
                        # using it), or if `delete_if_problem` is True.
                        # Note that it is important to walk through
                        # directories in alphabetical order so as to make
                        # sure all new processes only use the first one.
                        if cleanup:
                            age = time.time() - last_access_time(entry)
                            if delete_if_problem or age > self.age_thresh_del:
                                rmtree(root, ignore_nocleanup=True,
                                       msg='duplicated module',
                                       level=logging.DEBUG)
                            else:
                                _logger.debug('Found duplicated module not '
                                              'old enough yet to be deleted '
                                              '(age: %s): %s',
                                              age, entry)
                        continue

                    # Remember the map from a module's hash to the KeyData
                    # object associated with it.
                    self.module_hash_to_key_data[mod_hash] = key_data

                    for key in key_data.keys:
                        if key not in self.entry_from_key:
                            self.entry_from_key[key] = entry
                            # Assert that we have not already got this
                            # entry somehow.
                            assert entry not in self.module_from_name
                            # Store safe part of versioned keys.
                            if key[0]:
                                self.similar_keys.setdefault(
                                    get_safe_part(key),
                                    []).append(key)
                        else:
                            dir1 = os.path.dirname(self.entry_from_key[key])
                            dir2 = os.path.dirname(entry)
                            _logger.warning(
                                "The same cache key is associated to "
                                "different modules (%s and %s). This "
                                "is not supposed to happen! You may "
                                "need to manually delete your cache "
                                "directory to fix this.",
                                dir1, dir2)
                    # Clean up the name space to prevent bug.
                    if key_data.keys:
                        del key
                    self.loaded_key_pkl.add(key_pkl)
                else:
                    too_old_to_use.append(entry)

            # If the compilation failed, no key.pkl is in that
            # directory, but a mod.* should be there.
            # We do nothing here.

        # Clean up the name space to prevent bug.
        del root, files, subdirs

        # Remove entries that are not in the filesystem.
        items_copy = list(self.module_hash_to_key_data.items())
        for module_hash, key_data in items_copy:
            entry = key_data.get_entry()
            try:
                # Test to see that the file is [present and] readable.
                open(entry).close()
                gone = False
            except IOError:
                gone = True

            if gone:
                # Assert that we did not have one of the deleted files
                # loaded up and in use.
                # If so, it should not have been deleted. This should be
                # considered a failure of the OTHER process, that deleted
                # it.
                if entry in self.module_from_name:
                    _logger.warning("A module that was loaded by this "
                                    "ModuleCache can no longer be read from file "
                                    "%s... this could lead to problems.",
                                    entry)
                    del self.module_from_name[entry]

                _logger.info("deleting ModuleCache entry %s", entry)
                key_data.delete_keys_from(self.entry_from_key)
                del self.module_hash_to_key_data[module_hash]
                if key_data.keys and list(key_data.keys)[0][0]:
                    # this is a versioned entry, so should have been on
                    # disk. Something weird happened to cause this, so we
                    # are responding by printing a warning, removing
                    # evidence that we ever saw this mystery key.
                    pkl_file_to_remove = key_data.key_pkl
                    if not key_data.key_pkl.startswith("/tmp"):
                        # Under /tmp, file are removed periodically by the
                        # os. So it is normal that this happen from time to
                        # time.
                        _logger.warning("Removing key file %s because the "
                                        "corresponding module is gone from the "
                                        "file system.",
                                        pkl_file_to_remove)
                    self.loaded_key_pkl.remove(pkl_file_to_remove)

        if to_delete or to_delete_empty:
            with compilelock.lock_ctx():
                for a, kw in to_delete:
                    _rmtree(*a, **kw)
                for a, kw in to_delete_empty:
                    files = os.listdir(a[0])
                    if not files:
                        _rmtree(*a, **kw)

        _logger.debug('Time needed to refresh cache: %s',
                      (time.time() - start_time))

        return too_old_to_use

        Update cache data by walking the cache directory structure.

        Load key.pkl files that have not been loaded yet.
        Remove entries which have been removed from the filesystem.
        Also, remove malformed cache directories.

        Parameters
        ----------
        age_thresh_use
            Do not use modules other than this. Defaults to self.age_thresh_use.
        delete_if_problem : bool
            If True, cache entries that meet one of those two conditions are
            deleted:
            - Those for which unpickling the KeyData file fails with
              an unknown exception.
            - Duplicated modules, regardless of their age.
        cleanup : bool
            Do a cleanup of the cache removing expired and broken modules.

        Returns
        -------
        list
            A list of modules of age higher than age_thresh_use.

        
Update cache data by walking the cache directory structure.
Load key.pkl files that have not been loaded yet.
Remove entries which have been removed from the filesystem.
Also, remove malformed cache directories.

Parameters

age_thresh_use
Do not use modules other than this.

Returns

list
A list of modules of age higher than age_thresh_use.
Update cache data by walking the cache directory structure . Load key . pkl files that have not been loaded yet . Remove entries which have been removed from the filesystem . Also remove malformed cache directories . Parameters age_thresh_use Do not use modules other than this . Returns list A list of modules of age higher than age_thresh_use .
def refresh(self, age_thresh_use=None, delete_if_problem=False,
                cleanup=True):
        if age_thresh_use is None:
            age_thresh_use = self.age_thresh_use
        start_time = time.time()
        too_old_to_use = []
        to_delete = []
        to_delete_empty = []
        def rmtree(*args, **kwargs):
            if cleanup:
                to_delete.append((args, kwargs))
        def rmtree_empty(*args, **kwargs):
            if cleanup:
                to_delete_empty.append((args, kwargs))
        time_now = time.time()
        try:
            subdirs = sorted(os.listdir(self.dirname))
        except OSError:
            subdirs = []
        files, root = None, None  
        for subdirs_elem in subdirs:
            if subdirs_elem == 'lock_dir':
                continue
            root = os.path.join(self.dirname, subdirs_elem)
            if root == config.gpuarray.cache_path:
                continue
            key_pkl = os.path.join(root, 'key.pkl')
            if key_pkl in self.loaded_key_pkl:
                continue
            if not os.path.isdir(root):
                continue
            files = os.listdir(root)
            if not files:
                rmtree_empty(root, ignore_nocleanup=True,
                             msg="empty dir")
                continue
            if 'delete.me' in files:
                rmtree(root, ignore_nocleanup=True,
                       msg="delete.me found in dir")
                continue
            elif 'key.pkl' in files:
                try:
                    entry = module_name_from_dir(root, files=files)
                except ValueError:  
                    if not root.startswith("/tmp"):
                        _logger.warning("ModuleCache.refresh() Found key "
                                        "without dll in cache, deleting it. %s",
                                        key_pkl)
                    rmtree(root, ignore_nocleanup=True,
                           msg="missing module file", level=logging.INFO)
                    continue
                if (time_now - last_access_time(entry)) < age_thresh_use:
                    _logger.debug('refresh adding %s', key_pkl)
                    def unpickle_failure():
                        _logger.info("ModuleCache.refresh() Failed to "
                                     "unpickle cache file %s", key_pkl)
                    try:
                        with open(key_pkl, 'rb') as f:
                            key_data = pickle.load(f)
                    except EOFError:
                        unpickle_failure()
                        rmtree(root, ignore_nocleanup=True,
                               msg='broken cache directory [EOF]',
                               level=logging.WARNING)
                        continue
                    except Exception:
                        unpickle_failure()
                        if delete_if_problem:
                            rmtree(root, ignore_nocleanup=True,
                                   msg='broken cache directory',
                                   level=logging.INFO)
                        else:
                            pass
                        continue
                    if not isinstance(key_data, KeyData):
                        rmtree(root, ignore_nocleanup=True,
                               msg=(
                                   'invalid cache entry format -- this '
                                   'should not happen unless your cache '
                                   'was really old'),
                               level=logging.WARN)
                        continue
                    kd_entry = key_data.get_entry()
                    if kd_entry != entry:
                        if is_same_entry(entry, kd_entry):
                            key_data.entry = entry
                            key_data.key_pkl = key_pkl
                        else:
                            rmtree(root, ignore_nocleanup=True,
                                   msg='module file path mismatch',
                                   level=logging.INFO)
                            continue
                    to_del = [key for key in key_data.keys if not key[0]]
                    if to_del:
                        _logger.warning(
                            "ModuleCache.refresh() Found unversioned "
                            "key in cache, removing it. %s", key_pkl)
                        if len(to_del) != len(key_data.keys):
                            _logger.warning(
                                'Found a mix of unversioned and '
                                'versioned keys for the same '
                                'module %s', key_pkl)
                        rmtree(root, ignore_nocleanup=True,
                               msg="unversioned key(s) in cache",
                               level=logging.INFO)
                        continue
                    mod_hash = key_data.module_hash
                    if mod_hash in self.module_hash_to_key_data:
                        if cleanup:
                            age = time.time() - last_access_time(entry)
                            if delete_if_problem or age > self.age_thresh_del:
                                rmtree(root, ignore_nocleanup=True,
                                       msg='duplicated module',
                                       level=logging.DEBUG)
                            else:
                                _logger.debug('Found duplicated module not '
                                              'old enough yet to be deleted '
                                              '(age: %s): %s',
                                              age, entry)
                        continue
                    self.module_hash_to_key_data[mod_hash] = key_data
                    for key in key_data.keys:
                        if key not in self.entry_from_key:
                            self.entry_from_key[key] = entry
                            assert entry not in self.module_from_name
                            if key[0]:
                                self.similar_keys.setdefault(
                                    get_safe_part(key),
                                    []).append(key)
                        else:
                            dir1 = os.path.dirname(self.entry_from_key[key])
                            dir2 = os.path.dirname(entry)
                            _logger.warning(
                                "The same cache key is associated to "
                                "different modules (%s and %s). This "
                                "is not supposed to happen! You may "
                                "need to manually delete your cache "
                                "directory to fix this.",
                                dir1, dir2)
                    if key_data.keys:
                        del key
                    self.loaded_key_pkl.add(key_pkl)
                else:
                    too_old_to_use.append(entry)
        del root, files, subdirs
        items_copy = list(self.module_hash_to_key_data.items())
        for module_hash, key_data in items_copy:
            entry = key_data.get_entry()
            try:
                open(entry).close()
                gone = False
            except IOError:
                gone = True
            if gone:
                if entry in self.module_from_name:
                    _logger.warning("A module that was loaded by this "
                                    "ModuleCache can no longer be read from file "
                                    "%s... this could lead to problems.",
                                    entry)
                    del self.module_from_name[entry]
                _logger.info("deleting ModuleCache entry %s", entry)
                key_data.delete_keys_from(self.entry_from_key)
                del self.module_hash_to_key_data[module_hash]
                if key_data.keys and list(key_data.keys)[0][0]:
                    pkl_file_to_remove = key_data.key_pkl
                    if not key_data.key_pkl.startswith("/tmp"):
                        _logger.warning("Removing key file %s because the "
                                        "corresponding module is gone from the "
                                        "file system.",
                                        pkl_file_to_remove)
                    self.loaded_key_pkl.remove(pkl_file_to_remove)
        if to_delete or to_delete_empty:
            with compilelock.lock_ctx():
                for a, kw in to_delete:
                    _rmtree(*a, **kw)
                for a, kw in to_delete_empty:
                    files = os.listdir(a[0])
                    if not files:
                        _rmtree(*a, **kw)
        _logger.debug('Time needed to refresh cache: %s',
                      (time.time() - start_time))
        return too_old_to_use
def refresh ( self , age_thresh_use = None , delete_if_problem = False , cleanup = True ) : if age_thresh_use is None : age_thresh_use = self . age_thresh_use start_time = time . time ( ) too_old_to_use = [ ] to_delete = [ ] to_delete_empty = [ ] def rmtree ( * args , ** kwargs ) : if cleanup : to_delete . append ( ( args , kwargs ) ) def rmtree_empty ( * args , ** kwargs ) : if cleanup : to_delete_empty . append ( ( args , kwargs ) ) time_now = time . time ( ) try : subdirs = sorted ( os . listdir ( self . dirname ) ) except OSError : subdirs = [ ] files , root = None , None for subdirs_elem in subdirs : if subdirs_elem == 'lock_dir' : continue root = os . path . join ( self . dirname , subdirs_elem ) if root == config . gpuarray . cache_path : continue key_pkl = os . path . join ( root , 'key.pkl' ) if key_pkl in self . loaded_key_pkl : continue if not os . path . isdir ( root ) : continue files = os . listdir ( root ) if not files : rmtree_empty ( root , ignore_nocleanup = True , msg = "empty dir" ) continue if 'delete.me' in files : rmtree ( root , ignore_nocleanup = True , msg = "delete.me found in dir" ) continue elif 'key.pkl' in files : try : entry = module_name_from_dir ( root , files = files ) except ValueError : if not root . startswith ( "/tmp" ) : _logger . warning ( "ModuleCache.refresh() Found key " "without dll in cache, deleting it. %s" , key_pkl ) rmtree ( root , ignore_nocleanup = True , msg = "missing module file" , level = logging . INFO ) continue if ( time_now - last_access_time ( entry ) ) < age_thresh_use : _logger . debug ( 'refresh adding %s' , key_pkl ) def unpickle_failure ( ) : _logger . info ( "ModuleCache.refresh() Failed to " "unpickle cache file %s" , key_pkl ) try : with open ( key_pkl , 'rb' ) as f : key_data = pickle . load ( f ) except EOFError : unpickle_failure ( ) rmtree ( root , ignore_nocleanup = True , msg = 'broken cache directory [EOF]' , level = logging . WARNING ) continue except Exception : unpickle_failure ( ) if delete_if_problem : rmtree ( root , ignore_nocleanup = True , msg = 'broken cache directory' , level = logging . INFO ) else : pass continue if not isinstance ( key_data , KeyData ) : rmtree ( root , ignore_nocleanup = True , msg = ( 'invalid cache entry format -- this ' 'should not happen unless your cache ' 'was really old' ) , level = logging . WARN ) continue kd_entry = key_data . get_entry ( ) if kd_entry != entry : if is_same_entry ( entry , kd_entry ) : key_data . entry = entry key_data . key_pkl = key_pkl else : rmtree ( root , ignore_nocleanup = True , msg = 'module file path mismatch' , level = logging . INFO ) continue to_del = [ key for key in key_data . keys if not key [ 0 ] ] if to_del : _logger . warning ( "ModuleCache.refresh() Found unversioned " "key in cache, removing it. %s" , key_pkl ) if len ( to_del ) != len ( key_data . keys ) : _logger . warning ( 'Found a mix of unversioned and ' 'versioned keys for the same ' 'module %s' , key_pkl ) rmtree ( root , ignore_nocleanup = True , msg = "unversioned key(s) in cache" , level = logging . INFO ) continue mod_hash = key_data . module_hash if mod_hash in self . module_hash_to_key_data : if cleanup : age = time . time ( ) - last_access_time ( entry ) if delete_if_problem or age > self . age_thresh_del : rmtree ( root , ignore_nocleanup = True , msg = 'duplicated module' , level = logging . DEBUG ) else : _logger . debug ( 'Found duplicated module not ' 'old enough yet to be deleted ' '(age: %s): %s' , age , entry ) continue self . module_hash_to_key_data [ mod_hash ] = key_data for key in key_data . keys : if key not in self . entry_from_key : self . entry_from_key [ key ] = entry assert entry not in self . module_from_name if key [ 0 ] : self . similar_keys . setdefault ( get_safe_part ( key ) , [ ] ) . append ( key ) else : dir1 = os . path . dirname ( self . entry_from_key [ key ] ) dir2 = os . path . dirname ( entry ) _logger . warning ( "The same cache key is associated to " "different modules (%s and %s). This " "is not supposed to happen! You may " "need to manually delete your cache " "directory to fix this." , dir1 , dir2 ) if key_data . keys : del key self . loaded_key_pkl . add ( key_pkl ) else : too_old_to_use . append ( entry ) del root , files , subdirs items_copy = list ( self . module_hash_to_key_data . items ( ) ) for module_hash , key_data in items_copy : entry = key_data . get_entry ( ) try : open ( entry ) . close ( ) gone = False except IOError : gone = True if gone : if entry in self . module_from_name : _logger . warning ( "A module that was loaded by this " "ModuleCache can no longer be read from file " "%s... this could lead to problems." , entry ) del self . module_from_name [ entry ] _logger . info ( "deleting ModuleCache entry %s" , entry ) key_data . delete_keys_from ( self . entry_from_key ) del self . module_hash_to_key_data [ module_hash ] if key_data . keys and list ( key_data . keys ) [ 0 ] [ 0 ] : pkl_file_to_remove = key_data . key_pkl if not key_data . key_pkl . startswith ( "/tmp" ) : _logger . warning ( "Removing key file %s because the " "corresponding module is gone from the " "file system." , pkl_file_to_remove ) self . loaded_key_pkl . remove ( pkl_file_to_remove ) if to_delete or to_delete_empty : with compilelock . lock_ctx ( ) : for a , kw in to_delete : _rmtree ( * a , ** kw ) for a , kw in to_delete_empty : files = os . listdir ( a [ 0 ] ) if not files : _rmtree ( * a , ** kw ) _logger . debug ( 'Time needed to refresh cache: %s' , ( time . time ( ) - start_time ) ) return too_old_to_use
Update cache data by walking the cache directory structure.
Update cache data by walking the cache directory structure .
"""
        Update cache data by walking the cache directory structure.

        Load key.pkl files that have not been loaded yet.
        Remove entries which have been removed from the filesystem.
        Also, remove malformed cache directories.

        Parameters
        ----------
        age_thresh_use
            Do not use modules other than this. Defaults to self.age_thresh_use.
        delete_if_problem : bool
            If True, cache entries that meet one of those two conditions are
            deleted:
            - Those for which unpickling the KeyData file fails with
              an unknown exception.
            - Duplicated modules, regardless of their age.
        cleanup : bool
            Do a cleanup of the cache removing expired and broken modules.

        Returns
        -------
        list
            A list of modules of age higher than age_thresh_use.

        """ # add entries that are not in the entry_from_key dictionary # Go through directories in alphabetical order to ensure consistent # behavior. # This can happen if the dir don't exist. # To make sure the "del" below works # Never clean/remove lock_dir # Don't delete the gpuarray kernel cache # there is a key but no dll! # Under /tmp, file are removed periodically by the # os. So it is normal that this happens from time # to time. # Happened once... not sure why (would be worth # investigating if it ever happens again). # This exception is often triggered by keys # that contain references to classes that have # not yet been imported (e.g. when running two # different Theano-based scripts). They are not # necessarily broken, but we cannot load them # now. They will be loaded later if needed. # This is some old cache data, that does not fit # the new cache format. It would be possible to # update it, but it is not entirely safe since we # do not know the config options that were used. # As a result, we delete it instead (which is also # simpler to implement). # Check the path to the module stored in the KeyData # object matches the path to `entry`. There may be # a mismatch e.g. due to symlinks, or some directory # being renamed since last time cache was created. # Update KeyData object. Note that we also need # to update the key_pkl field, because it is # likely to be incorrect if the entry itself # was wrong. # This is suspicious. Better get rid of it. # Find unversioned keys from other processes. # TODO: check if this can happen at all # Since the version is in the module hash, all # keys should be unversioned. # This may happen when two processes running # simultaneously compiled the same module, one # after the other. We delete one once it is old # enough (to be confident there is no other process # using it), or if `delete_if_problem` is True. # Note that it is important to walk through # directories in alphabetical order so as to make # sure all new processes only use the first one. # Remember the map from a module's hash to the KeyData # object associated with it. # Assert that we have not already got this # entry somehow. # Store safe part of versioned keys. # Clean up the name space to prevent bug. # If the compilation failed, no key.pkl is in that # directory, but a mod.* should be there. # We do nothing here. # Clean up the name space to prevent bug. # Remove entries that are not in the filesystem. # Test to see that the file is [present and] readable. # Assert that we did not have one of the deleted files # loaded up and in use. # If so, it should not have been deleted. This should be # considered a failure of the OTHER process, that deleted # it. # this is a versioned entry, so should have been on # disk. Something weird happened to cause this, so we # are responding by printing a warning, removing # evidence that we ever saw this mystery key. # Under /tmp, file are removed periodically by the # os. So it is normal that this happen from time to # time.
self
age_thresh_use
delete_if_problem
cleanup