pub fn garbage_collect<L: FnOnce() -> HashSet<PathBuf>>(
        &mut self,
        get_living_files: L,
    ) -> crate::Result<GarbageCollectionResult> {
        info!("Garbage collect");
        let mut files_to_delete = vec![];

        // It is crucial to get the living files after acquiring the
        // read lock of meta informations. That way, we
        // avoid the following scenario.
        //
        // 1) we get the list of living files.
        // 2) someone creates a new file.
        // 3) we start garbage collection and remove this file
        // even though it is a living file.
        //
        // releasing the lock as .delete() will use it too.
        {
            let meta_informations_rlock = self
                .meta_informations
                .read()
                .expect("Managed directory rlock poisoned in garbage collect.");

            // The point of this second "file" lock is to enforce the following scenario
            // 1) process B tries to load a new set of searcher.
            // The list of segments is loaded
            // 2) writer change meta.json (for instance after a merge or a commit)
            // 3) gc kicks in.
            // 4) gc removes a file that was useful for process B, before process B opened it.
            match self.acquire_lock(&META_LOCK) {
                Ok(_meta_lock) => {
                    let living_files = get_living_files();
                    for managed_path in &meta_informations_rlock.managed_paths {
                        if !living_files.contains(managed_path) {
                            files_to_delete.push(managed_path.clone());
                        }
                    }
                }
                Err(err) => {
                    error!("Failed to acquire lock for GC");
                    return Err(crate::TantivyError::from(err));
                }
            }
        }

        let mut failed_to_delete_files = vec![];
        let mut deleted_files = vec![];

        for file_to_delete in files_to_delete {
            match self.delete(&file_to_delete) {
                Ok(_) => {
                    info!("Deleted {:?}", file_to_delete);
                    deleted_files.push(file_to_delete);
                }
                Err(file_error) => {
                    match file_error {
                        DeleteError::FileDoesNotExist(_) => {
                            deleted_files.push(file_to_delete.clone());
                        }
                        DeleteError::IoError { .. } => {
                            failed_to_delete_files.push(file_to_delete.clone());
                            if !cfg!(target_os = "windows") {
                                // On windows, delete is expected to fail if the file
                                // is mmapped.
                                error!("Failed to delete {:?}", file_to_delete);
                            }
                        }
                    }
                }
            }
        }

        if !deleted_files.is_empty() {
            // update the list of managed files by removing
            // the file that were removed.
            let mut meta_informations_wlock = self
                .meta_informations
                .write()
                .expect("Managed directory wlock poisoned (2).");
            let managed_paths_write = &mut meta_informations_wlock.managed_paths;
            for delete_file in &deleted_files {
                managed_paths_write.remove(delete_file);
            }
            save_managed_paths(self.directory.as_mut(), &meta_informations_wlock)?;
        }

        Ok(GarbageCollectionResult {
            deleted_files,
            failed_to_delete_files,
        })
    }
/// Garbage collect unused files.
///
/// Removes the files that were created by `tantivy` and are not
/// used by any segment anymore.
///
/// * `living_files` - List of files that are still used by the index.
///
/// The use a callback ensures that the list of living_files is computed
/// while we hold the lock on meta.
///
/// This method does not panick nor returns errors.
/// If a file cannot be deleted (for permission reasons for instance)
/// an error is simply logged, and the file remains in the list of managed
/// files.
Garbage collect unused files.
Removes the files that were created by `tantivy` and are not
used by any segment anymore.

`living_files` - List of files that are still used by the index.

The use a callback ensures that the list of living_files is computed
while we hold the lock on meta.

This method does not panick nor returns errors.
If a file cannot be deleted (for permission reasons for instance)
an error is simply logged, and the file remains in the list of managed
files.
Garbage collect unused files . Removes the files that were created by ` tantivy ` and are not used by any segment anymore . ` living_files ` - List of files that are still used by the index . The use a callback ensures that the list of living_files is computed while we hold the lock on meta . This method does not panick nor returns errors . If a file cannot be deleted ( for permission reasons for instance ) an error is simply logged and the file remains in the list of managed files .
pub fn garbage_collect<L: FnOnce() -> HashSet<PathBuf>>(
        &mut self,
        get_living_files: L,
    ) -> crate::Result<GarbageCollectionResult> {
        info!("Garbage collect");
        let mut files_to_delete = vec![];
        // It is crucial to get the living files after acquiring the
        // read lock of meta informations. That way, we
        // avoid the following scenario.
        //
        // 1) we get the list of living files.
        // 2) someone creates a new file.
        // 3) we start garbage collection and remove this file
        // even though it is a living file.
        //
        // releasing the lock as .delete() will use it too.
        {
            let meta_informations_rlock = self
                .meta_informations
                .read()
                .expect("Managed directory rlock poisoned in garbage collect.");
            // The point of this second "file" lock is to enforce the following scenario
            // 1) process B tries to load a new set of searcher.
            // The list of segments is loaded
            // 2) writer change meta.json (for instance after a merge or a commit)
            // 3) gc kicks in.
            // 4) gc removes a file that was useful for process B, before process B opened it.
            match self.acquire_lock(&META_LOCK) {
                Ok(_meta_lock) => {
                    let living_files = get_living_files();
                    for managed_path in &meta_informations_rlock.managed_paths {
                        if !living_files.contains(managed_path) {
                            files_to_delete.push(managed_path.clone());
                        }
                    }
                }
                Err(err) => {
                    error!("Failed to acquire lock for GC");
                    return Err(crate::TantivyError::from(err));
                }
            }
        }
        let mut failed_to_delete_files = vec![];
        let mut deleted_files = vec![];
        for file_to_delete in files_to_delete {
            match self.delete(&file_to_delete) {
                Ok(_) => {
                    info!("Deleted {:?}", file_to_delete);
                    deleted_files.push(file_to_delete);
                }
                Err(file_error) => {
                    match file_error {
                        DeleteError::FileDoesNotExist(_) => {
                            deleted_files.push(file_to_delete.clone());
                        }
                        DeleteError::IoError { .. } => {
                            failed_to_delete_files.push(file_to_delete.clone());
                            if !cfg!(target_os = "windows") {
                                // On windows, delete is expected to fail if the file
                                // is mmapped.
                                error!("Failed to delete {:?}", file_to_delete);
                            }
                        }
                    }
                }
            }
        }
        if !deleted_files.is_empty() {
            // update the list of managed files by removing
            // the file that were removed.
            let mut meta_informations_wlock = self
                .meta_informations
                .write()
                .expect("Managed directory wlock poisoned (2).");
            let managed_paths_write = &mut meta_informations_wlock.managed_paths;
            for delete_file in &deleted_files {
                managed_paths_write.remove(delete_file);
            }
            save_managed_paths(self.directory.as_mut(), &meta_informations_wlock)?;
        }
        Ok(GarbageCollectionResult {
            deleted_files,
            failed_to_delete_files,
        })
    }
pub fn garbage_collect < L : FnOnce ( ) -> HashSet < PathBuf > > ( & mut self , get_living_files : L , ) -> crate :: Result < GarbageCollectionResult > { info ! ( " " ) ; let mut files_to_delete = vec ! [ ] ; // It is crucial to get the living files after acquiring the // read lock of meta informations. That way, we // avoid the following scenario. // // 1) we get the list of living files. // 2) someone creates a new file. // 3) we start garbage collection and remove this file // even though it is a living file. // // releasing the lock as .delete() will use it too. { let meta_informations_rlock = self . meta_informations . read ( ) . expect ( " " ) ; // The point of this second "file" lock is to enforce the following scenario // 1) process B tries to load a new set of searcher. // The list of segments is loaded // 2) writer change meta.json (for instance after a merge or a commit) // 3) gc kicks in. // 4) gc removes a file that was useful for process B, before process B opened it. match self . acquire_lock ( & META_LOCK ) { Ok ( _meta_lock ) => { let living_files = get_living_files ( ) ; for managed_path in & meta_informations_rlock . managed_paths { if ! living_files . contains ( managed_path ) { files_to_delete . push ( managed_path . clone ( ) ) ; } } } Err ( err ) => { error ! ( " " ) ; return Err ( crate :: TantivyError :: from ( err ) ) ; } } } let mut failed_to_delete_files = vec ! [ ] ; let mut deleted_files = vec ! [ ] ; for file_to_delete in files_to_delete { match self . delete ( & file_to_delete ) { Ok ( _ ) => { info ! ( " " file_to_delete ) ; deleted_files . push ( file_to_delete ) ; } Err ( file_error ) => { match file_error { DeleteError :: FileDoesNotExist ( _ ) => { deleted_files . push ( file_to_delete . clone ( ) ) ; } DeleteError :: IoError { .. } => { failed_to_delete_files . push ( file_to_delete . clone ( ) ) ; if ! cfg ! ( target_os " " ) { // On windows, delete is expected to fail if the file // is mmapped. error ! ( " " file_to_delete ) ; } } } } } } if ! deleted_files . is_empty ( ) { // update the list of managed files by removing // the file that were removed. let mut meta_informations_wlock = self . meta_informations . write ( ) . expect ( " " ) ; let managed_paths_write = & mut meta_informations_wlock . managed_paths ; for delete_file in & deleted_files { managed_paths_write . remove ( delete_file ) ; } save_managed_paths ( self . directory . as_mut ( ) , & meta_informations_wlock ) ? ; } Ok ( GarbageCollectionResult { deleted_files , failed_to_delete_files , } ) }
Garbage collect unused files.
Garbage collect unused files .
&mut self
get_living_files
L
&mut self
get_living_files
L