montreal_forced_aligner/language_modeling/trainer.py
def prune_large_language_model(self) -> None:
        """Prune the large language model into small and medium versions"""
        self.log_info("Pruning large ngram model to medium and small versions...")
        small_mod_path = self.mod_path.replace(".mod", "_small.mod")
        med_mod_path = self.mod_path.replace(".mod", "_med.mod")
        subprocess.call(
            [
                "ngramshrink",
                f"--method={self.prune_method}",
                f"--theta={self.prune_thresh_medium}",
                self.mod_path,
                med_mod_path,
            ]
        )
        subprocess.call(["ngramprint", "--ARPA", med_mod_path, self.medium_arpa_path])

        self.log_debug("Finished pruning medium arpa!")
        subprocess.call(
            [
                "ngramshrink",
                f"--method={self.prune_method}",
                f"--theta={self.prune_thresh_small}",
                self.mod_path,
                small_mod_path,
            ]
        )
        subprocess.call(["ngramprint", "--ARPA", small_mod_path, self.small_arpa_path])

        self.log_debug("Finished pruning small arpa!")
        self.log_info("Done pruning!")
Prune the large language model into small and medium versions
Prune the large language model into small and medium versions
Prune the large language model into small and medium versions
def prune_large_language_model(self) -> None:
        self.log_info("Pruning large ngram model to medium and small versions...")
        small_mod_path = self.mod_path.replace(".mod", "_small.mod")
        med_mod_path = self.mod_path.replace(".mod", "_med.mod")
        subprocess.call(
            [
                "ngramshrink",
                f"--method={self.prune_method}",
                f"--theta={self.prune_thresh_medium}",
                self.mod_path,
                med_mod_path,
            ]
        )
        subprocess.call(["ngramprint", "--ARPA", med_mod_path, self.medium_arpa_path])
        self.log_debug("Finished pruning medium arpa!")
        subprocess.call(
            [
                "ngramshrink",
                f"--method={self.prune_method}",
                f"--theta={self.prune_thresh_small}",
                self.mod_path,
                small_mod_path,
            ]
        )
        subprocess.call(["ngramprint", "--ARPA", small_mod_path, self.small_arpa_path])
        self.log_debug("Finished pruning small arpa!")
        self.log_info("Done pruning!")
def prune_large_language_model ( self ) -> None : self . log_info ( "Pruning large ngram model to medium and small versions..." ) small_mod_path = self . mod_path . replace ( ".mod" , "_small.mod" ) med_mod_path = self . mod_path . replace ( ".mod" , "_med.mod" ) subprocess . call ( [ "ngramshrink" , f"--method={self.prune_method}" , f"--theta={self.prune_thresh_medium}" , self . mod_path , med_mod_path , ] ) subprocess . call ( [ "ngramprint" , "--ARPA" , med_mod_path , self . medium_arpa_path ] ) self . log_debug ( "Finished pruning medium arpa!" ) subprocess . call ( [ "ngramshrink" , f"--method={self.prune_method}" , f"--theta={self.prune_thresh_small}" , self . mod_path , small_mod_path , ] ) subprocess . call ( [ "ngramprint" , "--ARPA" , small_mod_path , self . small_arpa_path ] ) self . log_debug ( "Finished pruning small arpa!" ) self . log_info ( "Done pruning!" )
Prune the large language model into small and medium versions
Prune the large language model into small and medium versions
"""Prune the large language model into small and medium versions"""
self