montreal_forced_aligner/alignment/base.py
def ctms_to_textgrids_mp(self):
        """
        Multiprocessing function for exporting alignment CTM information as TextGrids

        See Also
        --------
        :class:`~montreal_forced_aligner.alignment.multiprocessing.AliToCtmFunction`
            Multiprocessing helper function for converting ali archives to CTM format
        :class:`~montreal_forced_aligner.alignment.multiprocessing.PhoneCtmProcessWorker`
            Multiprocessing helper class for processing CTM files
        :meth:`.CorpusAligner.phone_ctm_arguments`
            Job method for generating arguments for PhoneCtmProcessWorker
        :class:`~montreal_forced_aligner.alignment.multiprocessing.WordCtmProcessWorker`
            Multiprocessing helper class for processing word CTM files
        :meth:`.CorpusAligner.word_ctm_arguments`
            Job method for generating arguments for WordCtmProcessWorker
        :class:`~montreal_forced_aligner.alignment.multiprocessing.ExportTextGridProcessWorker`
            Multiprocessing helper class for exporting TextGrid files
        :meth:`.CorpusAligner.export_textgrid_arguments`
            Job method for generating arguments for ExportTextGridProcessWorker
        :kaldi_steps:`get_train_ctm`
            Reference Kaldi script

        """
        export_begin = time.time()
        manager = mp.Manager()
        textgrid_errors = manager.dict()
        error_catching = manager.dict()
        stopped = Stopped()
        if not self.overwrite:
            os.makedirs(self.backup_output_directory, exist_ok=True)

        self.logger.debug("Beginning to process ctm files...")
        word_procs = []
        phone_procs = []
        finished_processing = Stopped()
        to_process_queue = mp.JoinableQueue()
        for_write_queue = mp.JoinableQueue()
        total_files = len(self.files)
        word_ctm_args = self.word_ctm_arguments()
        phone_ctm_args = self.phone_ctm_arguments()
        export_args = self.export_textgrid_arguments()
        for j in self.jobs:
            word_p = WordCtmProcessWorker(
                j.name,
                to_process_queue,
                stopped,
                error_catching,
                word_ctm_args[j.name],
            )

            word_procs.append(word_p)
            word_p.start()

            phone_p = PhoneCtmProcessWorker(
                j.name,
                to_process_queue,
                stopped,
                error_catching,
                phone_ctm_args[j.name],
            )
            phone_p.start()
            phone_procs.append(phone_p)

        export_procs = []
        for j in self.jobs:
            export_proc = ExportTextGridProcessWorker(
                for_write_queue,
                stopped,
                finished_processing,
                textgrid_errors,
                export_args[j.name],
            )
            export_proc.start()
            export_procs.append(export_proc)
        try:
            with tqdm.tqdm(total=total_files) as pbar:
                while True:
                    try:
                        w_p, intervals = to_process_queue.get(timeout=1)
                    except Empty:
                        for proc in word_procs:
                            if not proc.finished_signal.stop_check():
                                break
                        for proc in phone_procs:
                            if not proc.finished_signal.stop_check():
                                break
                        else:
                            break
                        continue
                    to_process_queue.task_done()
                    if self.stopped.stop_check():
                        self.logger.debug("Got stop check, exiting")
                        continue
                    utt = self.utterances[intervals[0].utterance]
                    if w_p == "word":
                        utt.add_word_intervals(intervals)
                    else:
                        utt.add_phone_intervals(intervals)
                    file = self.files[utt.file_name]
                    if file.is_fully_aligned:
                        tiers = file.aligned_data
                        output_path = file.construct_output_path(
                            self.textgrid_output, self.backup_output_directory
                        )
                        duration = file.duration
                        for_write_queue.put((tiers, output_path, duration))
                        pbar.update(1)
        except Exception:
            stopped.stop()
            while True:
                try:
                    _ = to_process_queue.get(timeout=1)
                except Empty:
                    for proc in word_procs:
                        if not proc.finished_signal.stop_check():
                            break
                    for proc in phone_procs:
                        if not proc.finished_signal.stop_check():
                            break
                    else:
                        break
                    continue
                to_process_queue.task_done()
            exc_type, exc_value, exc_traceback = sys.exc_info()
            error_catching["main"] = "\n".join(
                traceback.format_exception(exc_type, exc_value, exc_traceback)
            )
        finally:
            finished_processing.stop()
            self.logger.debug("Waiting for processes to finish...")
            for i in range(self.num_jobs):
                word_procs[i].join()
                phone_procs[i].join()
            to_process_queue.join()

            for_write_queue.join()
            for i in range(self.num_jobs):
                export_procs[i].join()
            self.logger.debug(f"Export took {time.time() - export_begin} seconds")

        if error_catching:
            self.logger.error("Error was encountered in processing CTMs")
            for key, error in error_catching.items():
                self.logger.error(f"{key}:\n\n{error}")
            raise AlignmentExportError(error_catching)

        if textgrid_errors:
            self.logger.warning(
                f"There were {len(textgrid_errors)} errors encountered in generating TextGrids. "
                f"Check the output_errors.txt file in {os.path.join(self.textgrid_output)} "
                f"for more details"
            )
        output_textgrid_writing_errors(self.textgrid_output, textgrid_errors)

        Multiprocessing function for exporting alignment CTM information as TextGrids

        See Also
        --------
        :class:`~montreal_forced_aligner.alignment.multiprocessing.AliToCtmFunction`
            Multiprocessing helper function for converting ali archives to CTM format
        :class:`~montreal_forced_aligner.alignment.multiprocessing.PhoneCtmProcessWorker`
            Multiprocessing helper class for processing CTM files
        :meth:`.CorpusAligner.phone_ctm_arguments`
            Job method for generating arguments for PhoneCtmProcessWorker
        :class:`~montreal_forced_aligner.alignment.multiprocessing.WordCtmProcessWorker`
            Multiprocessing helper class for processing word CTM files
        :meth:`.CorpusAligner.word_ctm_arguments`
            Job method for generating arguments for WordCtmProcessWorker
        :class:`~montreal_forced_aligner.alignment.multiprocessing.ExportTextGridProcessWorker`
            Multiprocessing helper class for exporting TextGrid files
        :meth:`.CorpusAligner.export_textgrid_arguments`
            Job method for generating arguments for ExportTextGridProcessWorker
        :kaldi_steps:`get_train_ctm`
            Reference Kaldi script

        
Multiprocessing function for exporting alignment CTM information as TextGrids
See Also
Multiprocessing function for exporting alignment CTM information as TextGrids See Also
def ctms_to_textgrids_mp(self):
        export_begin = time.time()
        manager = mp.Manager()
        textgrid_errors = manager.dict()
        error_catching = manager.dict()
        stopped = Stopped()
        if not self.overwrite:
            os.makedirs(self.backup_output_directory, exist_ok=True)
        self.logger.debug("Beginning to process ctm files...")
        word_procs = []
        phone_procs = []
        finished_processing = Stopped()
        to_process_queue = mp.JoinableQueue()
        for_write_queue = mp.JoinableQueue()
        total_files = len(self.files)
        word_ctm_args = self.word_ctm_arguments()
        phone_ctm_args = self.phone_ctm_arguments()
        export_args = self.export_textgrid_arguments()
        for j in self.jobs:
            word_p = WordCtmProcessWorker(
                j.name,
                to_process_queue,
                stopped,
                error_catching,
                word_ctm_args[j.name],
            )
            word_procs.append(word_p)
            word_p.start()
            phone_p = PhoneCtmProcessWorker(
                j.name,
                to_process_queue,
                stopped,
                error_catching,
                phone_ctm_args[j.name],
            )
            phone_p.start()
            phone_procs.append(phone_p)
        export_procs = []
        for j in self.jobs:
            export_proc = ExportTextGridProcessWorker(
                for_write_queue,
                stopped,
                finished_processing,
                textgrid_errors,
                export_args[j.name],
            )
            export_proc.start()
            export_procs.append(export_proc)
        try:
            with tqdm.tqdm(total=total_files) as pbar:
                while True:
                    try:
                        w_p, intervals = to_process_queue.get(timeout=1)
                    except Empty:
                        for proc in word_procs:
                            if not proc.finished_signal.stop_check():
                                break
                        for proc in phone_procs:
                            if not proc.finished_signal.stop_check():
                                break
                        else:
                            break
                        continue
                    to_process_queue.task_done()
                    if self.stopped.stop_check():
                        self.logger.debug("Got stop check, exiting")
                        continue
                    utt = self.utterances[intervals[0].utterance]
                    if w_p == "word":
                        utt.add_word_intervals(intervals)
                    else:
                        utt.add_phone_intervals(intervals)
                    file = self.files[utt.file_name]
                    if file.is_fully_aligned:
                        tiers = file.aligned_data
                        output_path = file.construct_output_path(
                            self.textgrid_output, self.backup_output_directory
                        )
                        duration = file.duration
                        for_write_queue.put((tiers, output_path, duration))
                        pbar.update(1)
        except Exception:
            stopped.stop()
            while True:
                try:
                    _ = to_process_queue.get(timeout=1)
                except Empty:
                    for proc in word_procs:
                        if not proc.finished_signal.stop_check():
                            break
                    for proc in phone_procs:
                        if not proc.finished_signal.stop_check():
                            break
                    else:
                        break
                    continue
                to_process_queue.task_done()
            exc_type, exc_value, exc_traceback = sys.exc_info()
            error_catching["main"] = "\n".join(
                traceback.format_exception(exc_type, exc_value, exc_traceback)
            )
        finally:
            finished_processing.stop()
            self.logger.debug("Waiting for processes to finish...")
            for i in range(self.num_jobs):
                word_procs[i].join()
                phone_procs[i].join()
            to_process_queue.join()
            for_write_queue.join()
            for i in range(self.num_jobs):
                export_procs[i].join()
            self.logger.debug(f"Export took {time.time() - export_begin} seconds")
        if error_catching:
            self.logger.error("Error was encountered in processing CTMs")
            for key, error in error_catching.items():
                self.logger.error(f"{key}:\n\n{error}")
            raise AlignmentExportError(error_catching)
        if textgrid_errors:
            self.logger.warning(
                f"There were {len(textgrid_errors)} errors encountered in generating TextGrids. "
                f"Check the output_errors.txt file in {os.path.join(self.textgrid_output)} "
                f"for more details"
            )
        output_textgrid_writing_errors(self.textgrid_output, textgrid_errors)
def ctms_to_textgrids_mp ( self ) : export_begin = time . time ( ) manager = mp . Manager ( ) textgrid_errors = manager . dict ( ) error_catching = manager . dict ( ) stopped = Stopped ( ) if not self . overwrite : os . makedirs ( self . backup_output_directory , exist_ok = True ) self . logger . debug ( "Beginning to process ctm files..." ) word_procs = [ ] phone_procs = [ ] finished_processing = Stopped ( ) to_process_queue = mp . JoinableQueue ( ) for_write_queue = mp . JoinableQueue ( ) total_files = len ( self . files ) word_ctm_args = self . word_ctm_arguments ( ) phone_ctm_args = self . phone_ctm_arguments ( ) export_args = self . export_textgrid_arguments ( ) for j in self . jobs : word_p = WordCtmProcessWorker ( j . name , to_process_queue , stopped , error_catching , word_ctm_args [ j . name ] , ) word_procs . append ( word_p ) word_p . start ( ) phone_p = PhoneCtmProcessWorker ( j . name , to_process_queue , stopped , error_catching , phone_ctm_args [ j . name ] , ) phone_p . start ( ) phone_procs . append ( phone_p ) export_procs = [ ] for j in self . jobs : export_proc = ExportTextGridProcessWorker ( for_write_queue , stopped , finished_processing , textgrid_errors , export_args [ j . name ] , ) export_proc . start ( ) export_procs . append ( export_proc ) try : with tqdm . tqdm ( total = total_files ) as pbar : while True : try : w_p , intervals = to_process_queue . get ( timeout = 1 ) except Empty : for proc in word_procs : if not proc . finished_signal . stop_check ( ) : break for proc in phone_procs : if not proc . finished_signal . stop_check ( ) : break else : break continue to_process_queue . task_done ( ) if self . stopped . stop_check ( ) : self . logger . debug ( "Got stop check, exiting" ) continue utt = self . utterances [ intervals [ 0 ] . utterance ] if w_p == "word" : utt . add_word_intervals ( intervals ) else : utt . add_phone_intervals ( intervals ) file = self . files [ utt . file_name ] if file . is_fully_aligned : tiers = file . aligned_data output_path = file . construct_output_path ( self . textgrid_output , self . backup_output_directory ) duration = file . duration for_write_queue . put ( ( tiers , output_path , duration ) ) pbar . update ( 1 ) except Exception : stopped . stop ( ) while True : try : _ = to_process_queue . get ( timeout = 1 ) except Empty : for proc in word_procs : if not proc . finished_signal . stop_check ( ) : break for proc in phone_procs : if not proc . finished_signal . stop_check ( ) : break else : break continue to_process_queue . task_done ( ) exc_type , exc_value , exc_traceback = sys . exc_info ( ) error_catching [ "main" ] = "\n" . join ( traceback . format_exception ( exc_type , exc_value , exc_traceback ) ) finally : finished_processing . stop ( ) self . logger . debug ( "Waiting for processes to finish..." ) for i in range ( self . num_jobs ) : word_procs [ i ] . join ( ) phone_procs [ i ] . join ( ) to_process_queue . join ( ) for_write_queue . join ( ) for i in range ( self . num_jobs ) : export_procs [ i ] . join ( ) self . logger . debug ( f"Export took {time.time() - export_begin} seconds" ) if error_catching : self . logger . error ( "Error was encountered in processing CTMs" ) for key , error in error_catching . items ( ) : self . logger . error ( f"{key}:\n\n{error}" ) raise AlignmentExportError ( error_catching ) if textgrid_errors : self . logger . warning ( f"There were {len(textgrid_errors)} errors encountered in generating TextGrids. " f"Check the output_errors.txt file in {os.path.join(self.textgrid_output)} " f"for more details" ) output_textgrid_writing_errors ( self . textgrid_output , textgrid_errors )
Multiprocessing function for exporting alignment CTM information as TextGrids
See Also
Multiprocessing function for exporting alignment CTM information as TextGrids See Also
"""
        Multiprocessing function for exporting alignment CTM information as TextGrids

        See Also
        --------
        :class:`~montreal_forced_aligner.alignment.multiprocessing.AliToCtmFunction`
            Multiprocessing helper function for converting ali archives to CTM format
        :class:`~montreal_forced_aligner.alignment.multiprocessing.PhoneCtmProcessWorker`
            Multiprocessing helper class for processing CTM files
        :meth:`.CorpusAligner.phone_ctm_arguments`
            Job method for generating arguments for PhoneCtmProcessWorker
        :class:`~montreal_forced_aligner.alignment.multiprocessing.WordCtmProcessWorker`
            Multiprocessing helper class for processing word CTM files
        :meth:`.CorpusAligner.word_ctm_arguments`
            Job method for generating arguments for WordCtmProcessWorker
        :class:`~montreal_forced_aligner.alignment.multiprocessing.ExportTextGridProcessWorker`
            Multiprocessing helper class for exporting TextGrid files
        :meth:`.CorpusAligner.export_textgrid_arguments`
            Job method for generating arguments for ExportTextGridProcessWorker
        :kaldi_steps:`get_train_ctm`
            Reference Kaldi script

        """
self
class
`~montreal_forced_aligner.alignment.multiprocessing.AliToCtmFunction`
Multiprocessing helper function for converting ali archives to CTM format
` ~montreal_forced_aligner . alignment . multiprocessing . AliToCtmFunction ` Multiprocessing helper function for converting ali archives to CTM format
class
`~montreal_forced_aligner.alignment.multiprocessing.PhoneCtmProcessWorker`
Multiprocessing helper class for processing CTM files
` ~montreal_forced_aligner . alignment . multiprocessing . PhoneCtmProcessWorker ` Multiprocessing helper class for processing CTM files
meth
`.CorpusAligner.phone_ctm_arguments`
Job method for generating arguments for PhoneCtmProcessWorker
` . CorpusAligner . phone_ctm_arguments ` Job method for generating arguments for PhoneCtmProcessWorker
class
`~montreal_forced_aligner.alignment.multiprocessing.WordCtmProcessWorker`
Multiprocessing helper class for processing word CTM files
` ~montreal_forced_aligner . alignment . multiprocessing . WordCtmProcessWorker ` Multiprocessing helper class for processing word CTM files
meth
`.CorpusAligner.word_ctm_arguments`
Job method for generating arguments for WordCtmProcessWorker
` . CorpusAligner . word_ctm_arguments ` Job method for generating arguments for WordCtmProcessWorker
class
`~montreal_forced_aligner.alignment.multiprocessing.ExportTextGridProcessWorker`
Multiprocessing helper class for exporting TextGrid files
` ~montreal_forced_aligner . alignment . multiprocessing . ExportTextGridProcessWorker ` Multiprocessing helper class for exporting TextGrid files
meth
`.CorpusAligner.export_textgrid_arguments`
Job method for generating arguments for ExportTextGridProcessWorker
` . CorpusAligner . export_textgrid_arguments ` Job method for generating arguments for ExportTextGridProcessWorker
kaldi_steps
`get_train_ctm`
Reference Kaldi script
` get_train_ctm ` Reference Kaldi script