mmax2conll/mmax2conll/main.py
def dir_main(cls, input_dir, output_dir,
                 basedata_dir=c.WORDS_DIR,
                 markables_dir=c.MARKABLES_DIR,
                 allow_overwriting=c.ALLOW_OVERWRITING,
                 conll_extension=c.CONLL_EXTENSION,
                 words_files_extension=c.WORDS_FILES_EXTENSION,
                 coref_files_extension=c.COREF_FILES_EXTENSION,
                 sentences_files_extension=c.SENTENCES_FILES_EXTENSION,
                 log_on_error=c.LOG_ON_ERROR,
                 **kwargs):
        """
        Batch convert all files in a directory containing a `basedata_dir` and
        `markables_dir` directory as direct children.
        """
        basedata_dir = os.path.join(input_dir, basedata_dir)
        markables_dir = os.path.join(input_dir, markables_dir)

        words_files = {
            filename[:-len(words_files_extension)]
            for filename in os.listdir(basedata_dir)
            if filename.endswith(words_files_extension)
        }

        coref_files = {
            filename[:-len(coref_files_extension)]
            for filename in os.listdir(markables_dir)
            if filename.endswith(coref_files_extension)
        }

        if sentences_files_extension is not None:
            sentences_files = {
                filename[:-len(sentences_files_extension)]
                for filename in os.listdir(markables_dir)
                if filename.endswith(sentences_files_extension)
            }

        all_files = words_files & coref_files

        if words_files - coref_files:
            logger.warn(
                "The following files seem to be words files, but do not"
                " have corresponding coreference files:\n\t" + '\n\t'.join(
                    sorted(words_files - coref_files)
                )
            )

        if coref_files - words_files:
            logger.warn(
                "The following files seem to be coreference files, but do not"
                " have corresponding words files:\n\t" + '\n\t'.join(
                    sorted(coref_files - words_files)
                )
            )

        if sentences_files_extension is not None:
            all_files &= sentences_files
            if words_files - sentences_files:
                logger.warn(
                    "The following files seem to be words files, but do not"
                    " have corresponding sentences files:\n\t" + '\n\t'.join(
                        sorted(words_files - sentences_files)
                    )
                )

            if coref_files - sentences_files:
                logger.warn(
                    "The following files seem to be coreference files, but do"
                    " not have corresponding sentences files:\n\t" +
                    '\n\t'.join(
                        sorted(coref_files - sentences_files)
                    )
                )

            if sentences_files - words_files:
                logger.warn(
                    "The following files seem to be sentences files, but do"
                    " not have corresponding words files:\n\t"
                    + '\n\t'.join(
                        sorted(sentences_files - words_files)
                    )
                )

            if sentences_files - coref_files:
                logger.warn(
                    "The following files seem to be sentences files, but do"
                    " not have corresponding coreference files:\n\t"
                    + '\n\t'.join(
                        sorted(sentences_files - coref_files)
                    )
                )

        for name in sorted(all_files):
            words_file = os.path.join(basedata_dir, name) + \
                words_files_extension

            coref_file = os.path.join(markables_dir, name) + \
                coref_files_extension

            sentences_file = None \
                if sentences_files_extension is None \
                else os.path.join(markables_dir, name) \
                + sentences_files_extension

            output_file = os.path.join(output_dir, name) + conll_extension

            if os.path.exists(output_file):
                if allow_overwriting:
                    logger.warn(f"Overwriting {output_file}")
                else:
                    raise IOError(f"Will not overwrite: {output_file}")
            try:
                cls.single_main(
                    output_file,
                    words_file,
                    coref_file,
                    sentences_file,
                    words_files_extension=words_files_extension,
                    **kwargs
                )
            except Exception as e:
                if log_on_error:
                    logger.error(
                        f"{name} from {input_dir} is skipped: " + e.args[0]
                    )
                else:
                    e.args = (
                        f"While processing {name} from {input_dir}: " +
                        e.args[0],
                    ) + e.args[1:]
                    raise e

        Batch convert all files in a directory containing a `basedata_dir` and
        `markables_dir` directory as direct children.
        
Batch convert all files in a directory containing a `basedata_dir` and
`markables_dir` directory as direct children.
Batch convert all files in a directory containing a ` basedata_dir ` and ` markables_dir ` directory as direct children .
def dir_main(cls, input_dir, output_dir,
                 basedata_dir=c.WORDS_DIR,
                 markables_dir=c.MARKABLES_DIR,
                 allow_overwriting=c.ALLOW_OVERWRITING,
                 conll_extension=c.CONLL_EXTENSION,
                 words_files_extension=c.WORDS_FILES_EXTENSION,
                 coref_files_extension=c.COREF_FILES_EXTENSION,
                 sentences_files_extension=c.SENTENCES_FILES_EXTENSION,
                 log_on_error=c.LOG_ON_ERROR,
                 **kwargs):
        basedata_dir = os.path.join(input_dir, basedata_dir)
        markables_dir = os.path.join(input_dir, markables_dir)
        words_files = {
            filename[:-len(words_files_extension)]
            for filename in os.listdir(basedata_dir)
            if filename.endswith(words_files_extension)
        }
        coref_files = {
            filename[:-len(coref_files_extension)]
            for filename in os.listdir(markables_dir)
            if filename.endswith(coref_files_extension)
        }
        if sentences_files_extension is not None:
            sentences_files = {
                filename[:-len(sentences_files_extension)]
                for filename in os.listdir(markables_dir)
                if filename.endswith(sentences_files_extension)
            }
        all_files = words_files & coref_files
        if words_files - coref_files:
            logger.warn(
                "The following files seem to be words files, but do not"
                " have corresponding coreference files:\n\t" + '\n\t'.join(
                    sorted(words_files - coref_files)
                )
            )
        if coref_files - words_files:
            logger.warn(
                "The following files seem to be coreference files, but do not"
                " have corresponding words files:\n\t" + '\n\t'.join(
                    sorted(coref_files - words_files)
                )
            )
        if sentences_files_extension is not None:
            all_files &= sentences_files
            if words_files - sentences_files:
                logger.warn(
                    "The following files seem to be words files, but do not"
                    " have corresponding sentences files:\n\t" + '\n\t'.join(
                        sorted(words_files - sentences_files)
                    )
                )
            if coref_files - sentences_files:
                logger.warn(
                    "The following files seem to be coreference files, but do"
                    " not have corresponding sentences files:\n\t" +
                    '\n\t'.join(
                        sorted(coref_files - sentences_files)
                    )
                )
            if sentences_files - words_files:
                logger.warn(
                    "The following files seem to be sentences files, but do"
                    " not have corresponding words files:\n\t"
                    + '\n\t'.join(
                        sorted(sentences_files - words_files)
                    )
                )
            if sentences_files - coref_files:
                logger.warn(
                    "The following files seem to be sentences files, but do"
                    " not have corresponding coreference files:\n\t"
                    + '\n\t'.join(
                        sorted(sentences_files - coref_files)
                    )
                )
        for name in sorted(all_files):
            words_file = os.path.join(basedata_dir, name) + \
                words_files_extension
            coref_file = os.path.join(markables_dir, name) + \
                coref_files_extension
            sentences_file = None \
                if sentences_files_extension is None \
                else os.path.join(markables_dir, name) \
                + sentences_files_extension
            output_file = os.path.join(output_dir, name) + conll_extension
            if os.path.exists(output_file):
                if allow_overwriting:
                    logger.warn(f"Overwriting {output_file}")
                else:
                    raise IOError(f"Will not overwrite: {output_file}")
            try:
                cls.single_main(
                    output_file,
                    words_file,
                    coref_file,
                    sentences_file,
                    words_files_extension=words_files_extension,
                    **kwargs
                )
            except Exception as e:
                if log_on_error:
                    logger.error(
                        f"{name} from {input_dir} is skipped: " + e.args[0]
                    )
                else:
                    e.args = (
                        f"While processing {name} from {input_dir}: " +
                        e.args[0],
                    ) + e.args[1:]
                    raise e
def dir_main ( cls , input_dir , output_dir , basedata_dir = c . WORDS_DIR , markables_dir = c . MARKABLES_DIR , allow_overwriting = c . ALLOW_OVERWRITING , conll_extension = c . CONLL_EXTENSION , words_files_extension = c . WORDS_FILES_EXTENSION , coref_files_extension = c . COREF_FILES_EXTENSION , sentences_files_extension = c . SENTENCES_FILES_EXTENSION , log_on_error = c . LOG_ON_ERROR , ** kwargs ) : basedata_dir = os . path . join ( input_dir , basedata_dir ) markables_dir = os . path . join ( input_dir , markables_dir ) words_files = { filename [ : - len ( words_files_extension ) ] for filename in os . listdir ( basedata_dir ) if filename . endswith ( words_files_extension ) } coref_files = { filename [ : - len ( coref_files_extension ) ] for filename in os . listdir ( markables_dir ) if filename . endswith ( coref_files_extension ) } if sentences_files_extension is not None : sentences_files = { filename [ : - len ( sentences_files_extension ) ] for filename in os . listdir ( markables_dir ) if filename . endswith ( sentences_files_extension ) } all_files = words_files & coref_files if words_files - coref_files : logger . warn ( "The following files seem to be words files, but do not" " have corresponding coreference files:\n\t" + '\n\t' . join ( sorted ( words_files - coref_files ) ) ) if coref_files - words_files : logger . warn ( "The following files seem to be coreference files, but do not" " have corresponding words files:\n\t" + '\n\t' . join ( sorted ( coref_files - words_files ) ) ) if sentences_files_extension is not None : all_files &= sentences_files if words_files - sentences_files : logger . warn ( "The following files seem to be words files, but do not" " have corresponding sentences files:\n\t" + '\n\t' . join ( sorted ( words_files - sentences_files ) ) ) if coref_files - sentences_files : logger . warn ( "The following files seem to be coreference files, but do" " not have corresponding sentences files:\n\t" + '\n\t' . join ( sorted ( coref_files - sentences_files ) ) ) if sentences_files - words_files : logger . warn ( "The following files seem to be sentences files, but do" " not have corresponding words files:\n\t" + '\n\t' . join ( sorted ( sentences_files - words_files ) ) ) if sentences_files - coref_files : logger . warn ( "The following files seem to be sentences files, but do" " not have corresponding coreference files:\n\t" + '\n\t' . join ( sorted ( sentences_files - coref_files ) ) ) for name in sorted ( all_files ) : words_file = os . path . join ( basedata_dir , name ) + words_files_extension coref_file = os . path . join ( markables_dir , name ) + coref_files_extension sentences_file = None if sentences_files_extension is None else os . path . join ( markables_dir , name ) + sentences_files_extension output_file = os . path . join ( output_dir , name ) + conll_extension if os . path . exists ( output_file ) : if allow_overwriting : logger . warn ( f"Overwriting {output_file}" ) else : raise IOError ( f"Will not overwrite: {output_file}" ) try : cls . single_main ( output_file , words_file , coref_file , sentences_file , words_files_extension = words_files_extension , ** kwargs ) except Exception as e : if log_on_error : logger . error ( f"{name} from {input_dir} is skipped: " + e . args [ 0 ] ) else : e . args = ( f"While processing {name} from {input_dir}: " + e . args [ 0 ] , ) + e . args [ 1 : ] raise e
Batch convert all files in a directory containing a `basedata_dir` and
`markables_dir` directory as direct children.
Batch convert all files in a directory containing a ` basedata_dir ` and ` markables_dir ` directory as direct children .
"""
        Batch convert all files in a directory containing a `basedata_dir` and
        `markables_dir` directory as direct children.
        """
cls
input_dir
output_dir
basedata_dir
markables_dir
allow_overwriting
conll_extension
words_files_extension
coref_files_extension
sentences_files_extension
log_on_error