<NME> kraken.py
<BEF> #
# Copyright 2015 Benjamin Kiessling
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
"""
kraken.kraken
~~~~~~~~~~~~~

Command line drivers for recognition functionality.
"""
import os
import warnings
import logging
import pkg_resources

from typing import Dict, Union, List, cast, Any, IO, Callable
from rich.traceback import install
from functools import partial
from PIL import Image

import click

from kraken.lib import log
from kraken.lib.progress import KrakenProgressBar, KrakenDownloadProgressBar

warnings.simplefilter('ignore', UserWarning)

logging.captureWarnings(True)
logger = logging.getLogger('kraken')

# install rich traceback handler
install(suppress=[click])

APP_NAME = 'kraken'
SEGMENTATION_DEFAULT_MODEL = pkg_resources.resource_filename(__name__, 'blla.mlmodel')
DEFAULT_MODEL = ['en-default.mlmodel']
LEGACY_MODEL_DIR = '/usr/local/share/ocropus'

# raise default max image size to 20k * 20k pixels
Image.MAX_IMAGE_PIXELS = 20000 ** 2


def message(msg: str, **styles) -> None:
    if logger.getEffectiveLevel() >= 30:
        click.secho(msg, **styles)


def get_input_parser(type_str: str) -> Callable[[str], Dict[str, Any]]:
    if type_str == 'alto':
        from kraken.lib.xml import parse_alto
        return parse_alto
    elif type_str == 'page':
        from kraken.lib.xml import parse_page
        return parse_page
    elif type_str == 'xml':
        from kraken.lib.xml import parse_xml
        return parse_xml
    elif type_str == 'image':
        return Image.open


# chainable functions of functional components (binarization/segmentation/recognition)

def binarizer(threshold, zoom, escale, border, perc, range, low, high, input, output) -> None:
    from kraken import binarization

    ctx = click.get_current_context()
    if ctx.meta['first_process']:
        if ctx.meta['input_format_type'] != 'image':
            input = get_input_parser(ctx.meta['input_format_type'])(input)['image']
        ctx.meta['first_process'] = False
    else:
        raise click.UsageError('Binarization has to be the initial process.')

    try:
        im = Image.open(input)
    except IOError as e:
        raise click.BadParameter(str(e))
    message('Binarizing\t', nl=False)
    try:
        res = binarization.nlbin(im, threshold, zoom, escale, border, perc, range,
                                 low, high)
        if ctx.meta['last_process'] and ctx.meta['output_mode'] != 'native':
            with click.open_file(output, 'w', encoding='utf-8') as fp:
                fp = cast(IO[Any], fp)
                logger.info('Serializing as {} into {}'.format(ctx.meta['output_mode'], output))
                res.save(f'{output}.png')
                from kraken import serialization
                fp.write(serialization.serialize([],
                                                 image_name=f'{output}.png',
                                                 image_size=res.size,
                                                 template=ctx.meta['output_mode'],
                                                 processing_steps=ctx.meta['steps']))
        else:
            form = None
            ext = os.path.splitext(output)[1]
            if ext in ['.jpg', '.jpeg', '.JPG', '.JPEG', '']:
                form = 'png'
                if ext:
                    logger.warning('jpeg does not support 1bpp images. Forcing to png.')
            res.save(output, format=form)
        ctx.meta['base_image'] = output
    except Exception:
        if ctx.meta['raise_failed']:
            raise
        message('\u2717', fg='red')
        ctx.exit(1)
    message('\u2713', fg='green')


def segmenter(legacy, model, text_direction, scale, maxcolseps, black_colseps,
              remove_hlines, pad, mask, device, input, output) -> None:
    import json

    from kraken import pageseg
    from kraken import blla

    ctx = click.get_current_context()

    if ctx.meta['first_process']:
        if ctx.meta['input_format_type'] != 'image':
            input = get_input_parser(ctx.meta['input_format_type'])(input)['image']
        ctx.meta['first_process'] = False

    if 'base_image' not in ctx.meta:
        ctx.meta['base_image'] = input

    try:
        im = Image.open(input)
    except IOError as e:
        raise click.BadParameter(str(e))
    if mask:
        try:
            mask = Image.open(mask)
        except IOError as e:
            raise click.BadParameter(str(e))
    message('Segmenting\t', nl=False)
    try:
        if legacy:
            res = pageseg.segment(im,
                                  text_direction,
                                  scale,
                                  maxcolseps,
                                  black_colseps,
                                  no_hlines=remove_hlines,
                                  pad=pad,
                                  mask=mask)
        else:
            res = blla.segment(im, text_direction, mask=mask, model=model, device=device)
    except Exception:
        if ctx.meta['raise_failed']:
            raise
        message('\u2717', fg='red')
        ctx.exit(1)
    if ctx.meta['last_process'] and ctx.meta['output_mode'] != 'native':
        with click.open_file(output, 'w', encoding='utf-8') as fp:
            fp = cast(IO[Any], fp)
            logger.info('Serializing as {} into {}'.format(ctx.meta['output_mode'], output))
                raise click.UsageError(f'{input} invalid segmentation: {str(e)}')
    elif not bounds:
        if no_segmentation:
            print('loading no_seg')
            bounds = {'script_detection': False,
                      'text_direction': 'horizontal-lr',
                      'boxes': [(0, 0) + im.size]}
    else:
        with click.open_file(output, 'w') as fp:
            fp = cast(IO[Any], fp)
            json.dump(res, fp)
    message('\u2713', fg='green')


def recognizer(model, pad, no_segmentation, bidi_reordering, tags_ignore, input, output) -> None:

    import json

    from kraken import rpred

    ctx = click.get_current_context()

    bounds = None
    if 'base_image' not in ctx.meta:
        ctx.meta['base_image'] = input

    if ctx.meta['first_process']:
        if ctx.meta['input_format_type'] != 'image':
            doc = get_input_parser(ctx.meta['input_format_type'])(input)
            ctx.meta['base_image'] = doc['image']
            doc['text_direction'] = 'horizontal-lr'
            if doc['base_dir'] and bidi_reordering is True:
                message(f'Setting base text direction for BiDi reordering to {doc["base_dir"]} (from XML input file)')
                bidi_reordering = doc['base_dir']
            bounds = doc
    try:
        im = Image.open(ctx.meta['base_image'])
    except IOError as e:
        raise click.BadParameter(str(e))

    if not bounds and ctx.meta['base_image'] != input:
        with click.open_file(input, 'r') as fp:
            try:
                fp = cast(IO[Any], fp)
                bounds = json.load(fp)
            except ValueError as e:
                raise click.UsageError(f'{input} invalid segmentation: {str(e)}')
    elif not bounds:
        if no_segmentation:
            bounds = {'script_detection': False,
                      'text_direction': 'horizontal-lr',
                      'boxes': [(0, 0) + im.size]}
        else:
            raise click.UsageError('No line segmentation given. Add one with the input or run `segment` first.')
    elif no_segmentation:
        logger.warning('no_segmentation mode enabled but segmentation defined. Ignoring --no-segmentation option.')

    tags = set()
    # script detection
    if 'script_detection' in bounds and bounds['script_detection']:
        it = rpred.mm_rpred(model, im, bounds, pad,
                            bidi_reordering=bidi_reordering,
                            tags_ignore=tags_ignore)
    else:
        it = rpred.rpred(model['default'], im, bounds, pad,
                         bidi_reordering=bidi_reordering)

    preds = []

    with KrakenProgressBar() as progress:
        pred_task = progress.add_task('Processing', total=len(it), visible=True if not ctx.meta['verbose'] else False)
        for pred in it:
            preds.append(pred)
            progress.update(pred_task, advance=1)

    ctx = click.get_current_context()
    with click.open_file(output, 'w', encoding='utf-8') as fp:
        fp = cast(IO[Any], fp)
        message(f'Writing recognition results for {ctx.meta["orig_file"]}\t', nl=False)
        logger.info('Serializing as {} into {}'.format(ctx.meta['output_mode'], output))
        if ctx.meta['output_mode'] != 'native':
            from kraken import serialization
            fp.write(serialization.serialize(records=preds,
                                             image_name=ctx.meta['base_image'],
                                             image_size=Image.open(ctx.meta['base_image']).size,
                                             writing_mode=ctx.meta['text_direction'],
                                             scripts=tags,
                                             regions=bounds['regions'] if 'regions' in bounds else None,
                                             template=ctx.meta['output_mode'],
                                             processing_steps=ctx.meta['steps']))
        else:
            fp.write('\n'.join(s.prediction for s in preds))
        message('\u2713', fg='green')


@click.group(chain=True)
@click.version_option()
@click.option('-i', '--input',
              type=(click.Path(exists=True),  # type: ignore
                    click.Path(writable=True)),
              multiple=True,
              help='Input-output file pairs. Each input file (first argument) is mapped to one '
                   'output file (second argument), e.g. `-i input.png output.txt`')
@click.option('-I', '--batch-input', multiple=True, help='Glob expression to add multiple files at once.')
@click.option('-o', '--suffix', default='', show_default=True,
              help='Suffix for output files from batch and PDF inputs.')
@click.option('-v', '--verbose', default=0, count=True, show_default=True)
@click.option('-f', '--format-type', type=click.Choice(['image', 'alto', 'page', 'pdf', 'xml']), default='image',
              help='Sets the default input type. In image mode inputs are image '
                   'files, alto/page expects XML files in the respective format, pdf '
                   'expects PDF files with numbered suffixes added to output file '
                   'names as needed.')
@click.option('-p', '--pdf-format', default='{src}_{idx:06d}',
              show_default=True,
              help='Format for output of PDF files. valid fields '
                   'are `src` (source file), `idx` (page number), and `uuid` (v4 uuid). '
                   '`-o` suffixes are appended to this format string.')
@click.option('-h', '--hocr', 'serializer',
              help='Switch between hOCR, ALTO, abbyyXML, PageXML or "native" '
              'output. Native are plain image files for image, JSON for '
              'segmentation, and text for transcription output.',
              flag_value='hocr')
@click.option('-a', '--alto', 'serializer', flag_value='alto')
@click.option('-y', '--abbyy', 'serializer', flag_value='abbyyxml')
@click.option('-x', '--pagexml', 'serializer', flag_value='pagexml')
@click.option('-n', '--native', 'serializer', flag_value='native', default=True,
              show_default=True)
@click.option('-d', '--device', default='cpu', show_default=True,
              help='Select device to use (cpu, cuda:0, cuda:1, ...)')
@click.option('-r', '--raise-on-error/--no-raise-on-error', default=False, show_default=True,
              help='Raises the exception that caused processing to fail in the case of an error')
def cli(input, batch_input, suffix, verbose, format_type, pdf_format, serializer, device, raise_on_error):
    """
    Base command for recognition functionality.

    Inputs are defined as one or more pairs `-i input_file output_file`
    followed by one or more chainable processing commands. Likewise, verbosity
    is set on all subcommands with the `-v` switch.
    """
    ctx = click.get_current_context()
    if device != 'cpu':
        import torch
        try:
            torch.ones(1, device=device)
        except AssertionError as e:
            if raise_on_error:
                raise
            logger.error(f'Device {device} not available: {e.args[0]}.')
            ctx.exit(1)
    ctx.meta['device'] = device
    ctx.meta['input_format_type'] = format_type if format_type != 'pdf' else 'image'
    ctx.meta['raise_failed'] = raise_on_error
    ctx.meta['output_mode'] = serializer
    ctx.meta['verbose'] = verbose
    ctx.meta['steps'] = []
    log.set_logger(logger, level=30 - min(10 * verbose, 20))


@cli.result_callback()
def process_pipeline(subcommands, input, batch_input, suffix, verbose, format_type, pdf_format, **args):
    """
    Helper function calling the partials returned by each subcommand and
    placing their respective outputs in temporary files.
    """
    import glob
    import uuid
    import tempfile

    ctx = click.get_current_context()

    input = list(input)
    # expand batch inputs
    if batch_input and suffix:
        for batch_expr in batch_input:
            for in_file in glob.glob(batch_expr, recursive=True):
                input.append((in_file, '{}{}'.format(os.path.splitext(in_file)[0], suffix)))

    # parse pdfs
    if format_type == 'pdf':
        import pyvips

        if not batch_input:
            logger.warning('PDF inputs not added with batch option. Manual output filename will be ignored and `-o` utilized.')
        new_input = []
        num_pages = 0
        for (fpath, _) in input:
            doc = pyvips.Image.new_from_file(fpath, dpi=300, n=-1, access="sequential")
            if 'n-pages' in doc.get_fields():
                num_pages += doc.get('n-pages')

        with KrakenProgressBar() as progress:
            pdf_parse_task = progress.add_task('Extracting PDF pages', total=num_pages, visible=True if not ctx.meta['verbose'] else False)
            for (fpath, _) in input:
                try:
                    doc = pyvips.Image.new_from_file(fpath, dpi=300, n=-1, access="sequential")
                    if 'n-pages' not in doc.get_fields():
                        logger.warning('{fpath} does not contain pages. Skipping.')
                        continue
                    n_pages = doc.get('n-pages')

                    dest_dict = {'idx': -1, 'src': fpath, 'uuid': None}
                    for i in range(0, n_pages):
                        dest_dict['idx'] += 1
                        dest_dict['uuid'] = str(uuid.uuid4())
                        fd, filename = tempfile.mkstemp(suffix='.png')
                        os.close(fd)
                        doc = pyvips.Image.new_from_file(fpath, dpi=300, page=i, access="sequential")
                        logger.info(f'Saving temporary image {fpath}:{dest_dict["idx"]} to {filename}')
                        doc.write_to_file(filename)
                        new_input.append((filename, pdf_format.format(**dest_dict) + suffix))
                        progress.update(pdf_parse_task, advance=1)
                except pyvips.error.Error:
                    num_pages -= n_pages
                    progress.update(pdf_parse_task, total=num_pages)
                    logger.warning(f'{fpath} is not a PDF file. Skipping.')
        input = new_input
        ctx.meta['steps'].insert(0, {'category': 'preprocessing', 'description': 'PDF image extraction', 'settings': {}})

    for io_pair in input:
        ctx.meta['first_process'] = True
        ctx.meta['last_process'] = False
        ctx.meta['orig_file'] = io_pair[0]
        if 'base_image' in ctx.meta:
            del ctx.meta['base_image']
        try:
            tmps = [tempfile.mkstemp() for _ in subcommands[1:]]
            for tmp in tmps:
                os.close(tmp[0])
            fc = [io_pair[0]] + [tmp[1] for tmp in tmps] + [io_pair[1]]
            for idx, (task, input, output) in enumerate(zip(subcommands, fc, fc[1:])):
                if len(fc) - 2 == idx:
                    ctx.meta['last_process'] = True
                task(input=input, output=output)
        except Exception as e:
            logger.error(f'Failed processing {io_pair[0]}: {str(e)}')
            if ctx.meta['raise_failed']:
                raise
        finally:
            for f in fc[1:-1]:
                os.unlink(f)
            # clean up temporary PDF image files
            if format_type == 'pdf':
                logger.debug(f'unlinking {fc[0]}')
                os.unlink(fc[0])


@cli.command('binarize')
@click.pass_context
@click.option('--threshold', show_default=True, default=0.5, type=click.FLOAT)
@click.option('--zoom', show_default=True, default=0.5, type=click.FLOAT)
@click.option('--escale', show_default=True, default=1.0, type=click.FLOAT)
@click.option('--border', show_default=True, default=0.1, type=click.FLOAT)
@click.option('--perc', show_default=True, default=80, type=click.IntRange(1, 100))
@click.option('--range', show_default=True, default=20, type=click.INT)
@click.option('--low', show_default=True, default=5, type=click.IntRange(1, 100))
@click.option('--high', show_default=True, default=90, type=click.IntRange(1, 100))
def binarize(ctx, threshold, zoom, escale, border, perc, range, low, high):
    """
    Binarizes page images.
    """
    ctx.meta['steps'].append({'category': 'preprocessing',
                              'description': 'Image binarization',
                              'settings': {'threshold': threshold,
                                           'zoom': zoom,
                                           'escale': escale,
                                           'border': border,
                                           'perc': perc,
                                           'range': range,
                                           'low': low,
                                           'high': high}})

    return partial(binarizer, threshold, zoom, escale, border, perc, range,
                   low, high)


@cli.command('segment')
@click.pass_context
@click.option('-i', '--model',
              default=None,
              show_default=True, type=click.Path(exists=True),
              help='Baseline detection model to use')
@click.option('-x/-bl', '--boxes/--baseline', default=True, show_default=True,
              help='Switch between legacy box segmenter and neural baseline segmenter')
@click.option('-d', '--text-direction', default='horizontal-lr',
              show_default=True,
              type=click.Choice(['horizontal-lr', 'horizontal-rl',
                                 'vertical-lr', 'vertical-rl']),
              help='Sets principal text direction')
@click.option('--scale', show_default=True, default=None, type=click.FLOAT)
@click.option('-m', '--maxcolseps', show_default=True, default=2, type=click.INT)
@click.option('-b/-w', '--black-colseps/--white_colseps', show_default=True, default=False)
@click.option('-r/-l', '--remove_hlines/--hlines', show_default=True, default=True)
@click.option('-p', '--pad', show_default=True, type=(int, int), default=(0, 0),
              help='Left and right padding around lines')
@click.option('-m', '--mask', show_default=True, default=None,
              type=click.File(mode='rb', lazy=True), help='Segmentation mask '
              'suppressing page areas for line detection. 0-valued image '
              'regions are ignored for segmentation purposes. Disables column '
              'detection.')
def segment(ctx, model, boxes, text_direction, scale, maxcolseps,
            black_colseps, remove_hlines, pad, mask):
    """
    Segments page images into text lines.
    """
    if model and boxes:
        logger.warning(f'Baseline model ({model}) given but legacy segmenter selected. Forcing to -bl.')
        boxes = False

    if boxes is False:
        if not model:
            model = SEGMENTATION_DEFAULT_MODEL
        ctx.meta['steps'].append({'category': 'processing',
                                  'description': 'Baseline and region segmentation',
                                  'settings': {'model': os.path.basename(model),
                                               'text_direction': text_direction}})

        from kraken.lib.vgsl import TorchVGSLModel
        message(f'Loading ANN {model}\t', nl=False)
        try:
            model = TorchVGSLModel.load_model(model)
            model.to(ctx.meta['device'])
        except Exception:
            if ctx.meta['raise_failed']:
                raise
            message('\u2717', fg='red')
            ctx.exit(1)

        message('\u2713', fg='green')
    else:
        ctx.meta['steps'].append({'category': 'processing',
                                  'description': 'bounding box segmentation',
                                  'settings': {'text_direction': text_direction,
                                               'scale': scale,
                                               'maxcolseps': maxcolseps,
                                               'black_colseps': black_colseps,
                                               'remove_hlines': remove_hlines,
                                               'pad': pad}})

    return partial(segmenter, boxes, model, text_direction, scale, maxcolseps,
                   black_colseps, remove_hlines, pad, mask, ctx.meta['device'])


def _validate_mm(ctx, param, value):
    """
    Maps model mappings to a dictionary.
    """
    model_dict = {'ignore': []}  # type: Dict[str, Union[str, List[str]]]
    if len(value) == 1 and len(value[0].split(':')) == 1:
        model_dict['default'] = value[0]
        return model_dict
    try:
        for m in value:
            k, v = m.split(':')
            if v == 'ignore':
                model_dict['ignore'].append(k)  # type: ignore
            else:
                model_dict[k] = os.path.expanduser(v)
    except Exception:
        raise click.BadParameter('Mappings must be in format tag:model')
    return model_dict


@cli.command('ocr')
@click.pass_context
@click.option('-m', '--model', default=DEFAULT_MODEL, multiple=True,
              show_default=True, callback=_validate_mm,
              help='Path to an recognition model or mapping of the form '
              '$tag1:$model1. Add multiple mappings to run multi-model '
              'recognition based on detected tags. Use the default keyword '
              'for adding a catch-all model. Recognition on tags can be '
              'ignored with the model value ignore.')
@click.option('-p', '--pad', show_default=True, type=click.INT, default=16, help='Left and right '
              'padding around lines')
@click.option('-n', '--reorder/--no-reorder', show_default=True, default=True,
              help='Reorder code points to logical order')
@click.option('--base-dir', show_default=True, default='auto',
              type=click.Choice(['L', 'R', 'auto']), help='Set base text '
              'direction.  This should be set to the direction used during the '
              'creation of the training data. If set to `auto` it will be '
              'overridden by any explicit value given in the input files.')
@click.option('-s', '--no-segmentation', default=False, show_default=True, is_flag=True,
              help='Enables non-segmentation mode treating each input image as a whole line.')
@click.option('-d', '--text-direction', default='horizontal-tb',
              show_default=True,
              type=click.Choice(['horizontal-tb', 'vertical-lr', 'vertical-rl']),
              help='Sets principal text direction in serialization output')
@click.option('--threads', default=1, show_default=True, type=click.IntRange(1),
              help='Number of threads to use for OpenMP parallelization.')
def ocr(ctx, model, pad, reorder, base_dir, no_segmentation, text_direction, threads):
    """
    Recognizes text in line images.
    """
    from kraken.lib import models

    if ctx.meta['input_format_type'] != 'image' and no_segmentation:
        raise click.BadParameter('no_segmentation mode is incompatible with page/alto inputs')

    if reorder and base_dir != 'auto':
        reorder = base_dir

    # first we try to find the model in the absolue path, then ~/.kraken, then
    # LEGACY_MODEL_DIR
    nm = {}  # type: Dict[str, models.TorchSeqRecognizer]
    ign_tags = model.pop('ignore')
    for k, v in model.items():
        search = [v,
                  os.path.join(click.get_app_dir(APP_NAME), v),
                  os.path.join(LEGACY_MODEL_DIR, v)]
        location = None
        for loc in search:
            if os.path.isfile(loc):
                location = loc
                break
        if not location:
            raise click.BadParameter(f'No model for {v} found')
        message(f'Loading ANN {v}\t', nl=False)
        try:
            rnn = models.load_any(location, device=ctx.meta['device'])
            nm[k] = rnn
        except Exception:
            if ctx.meta['raise_failed']:
                raise
            message('\u2717', fg='red')
            ctx.exit(1)
        message('\u2713', fg='green')

    if 'default' in nm:
        from collections import defaultdict

        nn = defaultdict(lambda: nm['default'])  # type: Dict[str, models.TorchSeqRecognizer]
        nn.update(nm)
        nm = nn
    # thread count is global so setting it once is sufficient
    nm[k].nn.set_num_threads(threads)

    ctx.meta['steps'].append({'category': 'processing',
                              'description': 'Text line recognition',
                              'settings': {'text_direction': text_direction,
                                           'models': ' '.join(os.path.basename(v) for v in model.values()),
                                           'pad': pad,
                                           'bidi_reordering': reorder}})

    # set output mode
    ctx.meta['text_direction'] = text_direction
    return partial(recognizer,
                   model=nm,
                   pad=pad,
                   no_segmentation=no_segmentation,
                   bidi_reordering=reorder,
                   tags_ignore=ign_tags)


@cli.command('show')
@click.pass_context
@click.argument('model_id')
def show(ctx, model_id):
    """
    Retrieves model metadata from the repository.
    """
    from kraken import repo
    from kraken.lib.util import make_printable, is_printable

    desc = repo.get_description(model_id)

    chars = []
    combining = []
    for char in sorted(desc['graphemes']):
        if not is_printable(char):
            combining.append(make_printable(char))
        else:
            chars.append(char)
    message(
        'name: {}\n\n{}\n\n{}\nscripts: {}\nalphabet: {} {}\naccuracy: {:.2f}%\nlicense: {}\nauthor(s): {}\ndate: {}'.format(
            model_id, desc['summary'], desc['description'], ' '.join(
                desc['script']), ''.join(chars), ', '.join(combining), desc['accuracy'], desc['license']['id'], '; '.join(
                x['name'] for x in desc['creators']), desc['publication_date']))
    ctx.exit(0)


@cli.command('list')
@click.pass_context
def list_models(ctx):
    """
    Lists models in the repository.
    """
    from kraken import repo

    with KrakenProgressBar() as progress:
        download_task = progress.add_task('Retrieving model list', total=0, visible=True if not ctx.meta['verbose'] else False)
        model_list = repo.get_listing(lambda total, advance: progress.update(download_task, total=total, advance=advance))
    for id, metadata in model_list.items():
        message('{} ({}) - {}'.format(id, ', '.join(metadata['type']), metadata['summary']))
    ctx.exit(0)


@cli.command('get')
@click.pass_context
@click.argument('model_id')
def get(ctx, model_id):
    """
    Retrieves a model from the repository.
    """
    from kraken import repo

    try:
        os.makedirs(click.get_app_dir(APP_NAME))
    except OSError:
        pass

    with KrakenDownloadProgressBar() as progress:
        download_task = progress.add_task('Processing', total=0, visible=True if not ctx.meta['verbose'] else False)
        filename = repo.get_model(model_id, click.get_app_dir(APP_NAME),
                                  lambda total, advance: progress.update(download_task, total=total, advance=advance))
    message(f'Model name: {filename}')
    ctx.exit(0)


if __name__ == '__main__':
    cli()

<MSG> forgot to remove debug printf

<DFF> @@ -169,7 +169,6 @@ def recognizer(model, pad, no_segmentation, bidi_reordering, script_ignore, inpu
                 raise click.UsageError(f'{input} invalid segmentation: {str(e)}')
     elif not bounds:
         if no_segmentation:
-            print('loading no_seg')
             bounds = {'script_detection': False,
                       'text_direction': 'horizontal-lr',
                       'boxes': [(0, 0) + im.size]}

{'commit': "diff --git a/kraken/kraken.py b/kraken/kraken.py\nindex f576950..8226c38 100644\n--- a/kraken/kraken.py\n+++ b/kraken/kraken.py\n@@ -169,7 +169,6 @@ def recognizer(model, pad, no_segmentation, bidi_reordering, script_ignore, inpu\n                 raise click.UsageError(f'{input} invalid segmentation: {str(e)}')\n     elif not bounds:\n         if no_segmentation:\n-            print('loading no_seg')\n             bounds = {'script_detection': False,\n                       'text_direction': 'horizontal-lr',\n                       'boxes': [(0, 0) + im.size]}\n", 'commit_subject': 'forgot to remove debug printf', 'repo_name': 'mittagessen/kraken', 'committer': {'name': 'Benjamin Kiessling', 'email': '1e6b548472e82bd68375af6d81d3d89f213c84e7@l.unchti.me', 'time_sec': '1587477572', 'tz_offset': '120', 'date': {'seconds': '1587477572'}}, 'commit_tree': '3d74b202ead1747fd00676b9582f6a16a56a9d60', 'commit_message': 'forgot to remove debug printf\n', 'language_name': 'JavaScript', 'license': 'apache-2.0', 'hunks': "@@ -169,7 +169,6 @@ def recognizer(model, pad, no_segmentation, bidi_reordering, script_ignore, inpu\n                 raise click.UsageError(f'{input} invalid segmentation: {str(e)}')\n     elif not bounds:\n         if no_segmentation:\n-            print('loading no_seg')\n             bounds = {'script_detection': False,\n                       'text_direction': 'horizontal-lr',\n                       'boxes': [(0, 0) + im.size]}\n", 'addition_count': 0, 'deletion_count': 1, 'src_file': 'a/kraken/kraken.py', 'tgt_file': 'b/kraken/kraken.py', 'file_extension': '.py', 'before_file': '#\n# Copyright 2015 Benjamin Kiessling\n#\n# Licensed under the Apache License, Version 2.0 (the "License");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an "AS IS" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express\n# or implied. See the License for the specific language governing\n# permissions and limitations under the License.\n"""\nkraken.kraken\n~~~~~~~~~~~~~\n\nCommand line drivers for recognition functionality.\n"""\nimport os\nimport warnings\nimport logging\nimport pkg_resources\n\nfrom typing import Dict, Union, List, cast, Any, IO, Callable\nfrom rich.traceback import install\nfrom functools import partial\nfrom PIL import Image\n\nimport click\n\nfrom kraken.lib import log\nfrom kraken.lib.progress import KrakenProgressBar, KrakenDownloadProgressBar\n\nwarnings.simplefilter(\'ignore\', UserWarning)\n\nlogging.captureWarnings(True)\nlogger = logging.getLogger(\'kraken\')\n\n# install rich traceback handler\ninstall(suppress=[click])\n\nAPP_NAME = \'kraken\'\nSEGMENTATION_DEFAULT_MODEL = pkg_resources.resource_filename(__name__, \'blla.mlmodel\')\nDEFAULT_MODEL = [\'en-default.mlmodel\']\nLEGACY_MODEL_DIR = \'/usr/local/share/ocropus\'\n\n# raise default max image size to 20k * 20k pixels\nImage.MAX_IMAGE_PIXELS = 20000 ** 2\n\n\ndef message(msg: str, **styles) -> None:\n    if logger.getEffectiveLevel() >= 30:\n        click.secho(msg, **styles)\n\n\ndef get_input_parser(type_str: str) -> Callable[[str], Dict[str, Any]]:\n    if type_str == \'alto\':\n        from kraken.lib.xml import parse_alto\n        return parse_alto\n    elif type_str == \'page\':\n        from kraken.lib.xml import parse_page\n        return parse_page\n    elif type_str == \'xml\':\n        from kraken.lib.xml import parse_xml\n        return parse_xml\n    elif type_str == \'image\':\n        return Image.open\n\n\n# chainable functions of functional components (binarization/segmentation/recognition)\n\ndef binarizer(threshold, zoom, escale, border, perc, range, low, high, input, output) -> None:\n    from kraken import binarization\n\n    ctx = click.get_current_context()\n    if ctx.meta[\'first_process\']:\n        if ctx.meta[\'input_format_type\'] != \'image\':\n            input = get_input_parser(ctx.meta[\'input_format_type\'])(input)[\'image\']\n        ctx.meta[\'first_process\'] = False\n    else:\n        raise click.UsageError(\'Binarization has to be the initial process.\')\n\n    try:\n        im = Image.open(input)\n    except IOError as e:\n        raise click.BadParameter(str(e))\n    message(\'Binarizing\\t\', nl=False)\n    try:\n        res = binarization.nlbin(im, threshold, zoom, escale, border, perc, range,\n                                 low, high)\n        if ctx.meta[\'last_process\'] and ctx.meta[\'output_mode\'] != \'native\':\n            with click.open_file(output, \'w\', encoding=\'utf-8\') as fp:\n                fp = cast(IO[Any], fp)\n                logger.info(\'Serializing as {} into {}\'.format(ctx.meta[\'output_mode\'], output))\n                res.save(f\'{output}.png\')\n                from kraken import serialization\n                fp.write(serialization.serialize([],\n                                                 image_name=f\'{output}.png\',\n                                                 image_size=res.size,\n                                                 template=ctx.meta[\'output_mode\'],\n                                                 processing_steps=ctx.meta[\'steps\']))\n        else:\n            form = None\n            ext = os.path.splitext(output)[1]\n            if ext in [\'.jpg\', \'.jpeg\', \'.JPG\', \'.JPEG\', \'\']:\n                form = \'png\'\n                if ext:\n                    logger.warning(\'jpeg does not support 1bpp images. Forcing to png.\')\n            res.save(output, format=form)\n        ctx.meta[\'base_image\'] = output\n    except Exception:\n        if ctx.meta[\'raise_failed\']:\n            raise\n        message(\'\\u2717\', fg=\'red\')\n        ctx.exit(1)\n    message(\'\\u2713\', fg=\'green\')\n\n\ndef segmenter(legacy, model, text_direction, scale, maxcolseps, black_colseps,\n              remove_hlines, pad, mask, device, input, output) -> None:\n    import json\n\n    from kraken import pageseg\n    from kraken import blla\n\n    ctx = click.get_current_context()\n\n    if ctx.meta[\'first_process\']:\n        if ctx.meta[\'input_format_type\'] != \'image\':\n            input = get_input_parser(ctx.meta[\'input_format_type\'])(input)[\'image\']\n        ctx.meta[\'first_process\'] = False\n\n    if \'base_image\' not in ctx.meta:\n        ctx.meta[\'base_image\'] = input\n\n    try:\n        im = Image.open(input)\n    except IOError as e:\n        raise click.BadParameter(str(e))\n    if mask:\n        try:\n            mask = Image.open(mask)\n        except IOError as e:\n            raise click.BadParameter(str(e))\n    message(\'Segmenting\\t\', nl=False)\n    try:\n        if legacy:\n            res = pageseg.segment(im,\n                                  text_direction,\n                                  scale,\n                                  maxcolseps,\n                                  black_colseps,\n                                  no_hlines=remove_hlines,\n                                  pad=pad,\n                                  mask=mask)\n        else:\n            res = blla.segment(im, text_direction, mask=mask, model=model, device=device)\n    except Exception:\n        if ctx.meta[\'raise_failed\']:\n            raise\n        message(\'\\u2717\', fg=\'red\')\n        ctx.exit(1)\n    if ctx.meta[\'last_process\'] and ctx.meta[\'output_mode\'] != \'native\':\n        with click.open_file(output, \'w\', encoding=\'utf-8\') as fp:\n            fp = cast(IO[Any], fp)\n            logger.info(\'Serializing as {} into {}\'.format(ctx.meta[\'output_mode\'], output))\n                raise click.UsageError(f\'{input} invalid segmentation: {str(e)}\')\n    elif not bounds:\n        if no_segmentation:\n            print(\'loading no_seg\')\n            bounds = {\'script_detection\': False,\n                      \'text_direction\': \'horizontal-lr\',\n                      \'boxes\': [(0, 0) + im.size]}\n    else:\n        with click.open_file(output, \'w\') as fp:\n            fp = cast(IO[Any], fp)\n            json.dump(res, fp)\n    message(\'\\u2713\', fg=\'green\')\n\n\ndef recognizer(model, pad, no_segmentation, bidi_reordering, tags_ignore, input, output) -> None:\n\n    import json\n\n    from kraken import rpred\n\n    ctx = click.get_current_context()\n\n    bounds = None\n    if \'base_image\' not in ctx.meta:\n        ctx.meta[\'base_image\'] = input\n\n    if ctx.meta[\'first_process\']:\n        if ctx.meta[\'input_format_type\'] != \'image\':\n            doc = get_input_parser(ctx.meta[\'input_format_type\'])(input)\n            ctx.meta[\'base_image\'] = doc[\'image\']\n            doc[\'text_direction\'] = \'horizontal-lr\'\n            if doc[\'base_dir\'] and bidi_reordering is True:\n                message(f\'Setting base text direction for BiDi reordering to {doc["base_dir"]} (from XML input file)\')\n                bidi_reordering = doc[\'base_dir\']\n            bounds = doc\n    try:\n        im = Image.open(ctx.meta[\'base_image\'])\n    except IOError as e:\n        raise click.BadParameter(str(e))\n\n    if not bounds and ctx.meta[\'base_image\'] != input:\n        with click.open_file(input, \'r\') as fp:\n            try:\n                fp = cast(IO[Any], fp)\n                bounds = json.load(fp)\n            except ValueError as e:\n                raise click.UsageError(f\'{input} invalid segmentation: {str(e)}\')\n    elif not bounds:\n        if no_segmentation:\n            bounds = {\'script_detection\': False,\n                      \'text_direction\': \'horizontal-lr\',\n                      \'boxes\': [(0, 0) + im.size]}\n        else:\n            raise click.UsageError(\'No line segmentation given. Add one with the input or run `segment` first.\')\n    elif no_segmentation:\n        logger.warning(\'no_segmentation mode enabled but segmentation defined. Ignoring --no-segmentation option.\')\n\n    tags = set()\n    # script detection\n    if \'script_detection\' in bounds and bounds[\'script_detection\']:\n        it = rpred.mm_rpred(model, im, bounds, pad,\n                            bidi_reordering=bidi_reordering,\n                            tags_ignore=tags_ignore)\n    else:\n        it = rpred.rpred(model[\'default\'], im, bounds, pad,\n                         bidi_reordering=bidi_reordering)\n\n    preds = []\n\n    with KrakenProgressBar() as progress:\n        pred_task = progress.add_task(\'Processing\', total=len(it), visible=True if not ctx.meta[\'verbose\'] else False)\n        for pred in it:\n            preds.append(pred)\n            progress.update(pred_task, advance=1)\n\n    ctx = click.get_current_context()\n    with click.open_file(output, \'w\', encoding=\'utf-8\') as fp:\n        fp = cast(IO[Any], fp)\n        message(f\'Writing recognition results for {ctx.meta["orig_file"]}\\t\', nl=False)\n        logger.info(\'Serializing as {} into {}\'.format(ctx.meta[\'output_mode\'], output))\n        if ctx.meta[\'output_mode\'] != \'native\':\n            from kraken import serialization\n            fp.write(serialization.serialize(records=preds,\n                                             image_name=ctx.meta[\'base_image\'],\n                                             image_size=Image.open(ctx.meta[\'base_image\']).size,\n                                             writing_mode=ctx.meta[\'text_direction\'],\n                                             scripts=tags,\n                                             regions=bounds[\'regions\'] if \'regions\' in bounds else None,\n                                             template=ctx.meta[\'output_mode\'],\n                                             processing_steps=ctx.meta[\'steps\']))\n        else:\n            fp.write(\'\\n\'.join(s.prediction for s in preds))\n        message(\'\\u2713\', fg=\'green\')\n\n\n@click.group(chain=True)\n@click.version_option()\n@click.option(\'-i\', \'--input\',\n              type=(click.Path(exists=True),  # type: ignore\n                    click.Path(writable=True)),\n              multiple=True,\n              help=\'Input-output file pairs. Each input file (first argument) is mapped to one \'\n                   \'output file (second argument), e.g. `-i input.png output.txt`\')\n@click.option(\'-I\', \'--batch-input\', multiple=True, help=\'Glob expression to add multiple files at once.\')\n@click.option(\'-o\', \'--suffix\', default=\'\', show_default=True,\n              help=\'Suffix for output files from batch and PDF inputs.\')\n@click.option(\'-v\', \'--verbose\', default=0, count=True, show_default=True)\n@click.option(\'-f\', \'--format-type\', type=click.Choice([\'image\', \'alto\', \'page\', \'pdf\', \'xml\']), default=\'image\',\n              help=\'Sets the default input type. In image mode inputs are image \'\n                   \'files, alto/page expects XML files in the respective format, pdf \'\n                   \'expects PDF files with numbered suffixes added to output file \'\n                   \'names as needed.\')\n@click.option(\'-p\', \'--pdf-format\', default=\'{src}_{idx:06d}\',\n              show_default=True,\n              help=\'Format for output of PDF files. valid fields \'\n                   \'are `src` (source file), `idx` (page number), and `uuid` (v4 uuid). \'\n                   \'`-o` suffixes are appended to this format string.\')\n@click.option(\'-h\', \'--hocr\', \'serializer\',\n              help=\'Switch between hOCR, ALTO, abbyyXML, PageXML or "native" \'\n              \'output. Native are plain image files for image, JSON for \'\n              \'segmentation, and text for transcription output.\',\n              flag_value=\'hocr\')\n@click.option(\'-a\', \'--alto\', \'serializer\', flag_value=\'alto\')\n@click.option(\'-y\', \'--abbyy\', \'serializer\', flag_value=\'abbyyxml\')\n@click.option(\'-x\', \'--pagexml\', \'serializer\', flag_value=\'pagexml\')\n@click.option(\'-n\', \'--native\', \'serializer\', flag_value=\'native\', default=True,\n              show_default=True)\n@click.option(\'-d\', \'--device\', default=\'cpu\', show_default=True,\n              help=\'Select device to use (cpu, cuda:0, cuda:1, ...)\')\n@click.option(\'-r\', \'--raise-on-error/--no-raise-on-error\', default=False, show_default=True,\n              help=\'Raises the exception that caused processing to fail in the case of an error\')\ndef cli(input, batch_input, suffix, verbose, format_type, pdf_format, serializer, device, raise_on_error):\n    """\n    Base command for recognition functionality.\n\n    Inputs are defined as one or more pairs `-i input_file output_file`\n    followed by one or more chainable processing commands. Likewise, verbosity\n    is set on all subcommands with the `-v` switch.\n    """\n    ctx = click.get_current_context()\n    if device != \'cpu\':\n        import torch\n        try:\n            torch.ones(1, device=device)\n        except AssertionError as e:\n            if raise_on_error:\n                raise\n            logger.error(f\'Device {device} not available: {e.args[0]}.\')\n            ctx.exit(1)\n    ctx.meta[\'device\'] = device\n    ctx.meta[\'input_format_type\'] = format_type if format_type != \'pdf\' else \'image\'\n    ctx.meta[\'raise_failed\'] = raise_on_error\n    ctx.meta[\'output_mode\'] = serializer\n    ctx.meta[\'verbose\'] = verbose\n    ctx.meta[\'steps\'] = []\n    log.set_logger(logger, level=30 - min(10 * verbose, 20))\n\n\n@cli.result_callback()\ndef process_pipeline(subcommands, input, batch_input, suffix, verbose, format_type, pdf_format, **args):\n    """\n    Helper function calling the partials returned by each subcommand and\n    placing their respective outputs in temporary files.\n    """\n    import glob\n    import uuid\n    import tempfile\n\n    ctx = click.get_current_context()\n\n    input = list(input)\n    # expand batch inputs\n    if batch_input and suffix:\n        for batch_expr in batch_input:\n            for in_file in glob.glob(batch_expr, recursive=True):\n                input.append((in_file, \'{}{}\'.format(os.path.splitext(in_file)[0], suffix)))\n\n    # parse pdfs\n    if format_type == \'pdf\':\n        import pyvips\n\n        if not batch_input:\n            logger.warning(\'PDF inputs not added with batch option. Manual output filename will be ignored and `-o` utilized.\')\n        new_input = []\n        num_pages = 0\n        for (fpath, _) in input:\n            doc = pyvips.Image.new_from_file(fpath, dpi=300, n=-1, access="sequential")\n            if \'n-pages\' in doc.get_fields():\n                num_pages += doc.get(\'n-pages\')\n\n        with KrakenProgressBar() as progress:\n            pdf_parse_task = progress.add_task(\'Extracting PDF pages\', total=num_pages, visible=True if not ctx.meta[\'verbose\'] else False)\n            for (fpath, _) in input:\n                try:\n                    doc = pyvips.Image.new_from_file(fpath, dpi=300, n=-1, access="sequential")\n                    if \'n-pages\' not in doc.get_fields():\n                        logger.warning(\'{fpath} does not contain pages. Skipping.\')\n                        continue\n                    n_pages = doc.get(\'n-pages\')\n\n                    dest_dict = {\'idx\': -1, \'src\': fpath, \'uuid\': None}\n                    for i in range(0, n_pages):\n                        dest_dict[\'idx\'] += 1\n                        dest_dict[\'uuid\'] = str(uuid.uuid4())\n                        fd, filename = tempfile.mkstemp(suffix=\'.png\')\n                        os.close(fd)\n                        doc = pyvips.Image.new_from_file(fpath, dpi=300, page=i, access="sequential")\n                        logger.info(f\'Saving temporary image {fpath}:{dest_dict["idx"]} to {filename}\')\n                        doc.write_to_file(filename)\n                        new_input.append((filename, pdf_format.format(**dest_dict) + suffix))\n                        progress.update(pdf_parse_task, advance=1)\n                except pyvips.error.Error:\n                    num_pages -= n_pages\n                    progress.update(pdf_parse_task, total=num_pages)\n                    logger.warning(f\'{fpath} is not a PDF file. Skipping.\')\n        input = new_input\n        ctx.meta[\'steps\'].insert(0, {\'category\': \'preprocessing\', \'description\': \'PDF image extraction\', \'settings\': {}})\n\n    for io_pair in input:\n        ctx.meta[\'first_process\'] = True\n        ctx.meta[\'last_process\'] = False\n        ctx.meta[\'orig_file\'] = io_pair[0]\n        if \'base_image\' in ctx.meta:\n            del ctx.meta[\'base_image\']\n        try:\n            tmps = [tempfile.mkstemp() for _ in subcommands[1:]]\n            for tmp in tmps:\n                os.close(tmp[0])\n            fc = [io_pair[0]] + [tmp[1] for tmp in tmps] + [io_pair[1]]\n            for idx, (task, input, output) in enumerate(zip(subcommands, fc, fc[1:])):\n                if len(fc) - 2 == idx:\n                    ctx.meta[\'last_process\'] = True\n                task(input=input, output=output)\n        except Exception as e:\n            logger.error(f\'Failed processing {io_pair[0]}: {str(e)}\')\n            if ctx.meta[\'raise_failed\']:\n                raise\n        finally:\n            for f in fc[1:-1]:\n                os.unlink(f)\n            # clean up temporary PDF image files\n            if format_type == \'pdf\':\n                logger.debug(f\'unlinking {fc[0]}\')\n                os.unlink(fc[0])\n\n\n@cli.command(\'binarize\')\n@click.pass_context\n@click.option(\'--threshold\', show_default=True, default=0.5, type=click.FLOAT)\n@click.option(\'--zoom\', show_default=True, default=0.5, type=click.FLOAT)\n@click.option(\'--escale\', show_default=True, default=1.0, type=click.FLOAT)\n@click.option(\'--border\', show_default=True, default=0.1, type=click.FLOAT)\n@click.option(\'--perc\', show_default=True, default=80, type=click.IntRange(1, 100))\n@click.option(\'--range\', show_default=True, default=20, type=click.INT)\n@click.option(\'--low\', show_default=True, default=5, type=click.IntRange(1, 100))\n@click.option(\'--high\', show_default=True, default=90, type=click.IntRange(1, 100))\ndef binarize(ctx, threshold, zoom, escale, border, perc, range, low, high):\n    """\n    Binarizes page images.\n    """\n    ctx.meta[\'steps\'].append({\'category\': \'preprocessing\',\n                              \'description\': \'Image binarization\',\n                              \'settings\': {\'threshold\': threshold,\n                                           \'zoom\': zoom,\n                                           \'escale\': escale,\n                                           \'border\': border,\n                                           \'perc\': perc,\n                                           \'range\': range,\n                                           \'low\': low,\n                                           \'high\': high}})\n\n    return partial(binarizer, threshold, zoom, escale, border, perc, range,\n                   low, high)\n\n\n@cli.command(\'segment\')\n@click.pass_context\n@click.option(\'-i\', \'--model\',\n              default=None,\n              show_default=True, type=click.Path(exists=True),\n              help=\'Baseline detection model to use\')\n@click.option(\'-x/-bl\', \'--boxes/--baseline\', default=True, show_default=True,\n              help=\'Switch between legacy box segmenter and neural baseline segmenter\')\n@click.option(\'-d\', \'--text-direction\', default=\'horizontal-lr\',\n              show_default=True,\n              type=click.Choice([\'horizontal-lr\', \'horizontal-rl\',\n                                 \'vertical-lr\', \'vertical-rl\']),\n              help=\'Sets principal text direction\')\n@click.option(\'--scale\', show_default=True, default=None, type=click.FLOAT)\n@click.option(\'-m\', \'--maxcolseps\', show_default=True, default=2, type=click.INT)\n@click.option(\'-b/-w\', \'--black-colseps/--white_colseps\', show_default=True, default=False)\n@click.option(\'-r/-l\', \'--remove_hlines/--hlines\', show_default=True, default=True)\n@click.option(\'-p\', \'--pad\', show_default=True, type=(int, int), default=(0, 0),\n              help=\'Left and right padding around lines\')\n@click.option(\'-m\', \'--mask\', show_default=True, default=None,\n              type=click.File(mode=\'rb\', lazy=True), help=\'Segmentation mask \'\n              \'suppressing page areas for line detection. 0-valued image \'\n              \'regions are ignored for segmentation purposes. Disables column \'\n              \'detection.\')\ndef segment(ctx, model, boxes, text_direction, scale, maxcolseps,\n            black_colseps, remove_hlines, pad, mask):\n    """\n    Segments page images into text lines.\n    """\n    if model and boxes:\n        logger.warning(f\'Baseline model ({model}) given but legacy segmenter selected. Forcing to -bl.\')\n        boxes = False\n\n    if boxes is False:\n        if not model:\n            model = SEGMENTATION_DEFAULT_MODEL\n        ctx.meta[\'steps\'].append({\'category\': \'processing\',\n                                  \'description\': \'Baseline and region segmentation\',\n                                  \'settings\': {\'model\': os.path.basename(model),\n                                               \'text_direction\': text_direction}})\n\n        from kraken.lib.vgsl import TorchVGSLModel\n        message(f\'Loading ANN {model}\\t\', nl=False)\n        try:\n            model = TorchVGSLModel.load_model(model)\n            model.to(ctx.meta[\'device\'])\n        except Exception:\n            if ctx.meta[\'raise_failed\']:\n                raise\n            message(\'\\u2717\', fg=\'red\')\n            ctx.exit(1)\n\n        message(\'\\u2713\', fg=\'green\')\n    else:\n        ctx.meta[\'steps\'].append({\'category\': \'processing\',\n                                  \'description\': \'bounding box segmentation\',\n                                  \'settings\': {\'text_direction\': text_direction,\n                                               \'scale\': scale,\n                                               \'maxcolseps\': maxcolseps,\n                                               \'black_colseps\': black_colseps,\n                                               \'remove_hlines\': remove_hlines,\n                                               \'pad\': pad}})\n\n    return partial(segmenter, boxes, model, text_direction, scale, maxcolseps,\n                   black_colseps, remove_hlines, pad, mask, ctx.meta[\'device\'])\n\n\ndef _validate_mm(ctx, param, value):\n    """\n    Maps model mappings to a dictionary.\n    """\n    model_dict = {\'ignore\': []}  # type: Dict[str, Union[str, List[str]]]\n    if len(value) == 1 and len(value[0].split(\':\')) == 1:\n        model_dict[\'default\'] = value[0]\n        return model_dict\n    try:\n        for m in value:\n            k, v = m.split(\':\')\n            if v == \'ignore\':\n                model_dict[\'ignore\'].append(k)  # type: ignore\n            else:\n                model_dict[k] = os.path.expanduser(v)\n    except Exception:\n        raise click.BadParameter(\'Mappings must be in format tag:model\')\n    return model_dict\n\n\n@cli.command(\'ocr\')\n@click.pass_context\n@click.option(\'-m\', \'--model\', default=DEFAULT_MODEL, multiple=True,\n              show_default=True, callback=_validate_mm,\n              help=\'Path to an recognition model or mapping of the form \'\n              \'$tag1:$model1. Add multiple mappings to run multi-model \'\n              \'recognition based on detected tags. Use the default keyword \'\n              \'for adding a catch-all model. Recognition on tags can be \'\n              \'ignored with the model value ignore.\')\n@click.option(\'-p\', \'--pad\', show_default=True, type=click.INT, default=16, help=\'Left and right \'\n              \'padding around lines\')\n@click.option(\'-n\', \'--reorder/--no-reorder\', show_default=True, default=True,\n              help=\'Reorder code points to logical order\')\n@click.option(\'--base-dir\', show_default=True, default=\'auto\',\n              type=click.Choice([\'L\', \'R\', \'auto\']), help=\'Set base text \'\n              \'direction.  This should be set to the direction used during the \'\n              \'creation of the training data. If set to `auto` it will be \'\n              \'overridden by any explicit value given in the input files.\')\n@click.option(\'-s\', \'--no-segmentation\', default=False, show_default=True, is_flag=True,\n              help=\'Enables non-segmentation mode treating each input image as a whole line.\')\n@click.option(\'-d\', \'--text-direction\', default=\'horizontal-tb\',\n              show_default=True,\n              type=click.Choice([\'horizontal-tb\', \'vertical-lr\', \'vertical-rl\']),\n              help=\'Sets principal text direction in serialization output\')\n@click.option(\'--threads\', default=1, show_default=True, type=click.IntRange(1),\n              help=\'Number of threads to use for OpenMP parallelization.\')\ndef ocr(ctx, model, pad, reorder, base_dir, no_segmentation, text_direction, threads):\n    """\n    Recognizes text in line images.\n    """\n    from kraken.lib import models\n\n    if ctx.meta[\'input_format_type\'] != \'image\' and no_segmentation:\n        raise click.BadParameter(\'no_segmentation mode is incompatible with page/alto inputs\')\n\n    if reorder and base_dir != \'auto\':\n        reorder = base_dir\n\n    # first we try to find the model in the absolue path, then ~/.kraken, then\n    # LEGACY_MODEL_DIR\n    nm = {}  # type: Dict[str, models.TorchSeqRecognizer]\n    ign_tags = model.pop(\'ignore\')\n    for k, v in model.items():\n        search = [v,\n                  os.path.join(click.get_app_dir(APP_NAME), v),\n                  os.path.join(LEGACY_MODEL_DIR, v)]\n        location = None\n        for loc in search:\n            if os.path.isfile(loc):\n                location = loc\n                break\n        if not location:\n            raise click.BadParameter(f\'No model for {v} found\')\n        message(f\'Loading ANN {v}\\t\', nl=False)\n        try:\n            rnn = models.load_any(location, device=ctx.meta[\'device\'])\n            nm[k] = rnn\n        except Exception:\n            if ctx.meta[\'raise_failed\']:\n                raise\n            message(\'\\u2717\', fg=\'red\')\n            ctx.exit(1)\n        message(\'\\u2713\', fg=\'green\')\n\n    if \'default\' in nm:\n        from collections import defaultdict\n\n        nn = defaultdict(lambda: nm[\'default\'])  # type: Dict[str, models.TorchSeqRecognizer]\n        nn.update(nm)\n        nm = nn\n    # thread count is global so setting it once is sufficient\n    nm[k].nn.set_num_threads(threads)\n\n    ctx.meta[\'steps\'].append({\'category\': \'processing\',\n                              \'description\': \'Text line recognition\',\n                              \'settings\': {\'text_direction\': text_direction,\n                                           \'models\': \' \'.join(os.path.basename(v) for v in model.values()),\n                                           \'pad\': pad,\n                                           \'bidi_reordering\': reorder}})\n\n    # set output mode\n    ctx.meta[\'text_direction\'] = text_direction\n    return partial(recognizer,\n                   model=nm,\n                   pad=pad,\n                   no_segmentation=no_segmentation,\n                   bidi_reordering=reorder,\n                   tags_ignore=ign_tags)\n\n\n@cli.command(\'show\')\n@click.pass_context\n@click.argument(\'model_id\')\ndef show(ctx, model_id):\n    """\n    Retrieves model metadata from the repository.\n    """\n    from kraken import repo\n    from kraken.lib.util import make_printable, is_printable\n\n    desc = repo.get_description(model_id)\n\n    chars = []\n    combining = []\n    for char in sorted(desc[\'graphemes\']):\n        if not is_printable(char):\n            combining.append(make_printable(char))\n        else:\n            chars.append(char)\n    message(\n        \'name: {}\\n\\n{}\\n\\n{}\\nscripts: {}\\nalphabet: {} {}\\naccuracy: {:.2f}%\\nlicense: {}\\nauthor(s): {}\\ndate: {}\'.format(\n            model_id, desc[\'summary\'], desc[\'description\'], \' \'.join(\n                desc[\'script\']), \'\'.join(chars), \', \'.join(combining), desc[\'accuracy\'], desc[\'license\'][\'id\'], \'; \'.join(\n                x[\'name\'] for x in desc[\'creators\']), desc[\'publication_date\']))\n    ctx.exit(0)\n\n\n@cli.command(\'list\')\n@click.pass_context\ndef list_models(ctx):\n    """\n    Lists models in the repository.\n    """\n    from kraken import repo\n\n    with KrakenProgressBar() as progress:\n        download_task = progress.add_task(\'Retrieving model list\', total=0, visible=True if not ctx.meta[\'verbose\'] else False)\n        model_list = repo.get_listing(lambda total, advance: progress.update(download_task, total=total, advance=advance))\n    for id, metadata in model_list.items():\n        message(\'{} ({}) - {}\'.format(id, \', \'.join(metadata[\'type\']), metadata[\'summary\']))\n    ctx.exit(0)\n\n\n@cli.command(\'get\')\n@click.pass_context\n@click.argument(\'model_id\')\ndef get(ctx, model_id):\n    """\n    Retrieves a model from the repository.\n    """\n    from kraken import repo\n\n    try:\n        os.makedirs(click.get_app_dir(APP_NAME))\n    except OSError:\n        pass\n\n    with KrakenDownloadProgressBar() as progress:\n        download_task = progress.add_task(\'Processing\', total=0, visible=True if not ctx.meta[\'verbose\'] else False)\n        filename = repo.get_model(model_id, click.get_app_dir(APP_NAME),\n                                  lambda total, advance: progress.update(download_task, total=total, advance=advance))\n    message(f\'Model name: {filename}\')\n    ctx.exit(0)\n\n\nif __name__ == \'__main__\':\n    cli()\n', 'message': 'forgot to remove debug printf\n', 'author_name': 'Benjamin Kiessling', 'lang': 'py'}