streamcorpus_pipeline/_kvlayer.py
def parse_keys_and_ranges(i_str, keyfunc, rangefunc):
    '''Parse the :class:`from_kvlayer` input string.

    This accepts two formats.  In the textual format, it accepts any
    number of stream IDs in timestamp-docid format, separated by ``,``
    or ``;``, and processes those as individual stream IDs.  In the
    binary format, it accepts 20-byte key blobs (16 bytes md5 hash, 4
    bytes timestamp) split by ``;`` or ``<``; e.g., ``a<f;x`` loads
    scans keys `a` through `f` and loads singly key `x`.

    `keyfunc` and `rangefunc` are run as generators and their yields
    are yielded from this function.

    '''
    while i_str:
        m = _STREAM_ID_RE.match(i_str)
        if m:
            # old style text stream_id
            for retval in keyfunc(stream_id_to_kvlayer_key(m.group())):
                yield retval
            i_str = i_str[m.end():]
            while i_str and ((i_str[0] == ',') or (i_str[0] == ';')):
                i_str = i_str[1:]
            continue

        if len(i_str) == SI_KEY_LENGTH:
            # one key, get it.
            key = parse_si_key(i_str)
            for retval in keyfunc(key):
                yield retval
            return

        keya = i_str[:SI_KEY_LENGTH]
        splitc = i_str[SI_KEY_LENGTH]
        if splitc == '<':
            # range
            keyb = i_str[SI_KEY_LENGTH+1:SI_KEY_LENGTH+1+SI_KEY_LENGTH]
            i_str = i_str[SI_KEY_LENGTH+1+SI_KEY_LENGTH:]
            keya = parse_si_key(keya)
            keyb = parse_si_key(keyb)
            for retval in rangefunc(keya, keyb):
                yield retval
        elif splitc == ';':
            # keya is single key to load
            keya = parse_si_key(keya)
            for retval in keyfunc(keya):
                yield retval
            i_str = i_str[SI_KEY_LENGTH+1+1:]
        else:
            logger.error('bogus key splitter %s, %r', splitc, i_str)
            return
def parse_keys_and_ranges(i_str, keyfunc, rangefunc):
    '''Parse the :class:`from_kvlayer` input string.

    This accepts two formats.  In the textual format, it accepts any
    number of stream IDs in timestamp-docid format, separated by ``,``
    or ``;``, and processes those as individual stream IDs.  In the
    binary format, it accepts 20-byte key blobs (16 bytes md5 hash, 4
    bytes timestamp) split by ``;`` or ``<``; e.g., ``a<f;x`` loads
    scans keys `a` through `f` and loads singly key `x`.

    `keyfunc` and `rangefunc` are run as generators and their yields
    are yielded from this function.

    '''
    while i_str:
        m = _STREAM_ID_RE.match(i_str)
        if m:
            # old style text stream_id
            for retval in keyfunc(stream_id_to_kvlayer_key(m.group())):
                yield retval
            i_str = i_str[m.end():]
            while i_str and ((i_str[0] == ',') or (i_str[0] == ';')):
                i_str = i_str[1:]
            continue

        if len(i_str) == SI_KEY_LENGTH:
            # one key, get it.
            key = parse_si_key(i_str)
            for retval in keyfunc(key):
                yield retval
            return

        keya = i_str[:SI_KEY_LENGTH]
        splitc = i_str[SI_KEY_LENGTH]
        if splitc == '<':
            # range
            keyb = i_str[SI_KEY_LENGTH+1:SI_KEY_LENGTH+1+SI_KEY_LENGTH]
            i_str = i_str[SI_KEY_LENGTH+1+SI_KEY_LENGTH:]
            keya = parse_si_key(keya)
            keyb = parse_si_key(keyb)
            for retval in rangefunc(keya, keyb):
                yield retval
        elif splitc == ';':
            # keya is single key to load
            keya = parse_si_key(keya)
            for retval in keyfunc(keya):
                yield retval
            i_str = i_str[SI_KEY_LENGTH+1+1:]
        else:
            logger.error('bogus key splitter %s, %r', splitc, i_str)
            return
def parse_keys_and_ranges ( i_str , keyfunc , rangefunc ) : while i_str : m = _STREAM_ID_RE . match ( i_str ) if m : # old style text stream_id for retval in keyfunc ( stream_id_to_kvlayer_key ( m . group ( ) ) ) : yield retval i_str = i_str [ m . end ( ) : ] while i_str and ( ( i_str [ 0 ] == ',' ) or ( i_str [ 0 ] == ';' ) ) : i_str = i_str [ 1 : ] continue if len ( i_str ) == SI_KEY_LENGTH : # one key, get it. key = parse_si_key ( i_str ) for retval in keyfunc ( key ) : yield retval return keya = i_str [ : SI_KEY_LENGTH ] splitc = i_str [ SI_KEY_LENGTH ] if splitc == '<' : # range keyb = i_str [ SI_KEY_LENGTH + 1 : SI_KEY_LENGTH + 1 + SI_KEY_LENGTH ] i_str = i_str [ SI_KEY_LENGTH + 1 + SI_KEY_LENGTH : ] keya = parse_si_key ( keya ) keyb = parse_si_key ( keyb ) for retval in rangefunc ( keya , keyb ) : yield retval elif splitc == ';' : # keya is single key to load keya = parse_si_key ( keya ) for retval in keyfunc ( keya ) : yield retval i_str = i_str [ SI_KEY_LENGTH + 1 + 1 : ] else : logger . error ( 'bogus key splitter %s, %r' , splitc , i_str ) return
Parse the :class:`from_kvlayer` input string.

    This accepts two formats.  In the textual format, it accepts any
    number of stream IDs in timestamp-docid format, separated by ``,``
    or ``;``, and processes those as individual stream IDs.  In the
    binary format, it accepts 20-byte key blobs (16 bytes md5 hash, 4
    bytes timestamp) split by ``;`` or ``<``; e.g., ``a<f;x`` loads
    scans keys `a` through `f` and loads singly key `x`.

    `keyfunc` and `rangefunc` are run as generators and their yields
    are yielded from this function.
Parse the : class : from_kvlayer input string .