streamcorpus_pipeline/_get_name_info.py
def get_name_info(chunk_path, assert_one_date_hour=False, i_str=None,
                  chunk_type=Chunk):
    '''
    takes a chunk blob and obtains the date_hour, md5, num

    makes fields:
    i_str
    input_fname
    input_md5 - parsed from input filename if it contains '-%(md5)s-'
    md5
    num
    epoch_ticks
    target_names
    doc_ids_8
    date_hour
    rand8
    date_now
    time_now
    date_time_now
    '''
    assert i_str is not None, 'must provide i_str as keyword arg'

    name_info = dict()
    if i_str:
        name_info['i_str'] = i_str
    else:
        name_info['i_str'] = ''

    i_fname = i_str.split('/')[-1]
    i_fname = i_fname.split('.')[0]  ## strip off .sc[.xz[.gpg]]
    name_info['input_fname'] = i_fname 

    input_md5s = []
    for part in i_fname.split('-'):
        if len(part) == 32 and is_hex_32.match(part):
            input_md5s.append(part)
    name_info['input_md5'] = '-'.join(input_md5s)

    # TODO: return a dict-like object that does the expensive
    # calculation lazily, the name format might not even need that
    # value.
    ch = chunk_type(path=chunk_path, mode='rb')
    date_hours = set()
    target_names = set()
    doc_ids = set()
    epoch_ticks = None
    count = 0
    try:
        for si in ch:
            if chunk_type is Chunk:
                if epoch_ticks is None:
                    epoch_ticks = si.stream_time.epoch_ticks
                date_hours.add( si.stream_time.zulu_timestamp[:13] )
                doc_ids.add( si.doc_id )
                for annotator_id, ratings in si.ratings.items():
                    for rating in ratings:
                        target_name = rating.target.target_id.split('/')[-1]
                        target_names.add( target_name )
            count += 1
    except Exception, exc:
        logger.critical('failed to iter over chunk', exc_info=True)
        

    ## create the md5 property, so we can use it in the filename
    if hasattr(ch, 'md5_hexdigest'):
        name_info['md5'] = ch.md5_hexdigest
    else:
        try:
            data = open(chunk_path).read()
            name_info['md5'] = hashlib.md5(data).hexdigest()
        except Exception, exc:
            logger.critical('failed to compute md5', exc_info=True)
            name_info['md5'] = 'broken'
    name_info['num'] = count
    name_info['epoch_ticks'] = epoch_ticks

    name_info['target_names'] = '-'.join( target_names )
    name_info['doc_ids_8'] = '-'.join( [di[:8] for di in doc_ids] )

    if chunk_type is Chunk:
        if assert_one_date_hour:
            assert len(date_hours) == 1, \
                'got a chunk with other than one data_hour! ' + \
                repr(date_hours)

        if len(date_hours) > 0:
            date_hour = list(date_hours)[0]
            date_hour = date_hour.replace('T', '-')
        else:
            assert count == 0, (date_hours, count)
            date_hour = None
        name_info['date_hour'] = date_hour
    else:
        name_info['date_hour'] = 'NO-DATE-HOUR-FOR-FC'

    # TODO: in future lazy evaluation world, rand8 should return a
    # different value every time it is accessed so that a format could
    # be 'foo-{rand8}{rand8}'
    name_info['rand8'] = '%08x' % (random.randint(0, 0x7fffffff),)

    name_info['date_now'] = datetime.datetime.utcnow().strftime('%Y-%m-%d')
    name_info['time_now'] = datetime.datetime.utcnow().strftime('%H-%M-%S')
    name_info['date_time_now'] = datetime.datetime.utcnow().strftime('%Y-%m-%d-%H-%M-%S')

    return name_info
def get_name_info(chunk_path, assert_one_date_hour=False, i_str=None,
                  chunk_type=Chunk):
    '''
    takes a chunk blob and obtains the date_hour, md5, num

    makes fields:
    i_str
    input_fname
    input_md5 - parsed from input filename if it contains '-%(md5)s-'
    md5
    num
    epoch_ticks
    target_names
    doc_ids_8
    date_hour
    rand8
    date_now
    time_now
    date_time_now
    '''
    assert i_str is not None, 'must provide i_str as keyword arg'

    name_info = dict()
    if i_str:
        name_info['i_str'] = i_str
    else:
        name_info['i_str'] = ''

    i_fname = i_str.split('/')[-1]
    i_fname = i_fname.split('.')[0]  ## strip off .sc[.xz[.gpg]]
    name_info['input_fname'] = i_fname 

    input_md5s = []
    for part in i_fname.split('-'):
        if len(part) == 32 and is_hex_32.match(part):
            input_md5s.append(part)
    name_info['input_md5'] = '-'.join(input_md5s)

    # TODO: return a dict-like object that does the expensive
    # calculation lazily, the name format might not even need that
    # value.
    ch = chunk_type(path=chunk_path, mode='rb')
    date_hours = set()
    target_names = set()
    doc_ids = set()
    epoch_ticks = None
    count = 0
    try:
        for si in ch:
            if chunk_type is Chunk:
                if epoch_ticks is None:
                    epoch_ticks = si.stream_time.epoch_ticks
                date_hours.add( si.stream_time.zulu_timestamp[:13] )
                doc_ids.add( si.doc_id )
                for annotator_id, ratings in si.ratings.items():
                    for rating in ratings:
                        target_name = rating.target.target_id.split('/')[-1]
                        target_names.add( target_name )
            count += 1
    except Exception, exc:
        logger.critical('failed to iter over chunk', exc_info=True)
        

    ## create the md5 property, so we can use it in the filename
    if hasattr(ch, 'md5_hexdigest'):
        name_info['md5'] = ch.md5_hexdigest
    else:
        try:
            data = open(chunk_path).read()
            name_info['md5'] = hashlib.md5(data).hexdigest()
        except Exception, exc:
            logger.critical('failed to compute md5', exc_info=True)
            name_info['md5'] = 'broken'
    name_info['num'] = count
    name_info['epoch_ticks'] = epoch_ticks

    name_info['target_names'] = '-'.join( target_names )
    name_info['doc_ids_8'] = '-'.join( [di[:8] for di in doc_ids] )

    if chunk_type is Chunk:
        if assert_one_date_hour:
            assert len(date_hours) == 1, \
                'got a chunk with other than one data_hour! ' + \
                repr(date_hours)

        if len(date_hours) > 0:
            date_hour = list(date_hours)[0]
            date_hour = date_hour.replace('T', '-')
        else:
            assert count == 0, (date_hours, count)
            date_hour = None
        name_info['date_hour'] = date_hour
    else:
        name_info['date_hour'] = 'NO-DATE-HOUR-FOR-FC'

    # TODO: in future lazy evaluation world, rand8 should return a
    # different value every time it is accessed so that a format could
    # be 'foo-{rand8}{rand8}'
    name_info['rand8'] = '%08x' % (random.randint(0, 0x7fffffff),)

    name_info['date_now'] = datetime.datetime.utcnow().strftime('%Y-%m-%d')
    name_info['time_now'] = datetime.datetime.utcnow().strftime('%H-%M-%S')
    name_info['date_time_now'] = datetime.datetime.utcnow().strftime('%Y-%m-%d-%H-%M-%S')

    return name_info
def get_name_info ( chunk_path , assert_one_date_hour = False , i_str = None , chunk_type = Chunk ) : assert i_str is not None , 'must provide i_str as keyword arg' name_info = dict ( ) if i_str : name_info [ 'i_str' ] = i_str else : name_info [ 'i_str' ] = '' i_fname = i_str . split ( '/' ) [ - 1 ] i_fname = i_fname . split ( '.' ) [ 0 ] ## strip off .sc[.xz[.gpg]] name_info [ 'input_fname' ] = i_fname input_md5s = [ ] for part in i_fname . split ( '-' ) : if len ( part ) == 32 and is_hex_32 . match ( part ) : input_md5s . append ( part ) name_info [ 'input_md5' ] = '-' . join ( input_md5s ) # TODO: return a dict-like object that does the expensive # calculation lazily, the name format might not even need that # value. ch = chunk_type ( path = chunk_path , mode = 'rb' ) date_hours = set ( ) target_names = set ( ) doc_ids = set ( ) epoch_ticks = None count = 0 try : for si in ch : if chunk_type is Chunk : if epoch_ticks is None : epoch_ticks = si . stream_time . epoch_ticks date_hours . add ( si . stream_time . zulu_timestamp [ : 13 ] ) doc_ids . add ( si . doc_id ) for annotator_id , ratings in si . ratings . items ( ) : for rating in ratings : target_name = rating . target . target_id . split ( '/' ) [ - 1 ] target_names . add ( target_name ) count += 1 except Exception , exc : logger . critical ( 'failed to iter over chunk' , exc_info = True ) ## create the md5 property, so we can use it in the filename if hasattr ( ch , 'md5_hexdigest' ) : name_info [ 'md5' ] = ch . md5_hexdigest else : try : data = open ( chunk_path ) . read ( ) name_info [ 'md5' ] = hashlib . md5 ( data ) . hexdigest ( ) except Exception , exc : logger . critical ( 'failed to compute md5' , exc_info = True ) name_info [ 'md5' ] = 'broken' name_info [ 'num' ] = count name_info [ 'epoch_ticks' ] = epoch_ticks name_info [ 'target_names' ] = '-' . join ( target_names ) name_info [ 'doc_ids_8' ] = '-' . join ( [ di [ : 8 ] for di in doc_ids ] ) if chunk_type is Chunk : if assert_one_date_hour : assert len ( date_hours ) == 1 , 'got a chunk with other than one data_hour! ' + repr ( date_hours ) if len ( date_hours ) > 0 : date_hour = list ( date_hours ) [ 0 ] date_hour = date_hour . replace ( 'T' , '-' ) else : assert count == 0 , ( date_hours , count ) date_hour = None name_info [ 'date_hour' ] = date_hour else : name_info [ 'date_hour' ] = 'NO-DATE-HOUR-FOR-FC' # TODO: in future lazy evaluation world, rand8 should return a # different value every time it is accessed so that a format could # be 'foo-{rand8}{rand8}' name_info [ 'rand8' ] = '%08x' % ( random . randint ( 0 , 0x7fffffff ) , ) name_info [ 'date_now' ] = datetime . datetime . utcnow ( ) . strftime ( '%Y-%m-%d' ) name_info [ 'time_now' ] = datetime . datetime . utcnow ( ) . strftime ( '%H-%M-%S' ) name_info [ 'date_time_now' ] = datetime . datetime . utcnow ( ) . strftime ( '%Y-%m-%d-%H-%M-%S' ) return name_info
takes a chunk blob and obtains the date_hour, md5, num

    makes fields:
    i_str
    input_fname
    input_md5 - parsed from input filename if it contains '-%(md5)s-'
    md5
    num
    epoch_ticks
    target_names
    doc_ids_8
    date_hour
    rand8
    date_now
    time_now
    date_time_now
takes a chunk blob and obtains the date_hour md5 num