neuroscout/tasks/upload.py
def upload_collection(flask_app, filenames, runs, dataset_id, collection_id,
                      descriptions=None, cache=None):
    """ Create new Predictors from TSV files
    Args:
        filenames list of (str): List of paths to TSVs
        runs list of (int): List of run ids to apply events to
        dataset_id (int): Dataset id.
        collection_id (int): Id of collection object
        descriptions (dict): Optional descriptions for each column
        cache (obj): Optional flask cache object
    """
    if cache is None:
        from ..core import cache as cache
    if descriptions is None:
        descriptions = {}

    collection_object = PredictorCollection.query.filter_by(
        id=collection_id).one()

    # Load into pandas
    try:
        events = [pd.read_csv(f, sep='\t') for f in filenames]
    except Exception as e:
        update_record(
            collection_object,
            exception=e,
            traceback='Error reading event files'
        )
        raise

    # Check columns are all the same across all files
    cols = [set(e.columns) for e in events]
    common_cols = set.intersection(*cols)
    if not len(common_cols) == len(cols[0]):
        update_record(
            collection_object,
            traceback='Event files contain distinct columns'
        )
        raise Exception('Event files contain distinct columns')

    if not set(['onset', 'duration']).issubset(common_cols):
        update_record(
            collection_object,
            traceback='Not all columns have "onset" and "duration"'
        )
        raise Exception('Not all columns have "onset" and "duration"')

    pe_objects = []
    try:
        for col in common_cols - set(['onset', 'duration']):
            predictor = Predictor(
                name=col,
                source=f'Collection: {collection_object.collection_name}',
                dataset_id=dataset_id,
                predictor_collection_id=collection_object.id,
                private=True,
                description=descriptions.get(col))
            db.session.add(predictor)
            db.session.commit()

            for ix, e in enumerate(events):
                select = e[['onset', 'duration', col]].dropna()
                for run_id in runs[ix]:
                    # Add PredictorRun
                    pr, _ = get_or_create(
                        PredictorRun, predictor_id=predictor.id, run_id=run_id)
                    for _, row in select.iterrows():
                        row = row.to_dict()
                        pe_objects.append(
                            PredictorEvent(
                                predictor_id=predictor.id,
                                run_id=run_id, onset=row['onset'],
                                duration=row['duration'], value=row[col])
                            )
            collection_object.predictors.append(predictor)

        db.session.bulk_save_objects(pe_objects)
        db.session.commit()
    except Exception as e:
        cache.clear()
        db.session.rollback()
        update_record(
            collection_object,
            exception=e,
            traceback=f'Error creating predictors. Failed processing {col}'
        )
        raise

    cache.clear()
    return update_record(
        collection_object,
        status='OK'
    )
 Create new Predictors from TSV files
    Args:
        filenames list of (str): List of paths to TSVs
        runs list of (int): List of run ids to apply events to
        dataset_id (int): Dataset id.
        collection_id (int): Id of collection object
        descriptions (dict): Optional descriptions for each column
        cache (obj): Optional flask cache object
    
Create new Predictors from TSV files
Create new Predictors from TSV files
def upload_collection(flask_app, filenames, runs, dataset_id, collection_id,
                      descriptions=None, cache=None):
    if cache is None:
        from ..core import cache as cache
    if descriptions is None:
        descriptions = {}
    collection_object = PredictorCollection.query.filter_by(
        id=collection_id).one()
    try:
        events = [pd.read_csv(f, sep='\t') for f in filenames]
    except Exception as e:
        update_record(
            collection_object,
            exception=e,
            traceback='Error reading event files'
        )
        raise
    cols = [set(e.columns) for e in events]
    common_cols = set.intersection(*cols)
    if not len(common_cols) == len(cols[0]):
        update_record(
            collection_object,
            traceback='Event files contain distinct columns'
        )
        raise Exception('Event files contain distinct columns')
    if not set(['onset', 'duration']).issubset(common_cols):
        update_record(
            collection_object,
            traceback='Not all columns have "onset" and "duration"'
        )
        raise Exception('Not all columns have "onset" and "duration"')
    pe_objects = []
    try:
        for col in common_cols - set(['onset', 'duration']):
            predictor = Predictor(
                name=col,
                source=f'Collection: {collection_object.collection_name}',
                dataset_id=dataset_id,
                predictor_collection_id=collection_object.id,
                private=True,
                description=descriptions.get(col))
            db.session.add(predictor)
            db.session.commit()
            for ix, e in enumerate(events):
                select = e[['onset', 'duration', col]].dropna()
                for run_id in runs[ix]:
                    pr, _ = get_or_create(
                        PredictorRun, predictor_id=predictor.id, run_id=run_id)
                    for _, row in select.iterrows():
                        row = row.to_dict()
                        pe_objects.append(
                            PredictorEvent(
                                predictor_id=predictor.id,
                                run_id=run_id, onset=row['onset'],
                                duration=row['duration'], value=row[col])
                            )
            collection_object.predictors.append(predictor)
        db.session.bulk_save_objects(pe_objects)
        db.session.commit()
    except Exception as e:
        cache.clear()
        db.session.rollback()
        update_record(
            collection_object,
            exception=e,
            traceback=f'Error creating predictors. Failed processing {col}'
        )
        raise
    cache.clear()
    return update_record(
        collection_object,
        status='OK'
    )
def upload_collection ( flask_app , filenames , runs , dataset_id , collection_id , descriptions = None , cache = None ) : if cache is None : from . . core import cache as cache if descriptions is None : descriptions = { } collection_object = PredictorCollection . query . filter_by ( id = collection_id ) . one ( ) try : events = [ pd . read_csv ( f , sep = '\t' ) for f in filenames ] except Exception as e : update_record ( collection_object , exception = e , traceback = 'Error reading event files' ) raise cols = [ set ( e . columns ) for e in events ] common_cols = set . intersection ( * cols ) if not len ( common_cols ) == len ( cols [ 0 ] ) : update_record ( collection_object , traceback = 'Event files contain distinct columns' ) raise Exception ( 'Event files contain distinct columns' ) if not set ( [ 'onset' , 'duration' ] ) . issubset ( common_cols ) : update_record ( collection_object , traceback = 'Not all columns have "onset" and "duration"' ) raise Exception ( 'Not all columns have "onset" and "duration"' ) pe_objects = [ ] try : for col in common_cols - set ( [ 'onset' , 'duration' ] ) : predictor = Predictor ( name = col , source = f'Collection: {collection_object.collection_name}' , dataset_id = dataset_id , predictor_collection_id = collection_object . id , private = True , description = descriptions . get ( col ) ) db . session . add ( predictor ) db . session . commit ( ) for ix , e in enumerate ( events ) : select = e [ [ 'onset' , 'duration' , col ] ] . dropna ( ) for run_id in runs [ ix ] : pr , _ = get_or_create ( PredictorRun , predictor_id = predictor . id , run_id = run_id ) for _ , row in select . iterrows ( ) : row = row . to_dict ( ) pe_objects . append ( PredictorEvent ( predictor_id = predictor . id , run_id = run_id , onset = row [ 'onset' ] , duration = row [ 'duration' ] , value = row [ col ] ) ) collection_object . predictors . append ( predictor ) db . session . bulk_save_objects ( pe_objects ) db . session . commit ( ) except Exception as e : cache . clear ( ) db . session . rollback ( ) update_record ( collection_object , exception = e , traceback = f'Error creating predictors. Failed processing {col}' ) raise cache . clear ( ) return update_record ( collection_object , status = 'OK' )
Create new Predictors from TSV files
Create new Predictors from TSV files
""" Create new Predictors from TSV files
    Args:
        filenames list of (str): List of paths to TSVs
        runs list of (int): List of run ids to apply events to
        dataset_id (int): Dataset id.
        collection_id (int): Id of collection object
        descriptions (dict): Optional descriptions for each column
        cache (obj): Optional flask cache object
    """ # Load into pandas # Check columns are all the same across all files # Add PredictorRun
flask_app
filenames
runs
dataset_id
None
collection_id
Id of collection object
Id of collection object
descriptions
Optional descriptions for each column
Optional descriptions for each column
cache
Optional flask cache object
Optional flask cache object
filenames list of
List of paths to TSVs
List of paths to TSVs
runs list of
List of run ids to apply events to
List of run ids to apply events to