invenio_migrator/tasks/deposit.py
def create_files_and_sip(deposit, dep_pid):
    """Create deposit Bucket, Files and SIPs."""
    from invenio_pidstore.errors import PIDDoesNotExistError
    from invenio_pidstore.models import PersistentIdentifier, PIDStatus, \
        RecordIdentifier
    from invenio_sipstore.errors import SIPUserDoesNotExist
    from invenio_sipstore.models import SIP, RecordSIP, SIPFile
    from invenio_files_rest.models import Bucket, FileInstance, ObjectVersion
    from invenio_records_files.models import RecordsBuckets
    from invenio_db import db
    buc = Bucket.create()
    recbuc = RecordsBuckets(record_id=deposit.id, bucket_id=buc.id)
    db.session.add(recbuc)
    deposit.setdefault('_deposit', dict())
    deposit.setdefault('_buckets', dict(deposit=str(buc.id)))
    deposit.setdefault('_files', list())
    files = deposit.get('files', [])
    sips = deposit.get('sips', [])

    # Look for prereserved DOI (and recid)
    if 'drafts' in deposit:
        drafts = list(deposit['drafts'].items())
        if len(drafts) != 1:
            logger.exception('Deposit {dep_pid} has multiple drafts'.format(
                dep_pid=dep_pid))
        if len(drafts) == 1:
            draft_type, draft = drafts[0]
            draft_v = draft['values']
            if 'prereserve_doi' in draft_v:
                pre_recid = str(draft_v['prereserve_doi']['recid'])
                pre_doi = str(draft_v['prereserve_doi']['doi'])

                # If pre-reserve info available, try to reserve 'recid'
                try:
                    pid = PersistentIdentifier.get(pid_type='recid',
                                                   pid_value=str(pre_recid))
                except PIDDoesNotExistError:
                    # Reserve recid
                    pid = PersistentIdentifier.create(
                        pid_type='recid',
                        pid_value=str(pre_recid),
                        object_type='rec',
                        status=PIDStatus.RESERVED
                    )

                # If pre-reserve info available, try to reserve 'doi'
                try:
                    pid = PersistentIdentifier.get(pid_type='doi',
                                                   pid_value=str(pre_doi))
                except PIDDoesNotExistError:
                    # Reserve DOI
                    pid = PersistentIdentifier.create(
                        pid_type='doi',
                        pid_value=str(pre_doi),
                        object_type='rec',
                        status=PIDStatus.RESERVED
                    )

                if RecordIdentifier.query.get(int(pre_recid)) is None:
                    RecordIdentifier.insert(int(pre_recid))

    # Store the path -> FileInstance mappings for SIPFile creation later
    dep_file_instances = list()

    for file_ in files:
        size = file_['size']
        key = file_['name']
        # Warning: Assumes all checksums are MD5!
        checksum = 'md5:{0}'.format(file_['checksum'])
        fi = FileInstance.create()
        fi.set_uri(file_['path'], size, checksum)
        ov = ObjectVersion.create(buc, key, _file_id=fi.id)
        ext = splitext(ov.key)[1].lower()
        if ext.startswith('.'):
            ext = ext[1:]
        file_meta = dict(
            bucket=str(ov.bucket.id),
            key=ov.key,
            checksum=ov.file.checksum,
            size=ov.file.size,
            version_id=str(ov.version_id),
            type=ext,
        )
        deposit['_files'].append(file_meta)
        dep_file_instances.append((file_['path'], fi))

    # Get a recid from SIP information
    recid = None
    if sips:
        recids = [int(sip['metadata']['recid']) for sip in sips]
        if len(set(recids)) > 1:
            logger.error('Multiple recids ({recids}) found in deposit {depid}'
                         ' does not exists.'.format(recids=recids,
                                                    depid=dep_pid.pid_value))
            raise DepositMultipleRecids(dep_pid.pid_value, list(set(recids)))
        elif recids:  # If only one recid
            recid = recids[0]

    for idx, sip in enumerate(sips):
        agent = None
        user_id = None
        if sip['agents']:
            agent = dict(
                ip_address=empty_str_if_none(
                    sip['agents'][0].get('ip_address', "")),
                email=empty_str_if_none(
                    sip['agents'][0].get('email_address', "")),
            )
            user_id = sip['agents'][0]['user_id']
        if user_id == 0:
            user_id = None
        content = sip['package']
        sip_format = 'marcxml'
        try:
            sip = SIP.create(sip_format,
                             content,
                             user_id=user_id,
                             agent=agent)
        except SIPUserDoesNotExist:
            logger.exception('User ID {user_id} referred in deposit {depid} '
                             'does not exists.'.format(
                                 user_id=user_id, depid=dep_pid.pid_value))
            sip = SIP.create(sip_format,
                             content,
                             agent=agent)

        # Attach recid to SIP
        if recid:
            try:
                pid = PersistentIdentifier.get(pid_type='recid',
                                               pid_value=str(recid))
                record_sip = RecordSIP(sip_id=sip.id, pid_id=pid.id)
                db.session.add(record_sip)
            except PIDDoesNotExistError:
                logger.exception('Record {recid} referred in '
                                 'Deposit {depid} does not exists.'.format(
                                     recid=recid, depid=dep_pid.pid_value))
                if deposit['_p']['submitted'] is True:
                    logger.exception('Pair {recid}/{depid} was submitted,'
                                     ' (should it be unpublished?).'.format(
                                         recid=recid, depid=dep_pid.pid_value))
                else:
                    msg = 'Pair {recid}/{depid} was not submitted.'.format(
                        recid=recid, depid=dep_pid.pid_value)
                    logger.exception(msg)

                # Reserve recid
                pid = PersistentIdentifier.create(
                    pid_type='recid',
                    pid_value=str(recid),
                    object_type='rec',
                    status=PIDStatus.RESERVED
                )

                if RecordIdentifier.query.get(int(recid)) is None:
                    RecordIdentifier.insert(int(recid))
        if idx == 0:
            for fp, fi in dep_file_instances:
                sipf = SIPFile(sip_id=sip.id, filepath=fp, file_id=fi.id)
                db.session.add(sipf)
    deposit.commit()
    return deposit
def create_files_and_sip(deposit, dep_pid):
    """Create deposit Bucket, Files and SIPs."""
    from invenio_pidstore.errors import PIDDoesNotExistError
    from invenio_pidstore.models import PersistentIdentifier, PIDStatus, \
        RecordIdentifier
    from invenio_sipstore.errors import SIPUserDoesNotExist
    from invenio_sipstore.models import SIP, RecordSIP, SIPFile
    from invenio_files_rest.models import Bucket, FileInstance, ObjectVersion
    from invenio_records_files.models import RecordsBuckets
    from invenio_db import db
    buc = Bucket.create()
    recbuc = RecordsBuckets(record_id=deposit.id, bucket_id=buc.id)
    db.session.add(recbuc)
    deposit.setdefault('_deposit', dict())
    deposit.setdefault('_buckets', dict(deposit=str(buc.id)))
    deposit.setdefault('_files', list())
    files = deposit.get('files', [])
    sips = deposit.get('sips', [])

    # Look for prereserved DOI (and recid)
    if 'drafts' in deposit:
        drafts = list(deposit['drafts'].items())
        if len(drafts) != 1:
            logger.exception('Deposit {dep_pid} has multiple drafts'.format(
                dep_pid=dep_pid))
        if len(drafts) == 1:
            draft_type, draft = drafts[0]
            draft_v = draft['values']
            if 'prereserve_doi' in draft_v:
                pre_recid = str(draft_v['prereserve_doi']['recid'])
                pre_doi = str(draft_v['prereserve_doi']['doi'])

                # If pre-reserve info available, try to reserve 'recid'
                try:
                    pid = PersistentIdentifier.get(pid_type='recid',
                                                   pid_value=str(pre_recid))
                except PIDDoesNotExistError:
                    # Reserve recid
                    pid = PersistentIdentifier.create(
                        pid_type='recid',
                        pid_value=str(pre_recid),
                        object_type='rec',
                        status=PIDStatus.RESERVED
                    )

                # If pre-reserve info available, try to reserve 'doi'
                try:
                    pid = PersistentIdentifier.get(pid_type='doi',
                                                   pid_value=str(pre_doi))
                except PIDDoesNotExistError:
                    # Reserve DOI
                    pid = PersistentIdentifier.create(
                        pid_type='doi',
                        pid_value=str(pre_doi),
                        object_type='rec',
                        status=PIDStatus.RESERVED
                    )

                if RecordIdentifier.query.get(int(pre_recid)) is None:
                    RecordIdentifier.insert(int(pre_recid))

    # Store the path -> FileInstance mappings for SIPFile creation later
    dep_file_instances = list()

    for file_ in files:
        size = file_['size']
        key = file_['name']
        # Warning: Assumes all checksums are MD5!
        checksum = 'md5:{0}'.format(file_['checksum'])
        fi = FileInstance.create()
        fi.set_uri(file_['path'], size, checksum)
        ov = ObjectVersion.create(buc, key, _file_id=fi.id)
        ext = splitext(ov.key)[1].lower()
        if ext.startswith('.'):
            ext = ext[1:]
        file_meta = dict(
            bucket=str(ov.bucket.id),
            key=ov.key,
            checksum=ov.file.checksum,
            size=ov.file.size,
            version_id=str(ov.version_id),
            type=ext,
        )
        deposit['_files'].append(file_meta)
        dep_file_instances.append((file_['path'], fi))

    # Get a recid from SIP information
    recid = None
    if sips:
        recids = [int(sip['metadata']['recid']) for sip in sips]
        if len(set(recids)) > 1:
            logger.error('Multiple recids ({recids}) found in deposit {depid}'
                         ' does not exists.'.format(recids=recids,
                                                    depid=dep_pid.pid_value))
            raise DepositMultipleRecids(dep_pid.pid_value, list(set(recids)))
        elif recids:  # If only one recid
            recid = recids[0]

    for idx, sip in enumerate(sips):
        agent = None
        user_id = None
        if sip['agents']:
            agent = dict(
                ip_address=empty_str_if_none(
                    sip['agents'][0].get('ip_address', "")),
                email=empty_str_if_none(
                    sip['agents'][0].get('email_address', "")),
            )
            user_id = sip['agents'][0]['user_id']
        if user_id == 0:
            user_id = None
        content = sip['package']
        sip_format = 'marcxml'
        try:
            sip = SIP.create(sip_format,
                             content,
                             user_id=user_id,
                             agent=agent)
        except SIPUserDoesNotExist:
            logger.exception('User ID {user_id} referred in deposit {depid} '
                             'does not exists.'.format(
                                 user_id=user_id, depid=dep_pid.pid_value))
            sip = SIP.create(sip_format,
                             content,
                             agent=agent)

        # Attach recid to SIP
        if recid:
            try:
                pid = PersistentIdentifier.get(pid_type='recid',
                                               pid_value=str(recid))
                record_sip = RecordSIP(sip_id=sip.id, pid_id=pid.id)
                db.session.add(record_sip)
            except PIDDoesNotExistError:
                logger.exception('Record {recid} referred in '
                                 'Deposit {depid} does not exists.'.format(
                                     recid=recid, depid=dep_pid.pid_value))
                if deposit['_p']['submitted'] is True:
                    logger.exception('Pair {recid}/{depid} was submitted,'
                                     ' (should it be unpublished?).'.format(
                                         recid=recid, depid=dep_pid.pid_value))
                else:
                    msg = 'Pair {recid}/{depid} was not submitted.'.format(
                        recid=recid, depid=dep_pid.pid_value)
                    logger.exception(msg)

                # Reserve recid
                pid = PersistentIdentifier.create(
                    pid_type='recid',
                    pid_value=str(recid),
                    object_type='rec',
                    status=PIDStatus.RESERVED
                )

                if RecordIdentifier.query.get(int(recid)) is None:
                    RecordIdentifier.insert(int(recid))
        if idx == 0:
            for fp, fi in dep_file_instances:
                sipf = SIPFile(sip_id=sip.id, filepath=fp, file_id=fi.id)
                db.session.add(sipf)
    deposit.commit()
    return deposit
def create_files_and_sip ( deposit , dep_pid ) : from invenio_pidstore . errors import PIDDoesNotExistError from invenio_pidstore . models import PersistentIdentifier , PIDStatus , RecordIdentifier from invenio_sipstore . errors import SIPUserDoesNotExist from invenio_sipstore . models import SIP , RecordSIP , SIPFile from invenio_files_rest . models import Bucket , FileInstance , ObjectVersion from invenio_records_files . models import RecordsBuckets from invenio_db import db buc = Bucket . create ( ) recbuc = RecordsBuckets ( record_id = deposit . id , bucket_id = buc . id ) db . session . add ( recbuc ) deposit . setdefault ( '_deposit' , dict ( ) ) deposit . setdefault ( '_buckets' , dict ( deposit = str ( buc . id ) ) ) deposit . setdefault ( '_files' , list ( ) ) files = deposit . get ( 'files' , [ ] ) sips = deposit . get ( 'sips' , [ ] ) # Look for prereserved DOI (and recid) if 'drafts' in deposit : drafts = list ( deposit [ 'drafts' ] . items ( ) ) if len ( drafts ) != 1 : logger . exception ( 'Deposit {dep_pid} has multiple drafts' . format ( dep_pid = dep_pid ) ) if len ( drafts ) == 1 : draft_type , draft = drafts [ 0 ] draft_v = draft [ 'values' ] if 'prereserve_doi' in draft_v : pre_recid = str ( draft_v [ 'prereserve_doi' ] [ 'recid' ] ) pre_doi = str ( draft_v [ 'prereserve_doi' ] [ 'doi' ] ) # If pre-reserve info available, try to reserve 'recid' try : pid = PersistentIdentifier . get ( pid_type = 'recid' , pid_value = str ( pre_recid ) ) except PIDDoesNotExistError : # Reserve recid pid = PersistentIdentifier . create ( pid_type = 'recid' , pid_value = str ( pre_recid ) , object_type = 'rec' , status = PIDStatus . RESERVED ) # If pre-reserve info available, try to reserve 'doi' try : pid = PersistentIdentifier . get ( pid_type = 'doi' , pid_value = str ( pre_doi ) ) except PIDDoesNotExistError : # Reserve DOI pid = PersistentIdentifier . create ( pid_type = 'doi' , pid_value = str ( pre_doi ) , object_type = 'rec' , status = PIDStatus . RESERVED ) if RecordIdentifier . query . get ( int ( pre_recid ) ) is None : RecordIdentifier . insert ( int ( pre_recid ) ) # Store the path -> FileInstance mappings for SIPFile creation later dep_file_instances = list ( ) for file_ in files : size = file_ [ 'size' ] key = file_ [ 'name' ] # Warning: Assumes all checksums are MD5! checksum = 'md5:{0}' . format ( file_ [ 'checksum' ] ) fi = FileInstance . create ( ) fi . set_uri ( file_ [ 'path' ] , size , checksum ) ov = ObjectVersion . create ( buc , key , _file_id = fi . id ) ext = splitext ( ov . key ) [ 1 ] . lower ( ) if ext . startswith ( '.' ) : ext = ext [ 1 : ] file_meta = dict ( bucket = str ( ov . bucket . id ) , key = ov . key , checksum = ov . file . checksum , size = ov . file . size , version_id = str ( ov . version_id ) , type = ext , ) deposit [ '_files' ] . append ( file_meta ) dep_file_instances . append ( ( file_ [ 'path' ] , fi ) ) # Get a recid from SIP information recid = None if sips : recids = [ int ( sip [ 'metadata' ] [ 'recid' ] ) for sip in sips ] if len ( set ( recids ) ) > 1 : logger . error ( 'Multiple recids ({recids}) found in deposit {depid}' ' does not exists.' . format ( recids = recids , depid = dep_pid . pid_value ) ) raise DepositMultipleRecids ( dep_pid . pid_value , list ( set ( recids ) ) ) elif recids : # If only one recid recid = recids [ 0 ] for idx , sip in enumerate ( sips ) : agent = None user_id = None if sip [ 'agents' ] : agent = dict ( ip_address = empty_str_if_none ( sip [ 'agents' ] [ 0 ] . get ( 'ip_address' , "" ) ) , email = empty_str_if_none ( sip [ 'agents' ] [ 0 ] . get ( 'email_address' , "" ) ) , ) user_id = sip [ 'agents' ] [ 0 ] [ 'user_id' ] if user_id == 0 : user_id = None content = sip [ 'package' ] sip_format = 'marcxml' try : sip = SIP . create ( sip_format , content , user_id = user_id , agent = agent ) except SIPUserDoesNotExist : logger . exception ( 'User ID {user_id} referred in deposit {depid} ' 'does not exists.' . format ( user_id = user_id , depid = dep_pid . pid_value ) ) sip = SIP . create ( sip_format , content , agent = agent ) # Attach recid to SIP if recid : try : pid = PersistentIdentifier . get ( pid_type = 'recid' , pid_value = str ( recid ) ) record_sip = RecordSIP ( sip_id = sip . id , pid_id = pid . id ) db . session . add ( record_sip ) except PIDDoesNotExistError : logger . exception ( 'Record {recid} referred in ' 'Deposit {depid} does not exists.' . format ( recid = recid , depid = dep_pid . pid_value ) ) if deposit [ '_p' ] [ 'submitted' ] is True : logger . exception ( 'Pair {recid}/{depid} was submitted,' ' (should it be unpublished?).' . format ( recid = recid , depid = dep_pid . pid_value ) ) else : msg = 'Pair {recid}/{depid} was not submitted.' . format ( recid = recid , depid = dep_pid . pid_value ) logger . exception ( msg ) # Reserve recid pid = PersistentIdentifier . create ( pid_type = 'recid' , pid_value = str ( recid ) , object_type = 'rec' , status = PIDStatus . RESERVED ) if RecordIdentifier . query . get ( int ( recid ) ) is None : RecordIdentifier . insert ( int ( recid ) ) if idx == 0 : for fp , fi in dep_file_instances : sipf = SIPFile ( sip_id = sip . id , filepath = fp , file_id = fi . id ) db . session . add ( sipf ) deposit . commit ( ) return deposit
Create deposit Bucket, Files and SIPs.
Create deposit Bucket Files and SIPs .