ndio/remote/ndingest.py
def verify_path(self, data, verifytype):
        """
        Verify the path supplied.
        """
        # Insert try and catch blocks
        try:
            token_name = data["project"]["token_name"]
        except:
            token_name = data["project"]["project_name"]

        channel_names = list(data["channels"].copy().keys())
        imgsz = data['dataset']['imagesize']

        for i in range(0, len(channel_names)):
            channel_type = data["channels"][
                channel_names[i]]["channel_type"]
            path = data["channels"][channel_names[i]]["data_url"]
            aws_pattern = re.compile("^(http:\/\/)(.+)(\.s3\.amazonaws\.com)")
            file_type = data["channels"][channel_names[i]]["file_type"]
            if "offset" in data["dataset"]:
                offset = data["dataset"]["offset"][0]
            else:
                offset = 0

            if (aws_pattern.match(path)):
                verifytype = VERIFY_BY_SLICE

            if (channel_type == "timeseries"):
                timerange = data["dataset"]["timerange"]
                try:
                    assert(timerange[0] != timerange[1])
                except AssertionError:
                    raise ValueError('Timeseries values are the same, did you\
specify the time steps?')
                for j in range(timerange[0], timerange[1] + 1):
                    # Test for tifs or such? Currently test for just not
                    # empty
                    if (verifytype == VERIFY_BY_FOLDER):
                        work_path = "{}/{}/{}/time{}/".format(
                            path, token_name, channel_names[i], ("%04d" % j))
                    elif (verifytype == VERIFY_BY_SLICE):
                        work_path = "{}/{}/{}/time{}/{}.{}".format(
                            path, token_name, channel_names[i], ("%04d" % j),
                            ("%04d" % offset), file_type)
                    else:
                        raise TypeError('Incorrect verify method')
                    # Check for accessibility
                    try:
                        if (verifytype == VERIFY_BY_FOLDER):
                            resp = requests.head(work_path)
                            assert(resp.status_code == 200)
                        elif (verifytype == VERIFY_BY_SLICE):
                            resp = requests.get(
                                work_path, stream=True, verify=False)
                            with open('/tmp/img.{}'.format(file_type),
                                      'wb') as out_file:
                                shutil.copyfileobj(resp.raw, out_file)
                            out_file.close()
                            assert(resp.status_code == 200)
                            resp.close()
                    except AssertionError:
                        raise OSError('Files are not http accessible: \
                            Error: {}, Path: {}'.format(resp.status_code,
                                                        work_path))
                    # Attempt to Verify imagesize here

                    try:
                        if (verifytype == VERIFY_BY_SLICE):
                            assert(list(self.identify_imagesize(file_type)) ==
                                   imgsz[0:2])
                    except:
                        raise ValueError('File image size does not match\
provided image size.')

            else:
                # Test for tifs or such? Currently test for just not empty
                if (verifytype == VERIFY_BY_FOLDER):
                    work_path = "{}/{}/{}/".format(
                        path, token_name, channel_names[i])
                elif (verifytype == VERIFY_BY_SLICE):
                    work_path = "{}/{}/{}/{}.{}".format(
                        path, token_name, channel_names[i],
                        ("%04d" % offset), file_type)
                else:
                    raise TypeError('Incorrect verify method')
                # Check for accessibility
                if (verifytype == VERIFY_BY_FOLDER):
                    resp = requests.head(work_path)
                elif (verifytype == VERIFY_BY_SLICE):
                    resp = requests.get(work_path, stream=True, verify=False)
                    with open('/tmp/img.{}'.format(file_type),
                              'wb') as out_file:
                        shutil.copyfileobj(resp.raw, out_file)
                    out_file.close()
                    resp.close()
                if (resp.status_code >= 300):
                    raise OSError('Files are not http accessible: \
                            Error: {}, Path: {}'.format(resp.status_code,
                                                        work_path))
                # Attempt to Verify imagesize here

                try:
                    if (verifytype == VERIFY_BY_SLICE):
                        assert(list(self.identify_imagesize(file_type)) ==
                               imgsz[0:2])
                except:
                    raise ValueError('File image size does not match\
provided image size.')
def verify_path(self, data, verifytype):
        """
        Verify the path supplied.
        """
        # Insert try and catch blocks
        try:
            token_name = data["project"]["token_name"]
        except:
            token_name = data["project"]["project_name"]

        channel_names = list(data["channels"].copy().keys())
        imgsz = data['dataset']['imagesize']

        for i in range(0, len(channel_names)):
            channel_type = data["channels"][
                channel_names[i]]["channel_type"]
            path = data["channels"][channel_names[i]]["data_url"]
            aws_pattern = re.compile("^(http:\/\/)(.+)(\.s3\.amazonaws\.com)")
            file_type = data["channels"][channel_names[i]]["file_type"]
            if "offset" in data["dataset"]:
                offset = data["dataset"]["offset"][0]
            else:
                offset = 0

            if (aws_pattern.match(path)):
                verifytype = VERIFY_BY_SLICE

            if (channel_type == "timeseries"):
                timerange = data["dataset"]["timerange"]
                try:
                    assert(timerange[0] != timerange[1])
                except AssertionError:
                    raise ValueError('Timeseries values are the same, did you\
specify the time steps?')
                for j in range(timerange[0], timerange[1] + 1):
                    # Test for tifs or such? Currently test for just not
                    # empty
                    if (verifytype == VERIFY_BY_FOLDER):
                        work_path = "{}/{}/{}/time{}/".format(
                            path, token_name, channel_names[i], ("%04d" % j))
                    elif (verifytype == VERIFY_BY_SLICE):
                        work_path = "{}/{}/{}/time{}/{}.{}".format(
                            path, token_name, channel_names[i], ("%04d" % j),
                            ("%04d" % offset), file_type)
                    else:
                        raise TypeError('Incorrect verify method')
                    # Check for accessibility
                    try:
                        if (verifytype == VERIFY_BY_FOLDER):
                            resp = requests.head(work_path)
                            assert(resp.status_code == 200)
                        elif (verifytype == VERIFY_BY_SLICE):
                            resp = requests.get(
                                work_path, stream=True, verify=False)
                            with open('/tmp/img.{}'.format(file_type),
                                      'wb') as out_file:
                                shutil.copyfileobj(resp.raw, out_file)
                            out_file.close()
                            assert(resp.status_code == 200)
                            resp.close()
                    except AssertionError:
                        raise OSError('Files are not http accessible: \
                            Error: {}, Path: {}'.format(resp.status_code,
                                                        work_path))
                    # Attempt to Verify imagesize here

                    try:
                        if (verifytype == VERIFY_BY_SLICE):
                            assert(list(self.identify_imagesize(file_type)) ==
                                   imgsz[0:2])
                    except:
                        raise ValueError('File image size does not match\
provided image size.')

            else:
                # Test for tifs or such? Currently test for just not empty
                if (verifytype == VERIFY_BY_FOLDER):
                    work_path = "{}/{}/{}/".format(
                        path, token_name, channel_names[i])
                elif (verifytype == VERIFY_BY_SLICE):
                    work_path = "{}/{}/{}/{}.{}".format(
                        path, token_name, channel_names[i],
                        ("%04d" % offset), file_type)
                else:
                    raise TypeError('Incorrect verify method')
                # Check for accessibility
                if (verifytype == VERIFY_BY_FOLDER):
                    resp = requests.head(work_path)
                elif (verifytype == VERIFY_BY_SLICE):
                    resp = requests.get(work_path, stream=True, verify=False)
                    with open('/tmp/img.{}'.format(file_type),
                              'wb') as out_file:
                        shutil.copyfileobj(resp.raw, out_file)
                    out_file.close()
                    resp.close()
                if (resp.status_code >= 300):
                    raise OSError('Files are not http accessible: \
                            Error: {}, Path: {}'.format(resp.status_code,
                                                        work_path))
                # Attempt to Verify imagesize here

                try:
                    if (verifytype == VERIFY_BY_SLICE):
                        assert(list(self.identify_imagesize(file_type)) ==
                               imgsz[0:2])
                except:
                    raise ValueError('File image size does not match\
provided image size.')
def verify_path ( self , data , verifytype ) : # Insert try and catch blocks try : token_name = data [ "project" ] [ "token_name" ] except : token_name = data [ "project" ] [ "project_name" ] channel_names = list ( data [ "channels" ] . copy ( ) . keys ( ) ) imgsz = data [ 'dataset' ] [ 'imagesize' ] for i in range ( 0 , len ( channel_names ) ) : channel_type = data [ "channels" ] [ channel_names [ i ] ] [ "channel_type" ] path = data [ "channels" ] [ channel_names [ i ] ] [ "data_url" ] aws_pattern = re . compile ( "^(http:\/\/)(.+)(\.s3\.amazonaws\.com)" ) file_type = data [ "channels" ] [ channel_names [ i ] ] [ "file_type" ] if "offset" in data [ "dataset" ] : offset = data [ "dataset" ] [ "offset" ] [ 0 ] else : offset = 0 if ( aws_pattern . match ( path ) ) : verifytype = VERIFY_BY_SLICE if ( channel_type == "timeseries" ) : timerange = data [ "dataset" ] [ "timerange" ] try : assert ( timerange [ 0 ] != timerange [ 1 ] ) except AssertionError : raise ValueError ( 'Timeseries values are the same, did you\
specify the time steps?' ) for j in range ( timerange [ 0 ] , timerange [ 1 ] + 1 ) : # Test for tifs or such? Currently test for just not # empty if ( verifytype == VERIFY_BY_FOLDER ) : work_path = "{}/{}/{}/time{}/" . format ( path , token_name , channel_names [ i ] , ( "%04d" % j ) ) elif ( verifytype == VERIFY_BY_SLICE ) : work_path = "{}/{}/{}/time{}/{}.{}" . format ( path , token_name , channel_names [ i ] , ( "%04d" % j ) , ( "%04d" % offset ) , file_type ) else : raise TypeError ( 'Incorrect verify method' ) # Check for accessibility try : if ( verifytype == VERIFY_BY_FOLDER ) : resp = requests . head ( work_path ) assert ( resp . status_code == 200 ) elif ( verifytype == VERIFY_BY_SLICE ) : resp = requests . get ( work_path , stream = True , verify = False ) with open ( '/tmp/img.{}' . format ( file_type ) , 'wb' ) as out_file : shutil . copyfileobj ( resp . raw , out_file ) out_file . close ( ) assert ( resp . status_code == 200 ) resp . close ( ) except AssertionError : raise OSError ( 'Files are not http accessible: \
                            Error: {}, Path: {}' . format ( resp . status_code , work_path ) ) # Attempt to Verify imagesize here try : if ( verifytype == VERIFY_BY_SLICE ) : assert ( list ( self . identify_imagesize ( file_type ) ) == imgsz [ 0 : 2 ] ) except : raise ValueError ( 'File image size does not match\
provided image size.' ) else : # Test for tifs or such? Currently test for just not empty if ( verifytype == VERIFY_BY_FOLDER ) : work_path = "{}/{}/{}/" . format ( path , token_name , channel_names [ i ] ) elif ( verifytype == VERIFY_BY_SLICE ) : work_path = "{}/{}/{}/{}.{}" . format ( path , token_name , channel_names [ i ] , ( "%04d" % offset ) , file_type ) else : raise TypeError ( 'Incorrect verify method' ) # Check for accessibility if ( verifytype == VERIFY_BY_FOLDER ) : resp = requests . head ( work_path ) elif ( verifytype == VERIFY_BY_SLICE ) : resp = requests . get ( work_path , stream = True , verify = False ) with open ( '/tmp/img.{}' . format ( file_type ) , 'wb' ) as out_file : shutil . copyfileobj ( resp . raw , out_file ) out_file . close ( ) resp . close ( ) if ( resp . status_code >= 300 ) : raise OSError ( 'Files are not http accessible: \
                            Error: {}, Path: {}' . format ( resp . status_code , work_path ) ) # Attempt to Verify imagesize here try : if ( verifytype == VERIFY_BY_SLICE ) : assert ( list ( self . identify_imagesize ( file_type ) ) == imgsz [ 0 : 2 ] ) except : raise ValueError ( 'File image size does not match\
provided image size.' )
Verify the path supplied.
Verify the path supplied .