django/projects/uicloud/cloudrestapi/data_handler.py
def listDirectoryFromHdfs(path="/", hdfsHost="spark-master0", port="50070", fileType='DIRECTORY'):
    """
    list a specified directory from HDFS using webHDFS.
    """

    rootUrl = "http://{0}:{1}/webhdfs/v1".format(hdfsHost, port)
    # headers = {'Content-Type': 'application/json'}
    listUrl = rootUrl + path + "?op=LISTSTATUS"
    resp1 = requests.get(listUrl)

    outputList = []
    if resp1.status_code < 300:
        for item in resp1.json()['FileStatuses']['FileStatus']:
            if fileType == item['type']:
                outputList.append(item['pathSuffix'])
    else:
        logger.error("response Code:{0}, response Content:{1}".format(
            resp1.status_code, resp1.json()))
        return False

    return outputList

    list a specified directory from HDFS using webHDFS.
    
list a specified directory from HDFS using webHDFS.
list a specified directory from HDFS using webHDFS .
def listDirectoryFromHdfs(path="/", hdfsHost="spark-master0", port="50070", fileType='DIRECTORY'):
    rootUrl = "http://{0}:{1}/webhdfs/v1".format(hdfsHost, port)
    listUrl = rootUrl + path + "?op=LISTSTATUS"
    resp1 = requests.get(listUrl)
    outputList = []
    if resp1.status_code < 300:
        for item in resp1.json()['FileStatuses']['FileStatus']:
            if fileType == item['type']:
                outputList.append(item['pathSuffix'])
    else:
        logger.error("response Code:{0}, response Content:{1}".format(
            resp1.status_code, resp1.json()))
        return False
    return outputList
def listDirectoryFromHdfs ( path = "/" , hdfsHost = "spark-master0" , port = "50070" , fileType = 'DIRECTORY' ) : rootUrl = "http://{0}:{1}/webhdfs/v1" . format ( hdfsHost , port ) listUrl = rootUrl + path + "?op=LISTSTATUS" resp1 = requests . get ( listUrl ) outputList = [ ] if resp1 . status_code < 300 : for item in resp1 . json ( ) [ 'FileStatuses' ] [ 'FileStatus' ] : if fileType == item [ 'type' ] : outputList . append ( item [ 'pathSuffix' ] ) else : logger . error ( "response Code:{0}, response Content:{1}" . format ( resp1 . status_code , resp1 . json ( ) ) ) return False return outputList
list a specified directory from HDFS using webHDFS.
list a specified directory from HDFS using webHDFS .
"""
    list a specified directory from HDFS using webHDFS.
    """ # headers = {'Content-Type': 'application/json'}
path
hdfsHost
port
fileType