clustering/tasks.py
def write_clustured_data_to_files(
        model, docs_labels, cluster_centers,
        docids_features, relevant_terms=None
        ):
    """Write the doc_clusterlabels and cluster_centers to files"""
    cluster_data_location = settings.ENVIRON_CLUSTERING_DATA_LOCATION
    resource = Resource(
        cluster_data_location,
        Resource.FILE_AND_ENVIRONMENT
    )
    # create another resource(folder to keep files)
    path = os.path.join(
        resource.get_resource_location(),
        'cluster_model_{}'.format(model.id)
    )
    # create the directory
    p = subprocess.Popen(['mkdir', '-p', path], stdout=subprocess.PIPE)
    _, err = p.communicate()
    if err:
        print("Couldn't create cluster data files. {}".format(err))
        return
    # now create centers file
    center_path = os.path.join(path, settings.CLUSTERS_CENTERS_FILENAME)
    center_resource = Resource(center_path, Resource.FILE)
    # convert to python float first or it won't be json serializable
    centers_data = {
        i: compress_sparse_vector([float(y) for y in x])
        for i, x in enumerate(cluster_centers)
    }
    center_resource.write_data(json.dumps(centers_data))
    # now create labels file
    labels_path = os.path.join(path, settings.CLUSTERED_DOCS_LABELS_FILENAME)
    labels_resource = Resource(labels_path, Resource.FILE)
    # create dict
    dict_data = {x: {'label': y} for x, y in docs_labels}
    for doc_id, features in docids_features.items():
        # first make json serializable
        dict_data[doc_id]['features'] = [
            float(x) if isinstance(model.model, KMeansDoc2Vec) else x
            for x in features
        ]
    # Write docs_clusterlabels
    labels_resource.write_data(json.dumps(dict_data))
    # Write relevant terms if present
    if relevant_terms is not None:
        relevant_path = os.path.join(path, settings.RELEVANT_TERMS_FILENAME)
        relevant_resource = Resource(relevant_path, Resource.FILE)
        relevant_resource.write_data(json.dumps(list(relevant_terms)))
    print("Done writing data")
Write the doc_clusterlabels and cluster_centers to files
Write the doc_clusterlabels and cluster_centers to files
Write the doc_clusterlabels and cluster_centers to files
def write_clustured_data_to_files(
        model, docs_labels, cluster_centers,
        docids_features, relevant_terms=None
        ):
    cluster_data_location = settings.ENVIRON_CLUSTERING_DATA_LOCATION
    resource = Resource(
        cluster_data_location,
        Resource.FILE_AND_ENVIRONMENT
    )
    path = os.path.join(
        resource.get_resource_location(),
        'cluster_model_{}'.format(model.id)
    )
    p = subprocess.Popen(['mkdir', '-p', path], stdout=subprocess.PIPE)
    _, err = p.communicate()
    if err:
        print("Couldn't create cluster data files. {}".format(err))
        return
    center_path = os.path.join(path, settings.CLUSTERS_CENTERS_FILENAME)
    center_resource = Resource(center_path, Resource.FILE)
    centers_data = {
        i: compress_sparse_vector([float(y) for y in x])
        for i, x in enumerate(cluster_centers)
    }
    center_resource.write_data(json.dumps(centers_data))
    labels_path = os.path.join(path, settings.CLUSTERED_DOCS_LABELS_FILENAME)
    labels_resource = Resource(labels_path, Resource.FILE)
    dict_data = {x: {'label': y} for x, y in docs_labels}
    for doc_id, features in docids_features.items():
        dict_data[doc_id]['features'] = [
            float(x) if isinstance(model.model, KMeansDoc2Vec) else x
            for x in features
        ]
    labels_resource.write_data(json.dumps(dict_data))
    if relevant_terms is not None:
        relevant_path = os.path.join(path, settings.RELEVANT_TERMS_FILENAME)
        relevant_resource = Resource(relevant_path, Resource.FILE)
        relevant_resource.write_data(json.dumps(list(relevant_terms)))
    print("Done writing data")
def write_clustured_data_to_files ( model , docs_labels , cluster_centers , docids_features , relevant_terms = None ) : cluster_data_location = settings . ENVIRON_CLUSTERING_DATA_LOCATION resource = Resource ( cluster_data_location , Resource . FILE_AND_ENVIRONMENT ) path = os . path . join ( resource . get_resource_location ( ) , 'cluster_model_{}' . format ( model . id ) ) p = subprocess . Popen ( [ 'mkdir' , '-p' , path ] , stdout = subprocess . PIPE ) _ , err = p . communicate ( ) if err : print ( "Couldn't create cluster data files. {}" . format ( err ) ) return center_path = os . path . join ( path , settings . CLUSTERS_CENTERS_FILENAME ) center_resource = Resource ( center_path , Resource . FILE ) centers_data = { i : compress_sparse_vector ( [ float ( y ) for y in x ] ) for i , x in enumerate ( cluster_centers ) } center_resource . write_data ( json . dumps ( centers_data ) ) labels_path = os . path . join ( path , settings . CLUSTERED_DOCS_LABELS_FILENAME ) labels_resource = Resource ( labels_path , Resource . FILE ) dict_data = { x : { 'label' : y } for x , y in docs_labels } for doc_id , features in docids_features . items ( ) : dict_data [ doc_id ] [ 'features' ] = [ float ( x ) if isinstance ( model . model , KMeansDoc2Vec ) else x for x in features ] labels_resource . write_data ( json . dumps ( dict_data ) ) if relevant_terms is not None : relevant_path = os . path . join ( path , settings . RELEVANT_TERMS_FILENAME ) relevant_resource = Resource ( relevant_path , Resource . FILE ) relevant_resource . write_data ( json . dumps ( list ( relevant_terms ) ) ) print ( "Done writing data" )
Write the doc_clusterlabels and cluster_centers to files
Write the doc_clusterlabels and cluster_centers to files
"""Write the doc_clusterlabels and cluster_centers to files""" # create another resource(folder to keep files) # create the directory # now create centers file # convert to python float first or it won't be json serializable # now create labels file # create dict # first make json serializable # Write docs_clusterlabels # Write relevant terms if present
model
docs_labels
cluster_centers
docids_features
relevant_terms