# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 3 - PySpark Python3 Medium
#     language: python
#     name: pyspark3-med
# ---

sc

# +
nodes = "c18node14.acis.ufl.edu,c18node2.acis.ufl.edu,c18node6.acis.ufl.edu,c18node10.acis.ufl.edu,c18node12.acis.ufl.edu"
index = "idigbio"
#{"bool": 
#    {"must": [
#        {"term":{"recordset":"95773ebb-2f5f-43f0-a652-bfd8d5f4707a"}}
#        ]
#    }
#},
query = """{"query": {
    "bool": {
        "must": [
            {
                "term": {"recordset": "95773ebb-2f5f-43f0-a652-bfd8d5f4707a"}
            }
        ]
}}}"""


# -

field_set = ["uuid", "licenselogourl"]
fields = ",".join(field_set)

df = (sqlContext.read.format("org.elasticsearch.spark.sql")
    .option("es.read.field.include", fields)
    .option("es.nodes", nodes)
    .option("es.query", query)
    .load("{0}/mediarecords".format(index))
)  


print(df.count())
df.printSchema()

df.head(10)

StarcoderdataJupyterScriptsDedupFiltered