# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # CDA/GDC/PDC example for subject 09CO022
#
# ## Introduction
#
# This example was [developed by <NAME>](https://github.com/ianfore/cdatest/blob/8002ae1624b34e0e2f291022c295d4b778989de9/09CO022%20Example.ipynb) to determine if CDA could link together data from three different [NCI CRDC](https://datascience.cancer.gov/data-commons) nodes by the [CDA](https://datacommons.cancer.gov/cancer-data-aggregator).
#
# It is based on subject 09CO022 from the TCGA Colon Cancer project. This subject is found in at least three NCI CRDC nodes:
# * [Case ID 09CO022 from GDC](https://portal.gdc.cancer.gov/cases/c5421e34-e5c7-4ba5-aed9-146a5575fd8d)
# * [Case ID 09CO022 from PDC](https://proteomic.datacommons.cancer.gov/pdc/case/459e3b69-63d6-11e8-bcf1-0a2705229b82)
# * (possibly?) [Three images](https://pathology.cancerimagingarchive.net/pathdata/select.html?filter=%7B%22study%22%3A%7B%22match%22%3A%22CPTAC%22%7D%2C%22subject_id%22%3A%7B%22match%22%3A%2209CO022%22%7D%7D) from The Cancer Imaging Archive (TCIA).
#
# We won't include images from the TCIA for now, since I don't think those images have been imported into IDC yet.

# ## Setup
#
# We only need to install the [cda-python](https://github.com/CancerDataAggregator/cda-python) library so we can access the CDA API.

# +
import sys

# Install cdapython so we can access the CDA API.
# !{sys.executable} -m pip install git+https://github.com/CancerDataAggregator/cda-python.git
    
# Install pandas.
# !{sys.executable} -m pip install pandas
# -

# ## What do we know about Subject 09CO022?
#
# Let's start by searching each node individually.

submitter_id = '09CO022'

# ### From GDC
#
# We can search GDC using the [GDC Cases API](https://docs.gdc.cancer.gov/API/Users_Guide/Search_and_Retrieval/#cases-endpoint) for `submitted_id == 09CO022`. Note that with the GDC API, you specifically have to request that certain sections (`diagnoses` and `samples` in this example) are expanded -- other sections, such as `aliquot_ids` only include identifiers, unless we explicitly ask for them to be expanded as well.

# Load required packages.
import requests
import json
import pandas

# +
# Search by subject_id.
cases_endpt = "https://api.gdc.cancer.gov/cases"

filters = {
    "op": "in",
    "content":{
        "field": "submitter_id",
        "value": [ submitter_id ]
    }
}

params = {
    "filters": json.dumps(filters),
    "expand": "diagnoses,samples",
    "format": "JSON",
    "size": "2"
}

response = requests.get(cases_endpt, params = params)
result = json.loads(response.content)

pandas.DataFrame(result['data']['hits'])

# +
cases = result['data']['hits']

# We should only have one case for subject 09CO022.
assert len(cases) == 1
case = cases[0]

# Write this to a file.
with open('gdc_subject_09CO022.json', 'w') as f:
    json.dump(case, f, indent=4, sort_keys=True)

# What information do we have on this case?
pandas.DataFrame(cases)
# -

# #### In other formats
#
# While we anticipate that JSON will be the best format to work with, the GDC API also allows us to download data in TSV and XML formats. So let's download those as well so we can compare between them.

# +
# Write to TSV.
params = {
    "filters": json.dumps(filters),
    "expand": "diagnoses,samples",
    "format": "TSV",
    "size": "2"
}

response = requests.get(cases_endpt, params = params)

# Write this to a file.
with open('gdc_subject_09CO022.tsv', 'w') as f:
    f.write(response.text)
    
print(f"{len(response.text)} characters written to 'gdc_subject_09CO022.tsv'")

# Write to XML.
params = {
    "filters": json.dumps(filters),
    "expand": "diagnoses,samples",
    "format": "XML",
    "size": "2"
}

response = requests.get(cases_endpt, params = params)

# Write this to a file.
import xml.dom.minidom

dom = xml.dom.minidom.parseString(response.text)
pretty_xml_as_string = dom.toprettyxml()

with open('gdc_subject_09CO022.xml', 'w') as f:
    f.write(pretty_xml_as_string)
    
print(f"{len(response.text)} characters written to 'gdc_subject_09CO022.xml'")
# -

# ### From PDC
#
# PDC has a GraphQL endpoint we can use to make these queries.

# +
# Search for case submitter ID 09CO022:

# I got this query from https://pdc.cancer.gov/data-dictionary/publicapi-documentation/#!/Case/case
graphql_query = """{
    case (
        case_submitter_id: "%s"
        acceptDUA: true
    ) {
        case_id case_submitter_id project_submitter_id days_to_lost_to_followup disease_type
        index_date lost_to_followup primary_site
        externalReferences {
            external_reference_id
            reference_resource_shortname reference_resource_name reference_entity_location
        }
        demographics {
            demographic_id ethnicity gender demographic_submitter_id race cause_of_death days_to_birth
            days_to_death vital_status year_of_birth year_of_death
        }
        samples {
            sample_id sample_submitter_id sample_type sample_type_id gdc_sample_id gdc_project_id
            biospecimen_anatomic_site composition current_weight days_to_collection days_to_sample_procurement
            diagnosis_pathologically_confirmed freezing_method initial_weight intermediate_dimension is_ffpe
            longest_dimension method_of_sample_procurement oct_embedded pathology_report_uuid preservation_method
            sample_type_id shortest_dimension time_between_clamping_and_freezing time_between_excision_and_freezing
            tissue_type tumor_code tumor_code_id tumor_descriptor
            aliquots {
                aliquot_id aliquot_submitter_id analyte_type
                aliquot_run_metadata {
                    aliquot_run_metadata_id
                }
            }
        }
        diagnoses {
            diagnosis_id tissue_or_organ_of_origin age_at_diagnosis primary_diagnosis tumor_grade tumor_stage
            diagnosis_submitter_id classification_of_tumor days_to_last_follow_up days_to_last_known_disease_status
            days_to_recurrence last_known_disease_status morphology progression_or_recurrence
            site_of_resection_or_biopsy prior_malignancy ajcc_clinical_m ajcc_clinical_n ajcc_clinical_stage
            ajcc_clinical_t ajcc_pathologic_m ajcc_pathologic_n ajcc_pathologic_stage ajcc_pathologic_t
            ann_arbor_b_symptoms ann_arbor_clinical_stage ann_arbor_extranodal_involvement ann_arbor_pathologic_stage
            best_overall_response burkitt_lymphoma_clinical_variant circumferential_resection_margin
            colon_polyps_history days_to_best_overall_response days_to_diagnosis days_to_hiv_diagnosis
            days_to_new_event figo_stage hiv_positive hpv_positive_type hpv_status iss_stage laterality
            ldh_level_at_diagnosis ldh_normal_range_upper lymph_nodes_positive lymphatic_invasion_present
            method_of_diagnosis new_event_anatomic_site new_event_type overall_survival perineural_invasion_present
            prior_treatment progression_free_survival progression_free_survival_event residual_disease
            vascular_invasion_present year_of_diagnosis icd_10_code synchronous_malignancy
            tumor_largest_dimension_diameter
        }
    }
}""" % (submitter_id)

params = {
    "query": graphql_query
}

pdc_graphql_endpoint = "https://pdc.cancer.gov/graphql"

response = requests.get(pdc_graphql_endpoint, params = params)
result = json.loads(response.content)

cases = result['data']['case']
assert len(cases) == 1
case = cases[0]

# Write this to a file.
with open('pdc_subject_09CO022.json', 'w') as f:
    json.dump(case, f, indent=4, sort_keys=True)


pandas.DataFrame(cases)
# -

# ### From CDA by Case ID
#
# Note that for cases extracted from GDC/PDC, we can look up the same record on CDA by using the case_id.

# +
from cdapython import Q, columns

case_id = case['case_id']
query_by_case_id = Q(f'ResearchSubject.id = "{case_id}"')
results = query_by_case_id.run(limit=2) 
print(results)

cda_entry = results[0]

# Write this to a file.
with open(f'cda_case_{case_id}.json', 'w') as f:
    json.dump(cda_entry, f, indent=4, sort_keys=True)

# cda_entry
# -

columns()

# The big advantage with CDA is that we can query the submitter ID directly (by using the Specimen.derived_from_subject option) to get a list of all the cases associated with a particular subject.

# +
query_by_derived = Q(f'ResearchSubject.Specimen.derived_from_subject = "{submitter_id}"')
results = query_by_derived.run(limit=100)
print(results)

result_list = []
for result in results:
    result_list.append(result)

# Write this to a file.
with open(f'cda_derived_from_{submitter_id}.json', 'w') as f:
    json.dump(result_list, f, indent=4, sort_keys=True)

pandas.DataFrame(result_list)

StarcoderdataJupyterScriptsDedupFiltered