# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Environment (conda_tensorflow_p36)
#     language: python
#     name: python3
# ---

#
#
# ![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)
#
# [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/healthcare/Visit_Normalization.ipynb)
#

# ## 1. Colab Setup

# Import license keys

# + colab={"base_uri": "https://localhost:8080/", "height": 73, "resources": {"http://localhost:8080/nbextensions/google.colab/files.js": {"data": "<KEY>", "headers": [["content-type", "application/javascript"]], "ok": true, "status": 200, "status_text": ""}}} id="swFak06MUMGN" outputId="0ece349b-1457-4077-c346-959673452752"
import json
import os

from google.colab import files

license_keys = files.upload()

with open(list(license_keys.keys())[0]) as f:
    license_keys = json.load(f)

# Defining license key-value pairs as local variables
locals().update(license_keys)

# Adding license key-value pairs to environment variables
os.environ.update(license_keys)
# -

# Install dependencies

# + id="e80O9g4lUUdQ"
# Installing pyspark and spark-nlp
# ! pip install --upgrade -q pyspark==3.1.2 spark-nlp==$PUBLIC_VERSION

# Installing Spark NLP Healthcare
# ! pip install --upgrade -q spark-nlp-jsl==$JSL_VERSION  --extra-index-url https://pypi.johnsnowlabs.com/$SECRET

# Installing Spark NLP Display Library for visualization
# ! pip install -q spark-nlp-display
# -

# Import dependencies into Python and start the Spark session

# + colab={"base_uri": "https://localhost:8080/"} id="RvxNLVuJUUEX" outputId="4634ac59-bafe-4c71-ec32-9db86010ff18"
import json
import os
from pyspark.ml import Pipeline,PipelineModel
from pyspark.sql import SparkSession

from sparknlp.annotator import *
from sparknlp_jsl.annotator import *
from sparknlp.base import *
import sparknlp_jsl
import sparknlp

from pyspark.sql import functions as F

params = {"spark.driver.memory":"16G",
"spark.kryoserializer.buffer.max":"2000M",
"spark.driver.maxResultSize":"2000M"}

spark = sparknlp_jsl.start(license_keys['SECRET'],params=params)

print ("Spark NLP Version :", sparknlp.version())
print ("Spark NLP_JSL Version :", sparknlp_jsl.version())
# -

# ## 2. Select the NER model and construct the pipeline

# + id="fI_tI1vRUGM_"
model = 'visit_normalization'

# + id="hp3kShVqUGM_"
input_list = ["""Sample Name: Mesothelioma - Pleural Biopsy
Description: Right pleural effusion and suspected malignant mesothelioma. (Medical Transcription Sample Report)
PREOPERATIVE DIAGNOSIS:  Right pleural effusion and suspected malignant mesothelioma.
POSTOPERATIVE DIAGNOSIS: Right pleural effusion, suspected malignant mesothelioma.
PROCEDURE:  Right VATS pleurodesis and pleural biopsy.
ANESTHESIA: General double-lumen endotracheal.
DESCRIPTION OF FINDINGS:  Right pleural effusion, firm nodules, diffuse scattered throughout the right pleura and diaphragmatic surface.
SPECIMEN:  Pleural biopsies for pathology and microbiology.
INDICATIONS:  Briefly, this is a 66-year-old gentleman who has been transferred from an outside hospital after a pleural effusion had been drained and biopsies taken from the right chest that were thought to be consistent with mesothelioma. Upon transfer, he had a right pleural effusion demonstrated on x-ray as well as some shortness of breath and dyspnea on exertion. The risks, benefits, and alternatives to right VATS pleurodesis and pleural biopsy were discussed with the patient and his family and they wished to proceed.
Dr. X was present for the entire procedure which was right VATS pleurodesis and pleural biopsies.The counts were correct x2 at the end of the case.""",
             """Sample Name: MediPort Placement
Description: Rhabdomyosarcoma of the left orbit. Left subclavian vein MediPort placement. Needs chemotherapy. (Medical Transcription Sample Report)
PREOPERATIVE DIAGNOSIS: Rhabdomyosarcoma of the left orbit.
POSTOPERATIVE DIAGNOSIS: Rhabdomyosarcoma of the left orbit.
PROCEDURE:  Left subclavian vein MediPort placement (7.5-French single-lumen).
INDICATIONS FOR PROCEDURE:  This patient is a 16-year-old girl, with newly diagnosed rhabdomyosarcoma of the left orbit. The patient is being taken to the operating room for MediPort placement. She needs chemotherapy.
DESCRIPTION OF PROCEDURE:  The patient was taken to the operating room, placed supine, put under general endotracheal anesthesia. The patient's neck, chest, and shoulders were prepped and draped in usual sterile fashion. An incision was made on the left shoulder area. The left subclavian vein was cannulated. The wire was passed, which was in good position under fluoro, using Seldinger Technique. Near wire incision site made a pocket above the fascia and sutured in a size 7.5-French single-lumen MediPort into the pocket in 4 places using 3-0 Nurolon. I then sized the catheter under fluoro and placed introducer and dilator over the wire, removed the wire and dilator, placed the catheter through the introducer and removed the introducer. The line tip was in good position under fluoro. It withdrew and flushed well. I then closed the incision using 4-0 Vicryl, 5-0 Monocryl for the skin, and dressed with Steri-Strips. Accessed the ports with a 1-inch 20-gauge Huber needle, and it withdrew and flushed well with final heparin flush. We secured this with Tegaderm. The patient is then to undergo bilateral bone marrow biopsy and lumbar puncture by Oncology.""",
             """Sample Name: Leiomyosarcoma
Description: Discharge summary of patient with leiomyosarcoma and history of pulmonary embolism, subdural hematoma, pancytopenia, and pneumonia. (Medical Transcription Sample Report)
ADMITTING DIAGNOSES: 1. Leiomyosarcoma.2. History of pulmonary embolism.3. History of subdural hematoma.4. Pancytopenia.5. History of pneumonia.
PROCEDURES DURING HOSPITALIZATION: 1. Cycle six of CIVI-CAD (Cytoxan, Adriamycin, and DTIC) from 07/22/2008 to 07/29/2008.2. CTA, chest PE study showing no evidence for pulmonary embolism.
3. Head CT showing no evidence of acute intracranial abnormalities.4. Sinus CT, normal mini-CT of the paranasal sinuses.
HOSPITAL COURSE: 1. Leiomyosarcoma, the patient was admitted to Hem/Onco B Service under attending Dr. XYZ for cycle six of continuous IV infusion Cytoxan, Adriamycin, and DTIC, which she tolerated well.2. History of pulmonary embolism. Upon admission, the patient reported an approximate two-week history of dyspnea on exertion and some mild chest pain. She underwent a CTA, which showed no evidence of pulmonary embolism and the patient was started on prophylactic doses of Lovenox at 40 mg a day. She had no further complaints throughout the hospitalization with any shortness of breath or chest pain.3. History of subdural hematoma, also on admission the patient noted some mild intermittent headaches that were fleeting in nature, several a day that would resolve on their own. Her headaches were not responding to pain medication and so on 07/24/2008, we obtained a head CT that showed no evidence of acute intracranial abnormalities. The patient also had a history of sinusitis and so a sinus CT scan was obtained, which was normal.4. Pancytopenia. On admission, the patient's white blood count was 3.4, hemoglobin 11.3, platelet count 82, and ANC of 2400. The patient's counts were followed throughout admission. She did not require transfusion of red blood cells or platelets; however, on 07/26/2008 her ANC did dip to 900 and she was placed on neutropenic diet. At discharge her ANC is back up to 1100 and she is taken off neutropenic diet. Her white blood cell count at discharge was 1.4 and her hemoglobin was 11.2 with a platelet count of 140.
5. History of pneumonia. During admission, the patient did not exhibit any signs or symptoms of pneumonia.
DISPOSITION:  Home in stable condition.
DIET:  Regular and less neutropenic.
ACTIVITY:  Resume same activity.
FOLLOWUP: The patient will have lab work at Dr. XYZ on 08/05/2008 and she will also return to the cancer center on 08/12/2008 at 10:20 a.m. The patient is also advised to monitor for any fevers greater than 100.5 and should she have any further problems in the meantime to please call in to be seen sooner.""",
             """Sample Name: Sickle Cell Anemia - ER Visit
Description: A 19-year-old known male with sickle cell anemia comes to the emergency room on his own with 3-day history of back pain.
(Medical Transcription Sample Report)
HISTORY OF PRESENT ILLNESS:  This is a 19-year-old known male with sickle cell anemia. He comes to the emergency room on his own with 3-day history of back pain. He is on no medicines. He does live with a room mate. Appetite is decreased. No diarrhea, vomiting. Voiding well. Bowels have been regular. Denies any abdominal pain. Complains of a slight headaches, but his main concern is back ache that extends from above the lower T-spine to the lumbosacral spine. The patient is not sure of his immunizations. The patient does have sickle cell and hemoglobin is followed in the Hematology Clinic.
ALLERGIES:  THE PATIENT IS ALLERGIC TO TYLENOL WITH CODEINE, but he states he can get morphine along with Benadryl.
MEDICATIONS:  He was previously on folic acid. None at the present time.
PAST SURGICAL HISTORY:  He has had no surgeries in the past.
FAMILY HISTORY:  Positive for diabetes, hypertension and cancer.
SOCIAL HISTORY:  He denies any smoking or drug usage.
PHYSICAL EXAMINATION:  VITAL SIGNS: On examination, the patient has a temp of 37 degrees tympanic, pulse was recorded at 37 per minute, but subsequently it was noted to be 66 per minute, respiratory rate is 24 per minute and blood pressure is 149/66, recheck blood pressure was 132/72.""",
             """Sample Name: Consult - Breast Cancer - 1
Description: The patient is a 57-year-old female with invasive ductal carcinoma of the left breast, T1c, Nx, M0 left breast carcinoma.
(Medical Transcription Sample Report)
CHIEF COMPLAINT:  Left breast cancer.
MEDICATION: She is currently on omeprazole for reflux and indigestion.
ALLERGIES: SHE HAS NO KNOWN DRUG ALLERGIES.
REVIEW OF SYSTEMS: Negative for any recent febrile illnesses, chest pains or shortness of breath. Positive for restless leg syndrome. Negative for any unexplained weight loss and no change in bowel or bladder habits.
FAMILY HISTORY: Positive for breast cancer in her mother and also mesothelioma from possible asbestosis or asbestos exposure.
SOCIAL HISTORY: The patient works as a school teacher and teaching high school.
PHYSICAL EXAMINATION: The patient is a white female, alert and oriented x 3, appears her stated age of 57. Head is atraumatic and normocephalic. Sclerae are anicteric. Regular rate and rhythm. 
RECOMMENDATIONS:  I have discussed with the patient in detail about the diagnosis of breast cancer and the surgical options, and medical oncologist has discussed with her issues about adjuvant or neoadjuvant chemotherapy. We have decided to recommend to the patient breast conservation surgery with left breast lumpectomy with preoperative sentinel lymph node injection and mapping and left axillary dissection. The possibility of further surgery requiring wider lumpectomy or even completion mastectomy was explained to the patient. The procedure and risks of the surgery were explained to include, but not limited to extra bleeding, infection, unsightly scar formation, the possibility of local recurrence, the possibility of left upper extremity lymphedema was explained. Local numbness, paresthesias or chronic pain was explained. The patient was given an educational brochure and several brochures about the diagnosis and treatment of breast cancers. She was certainly encouraged to obtain further surgical medical opinions prior to proceeding. I believe the patient has given full informed consent and desires to proceed with the above.""",
             """Sample Name: Consult - Breast Cancer - 1
Description: A female with a history of peritoneal mesothelioma who has received prior intravenous chemotherapy.
(Medical Transcription Sample Report)
REASON FOR ADMISSION: Intraperitoneal chemotherapy.
MEDICATIONS:  Norco 10 per 325 one to two p.o. q.4h. p.r.n. pain, atenolol 50 mg p.o. b.i.d., Levoxyl 75 mcg p.o. daily, Phenergan 25 mg p.o. q.4-6h. p.r.n. nausea, lorazepam 0.5 mg every 8 hours as needed for anxiety, Ventolin HFA 2 puffs q.6h. p.r.n., Plavix 75 mg p.o. daily, Norvasc 10 mg p.o. daily, Cymbalta 60 mg p.o. daily, and Restoril 30 mg at bedtime as needed for sleep.
ALLERGIES:  THE PATIENT STATES THAT ON OCCASION LORAZEPAM DOSE PRODUCE HALLUCINATIONS, AND SHE HAD DIFFICULTY TOLERATING ATIVAN.
PHYSICAL EXAMINATION: Vital Signs : The patient's height is 165 cm, weight is 77 kg. BSA is 1.8 sq m. The vital signs reveal blood pressure to be 158/75, heart rate 61 per minute with a regular sinus rhythm, temperature of 96.6 degrees, respiratory rate 18 with an SpO2 of 100% on room air. She is normally developed; well nourished; very cooperative; oriented to person, place, and time; and in no distress at this time. She is anicteric.
DIAGNOSTIC IMPRESSION:
1. Intraperitoneal mesothelioma, partial remission, as noted by CT scan of the abdomen.2. Presumed left lower pole kidney hemorrhagic cyst.3. History of hypertension.4. Type 1 bipolar disease.
PLAN:  The patient will have appropriate laboratory studies done. A left renal ultrasound is requested to further delineate the possible hemorrhagic cyst in the lower left pole of the left kidney. Interventional radiology will access for ports in the abdomen. She will receive chemotherapy intraperitoneally. The plan will be to use intraperitoneal Taxol."""]

# + colab={"base_uri": "https://localhost:8080/"} id="EPF7EV4tUGNI" outputId="27e7f645-b835-461c-9635-a7764b6b8f5f"
documentAssembler = DocumentAssembler()\
      .setInputCol("text")\
      .setOutputCol("document")
 
sentenceDetector = SentenceDetectorDLModel.pretrained()\
      .setInputCols(["document"])\
      .setOutputCol("sentence")
 
tokenizer = Tokenizer()\
      .setInputCols(["sentence"])\
      .setOutputCol("token")\
 
word_embeddings = WordEmbeddingsModel.pretrained("embeddings_clinical", "en", "clinical/models")\
      .setInputCols(["sentence", "token"])\
      .setOutputCol("embeddings")
 
c2doc = Chunk2Doc()\
      .setInputCols("ner_chunk")\
      .setOutputCol("ner_chunk_doc") 
 
clinical_ner = MedicalNerModel.pretrained("ner_jsl_slim", "en", "clinical/models") \
      .setInputCols(["sentence", "token", "embeddings"]) \
      .setOutputCol("ner")
 
ner_converter = NerConverter() \
      .setInputCols(["sentence", "token", "ner"]) \
      .setOutputCol("ner_chunk")\
      .setWhiteList(["Header"])

pipeline = Pipeline(
    stages = [
        documentAssembler,
        sentenceDetector,
        tokenizer,
        word_embeddings,
        clinical_ner,
        ner_converter
    ])
 
empty_df = spark.createDataFrame([[""]]).toDF('text')
pipeline_model = pipeline.fit(empty_df)

# + id="00g3XmF-UGNI"
files = [f"{i}.txt" for i in (range(1, len(input_list)+1))]

# + id="ZRxv2WKjUGNJ"
import pandas as pd

df = spark.createDataFrame(pd.DataFrame({'text': input_list, 'file' : files}))

# + id="7AaKoK5sUGNJ"
light_pipeline = LightPipeline(pipeline_model)

# + [markdown] id="XjgpBmXmUGNK"
# ## 3. NER Visuzalization

# + colab={"base_uri": "https://localhost:8080/"} id="Ev-y-FCOUGNL" outputId="5d2723b7-68c1-4cee-e1ba-301fce62d4d8"
result_ner = pipeline_model.transform(df)
result_ner.select(F.explode(F.arrays_zip('ner_chunk.result', 'ner_chunk.metadata')).alias("cols")) \
      .select(F.expr("cols['0']").alias("chunk"),
              F.expr("cols['1']['entity']").alias("ner_label"))\
      .show(truncate=False)
result_ner = result_ner.toPandas()

# + colab={"base_uri": "https://localhost:8080/", "height": 733} id="wH9cWgJyUGNL" outputId="8d0d9a0a-108d-43e6-c06a-0b354a12fb9d"
from sparknlp_display import NerVisualizer

for i, sample in enumerate(input_list[:1]):
    
    print("*"*30)
    print(i+1)
    print("*"*30)
    
    light_result = light_pipeline.fullAnnotate(sample)
    
    visualiser = NerVisualizer()

    visualiser.display(light_result[0], label_col='ner_chunk', document_col='document')

# + [markdown] id="gvnP_Vq4UGNN"
# ## 4. Sections

# + id="ZYntH1h0UGNN"
light_pipeline = LightPipeline(pipeline_model)

result = light_pipeline.transform(df).toPandas()

# + id="I_yD937tUGNN"
result_exploded = result[["text", 'file', "ner_chunk"]].explode("ner_chunk")

# + id="f9mj9TVkUGNO"
result_exploded['start'] = result_exploded.ner_chunk.apply(lambda x: x[1])
result_exploded['end'] = result_exploded.ner_chunk.apply(lambda x: x[2])
result_exploded['section_header'] = result_exploded.ner_chunk.apply(lambda x: x[3])


# + id="bUMi6J84UGNO"
def get_text(text,sta,end):
    return text[sta:end]
df = pd.DataFrame()
for file, group in result_exploded.groupby("file"):
    group['section_start'] = group.end + 1
    group['section_end'] = group.start.shift(-1).fillna(-1).astype(int)
    group['section_text'] = group.apply(lambda x: get_text(x.text, x.section_start, x.section_end), axis=1)
    df = df.append(group[['file', 'section_header', 'section_text']])
    df.section_header = df.section_header.str.replace(":", "")

# + id="ktQmWXvfUGNO"
#export results
for file, group in df.groupby("file"):
    group.index = group.section_header
    group[['section_text']].T.to_json(OUTPUT_FILE_PATH + "HEADER_" + file.split(".")[0] + ".json")

# + colab={"base_uri": "https://localhost:8080/", "height": 133} id="Px9fgJnDfxuu" outputId="c6051d17-9e4b-474e-89bf-7a2319eb53e1"
pd.read_json(OUTPUT_FILE_PATH + '/HEADER_1.json')

# + id="KBZpOPyTgU7Q"


StarcoderdataJupyterScriptsDedupFiltered