Scraping Clinical Studies database (via ClinicalTrials.Gov) I have written the following script to scrape data from the U.S. National Library of Medicine website ClinicalTrials.Gov based on an NCTID.

def clinicalTrialsGov (nctid):
    data = BeautifulSoup(requests.get("https://clinicaltrials.gov/ct2/show/" + nctid + "?displayxml=true").text, "xml")
    subset = ['study_type', 'allocation', 'intervention_model', 'primary_purpose', 'masking', 'enrollment', 'official_title', 'condition', 'minimum_age', 'maximum_age', 'gender', 'healthy_volunteers', 'phase', 'primary_outcome', 'secondary_outcome', 'number_of_arms']
    tag_matches = data.find_all(subset)
    tag_dict = dict((str('ct' + tag_matches[i].name.capitalize()), tag_matches[i].text) for i in range(0, len(tag_matches)))
    tag_dict = multipleFields(data, ['intervention_name'], tag_dict)
    tag_dict = multipleFields(data, ['intervention_type'], tag_dict)
    tag_dict = multipleFields(data, ['arm_group_type'], tag_dict)
    tag_dict['ctID'] = nctid
    #for key in tag_dict:
        #print(key + ': ' + tag_dict[key])
    return removeEmptyKeys(tag_dict)

def multipleFields (data, subset, tagDict):
    fields = data.find_all(subset)
    field = []
    try:
        for each in fields:
            field.append(each.text)
        tagDict[str('ct' + subset[0].capitalize())] = ", ".join(field)
        return tagDict
    except:
        return tagDict

def removeEmptyKeys (dict1):
    newDict = {}
    for key in dict1:
        if str(dict1[key]) is not '':
            newDict[key] = dict1[key]
    return newDict


What can I do to make this process more efficent?
I hope I'm not too late.

There are a few things you could do :

Solution 1

import requests
from bs4 import BeautifulSoup
import pprint

def clinicalTrialsGov (nctid):
    data = BeautifulSoup(requests.get("https://clinicaltrials.gov/ct2/show/" + nctid + "?displayxml=true").text, "xml")
    subset = ['study_type', 'allocation', 'intervention_model', 'primary_purpose', 'masking', 'enrollment', 'official_title', 'condition', 'minimum_age', 'maximum_age', 'gender', 'healthy_volunteers', 'phase', 'primary_outcome', 'secondary_outcome', 'number_of_arms']
    tag_matches = data.find_all(subset)
    tag_dict = {'ct' + current_tag.name.capitalize(): current_tag.text for current_tag in tag_matches}
    tag_dict = multipleFields(data, 'intervention_name', tag_dict)
    tag_dict = multipleFields(data, 'intervention_type', tag_dict)
    tag_dict = multipleFields(data, 'arm_group_type', tag_dict)
    tag_dict['ctID'] = nctid
    return removeEmptyKeys(tag_dict)

def multipleFields (data, subset, tagDict):
    fields = data.find_all(subset)
    field = [each.text for each in fields]
    tagDict['ct' + subset.capitalize()] = ", ".join(field)
    return tagDict

def removeEmptyKeys (dict1):
    newDict = {k:v for (k, v) in dict1.items() if v}
    return newDict

pprint.pprint(clinicalTrialsGov("NCT01220960"))



I have used a dictionary comprehension to define tag_dict and newDict. This is similar to a list comprehension or a generator expression but specialized for dictionaries
I have removed the try … except from multipleFields because I don't see in which case an exception will be raised (especially since you didn't specify which one you were trying to catch) 
I have presumed that subset in multipleFields() is a string and not a list of strings since you were looking only for one tag
I have used a list comprehension to define field in multipleFields()
I have used the pprint module to see better the answer.