def tap_query(querystr,<EOL>simbad_mirror='<STR_LIT>',<EOL>returnformat='<STR_LIT>',<EOL>forcefetch=False,<EOL>cachedir='<STR_LIT>',<EOL>verbose=True,<EOL>timeout=<NUM_LIT>,<EOL>refresh=<NUM_LIT>,<EOL>maxtimeout=<NUM_LIT>,<EOL>maxtries=<NUM_LIT:3>,<EOL>complete_query_later=False,<EOL>jitter=<NUM_LIT>):
<EOL>inputparams = TAP_PARAMS.copy()<EOL>inputparams['<STR_LIT>'] = querystr[::]<EOL>if returnformat in RETURN_FORMATS:<EOL><INDENT>inputparams['<STR_LIT>'] = returnformat<EOL><DEDENT>else:<EOL><INDENT>LOGWARNING('<STR_LIT>' %<EOL>returnformat)<EOL>inputparams['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT>if '<STR_LIT>' in cachedir:<EOL><INDENT>cachedir = os.path.expanduser(cachedir)<EOL><DEDENT>if not os.path.exists(cachedir):<EOL><INDENT>os.makedirs(cachedir)<EOL><DEDENT>xcachekey = '<STR_LIT:->'.join([repr(inputparams[x])<EOL>for x in sorted(inputparams.keys())])<EOL>cachekey = hashlib.sha256(xcachekey.encode()).hexdigest()<EOL>cachefname = os.path.join(<EOL>cachedir,<EOL>'<STR_LIT>' % (cachekey, RETURN_FORMATS[returnformat])<EOL>)<EOL>provenance = '<STR_LIT>'<EOL>incomplete_qpklf = os.path.join(<EOL>cachedir,<EOL>'<STR_LIT>' % cachekey<EOL>)<EOL>if (not forcefetch and<EOL>complete_query_later and<EOL>os.path.exists(incomplete_qpklf)):<EOL><INDENT>with open(incomplete_qpklf, '<STR_LIT:rb>') as infd:<EOL><INDENT>incomplete_qinfo = pickle.load(infd)<EOL><DEDENT>LOGWARNING('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL>waitdone = False<EOL>timeelapsed = <NUM_LIT:0.0><EOL>simbad_mirror = incomplete_qinfo['<STR_LIT>']<EOL>status_url = incomplete_qinfo['<STR_LIT>']<EOL>phasekeyword = incomplete_qinfo['<STR_LIT>']<EOL>resultkeyword = incomplete_qinfo['<STR_LIT>']<EOL>while not waitdone:<EOL><INDENT>if timeelapsed > maxtimeout:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(maxtimeout,<EOL>repr(inputparams),<EOL>status_url))<EOL>return None<EOL><DEDENT>try:<EOL><INDENT>resreq = requests.get(status_url,<EOL>timeout=timeout)<EOL>resreq.raise_for_status()<EOL>resxml = parseString(resreq.text)<EOL>jobstatuselem = (<EOL>resxml.getElementsByTagName(phasekeyword)[<NUM_LIT:0>]<EOL>)<EOL>jobstatus = jobstatuselem.firstChild.toxml()<EOL>if jobstatus == '<STR_LIT>':<EOL><INDENT>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>waitdone = True<EOL><DEDENT>elif jobstatus != '<STR_LIT>':<EOL><INDENT>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% (timeelapsed, jobstatus, status_url))<EOL><DEDENT>time.sleep(refresh)<EOL>timeelapsed = timeelapsed + refresh<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(status_url,<EOL>resreq.text))<EOL>os.remove(incomplete_qpklf)<EOL>return None<EOL><DEDENT><DEDENT>except requests.exceptions.Timeout as e:<EOL><INDENT>LOGEXCEPTION(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(repr(inputparams), status_url)<EOL>)<EOL>return None<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(repr(inputparams),<EOL>status_url,<EOL>resreq.text)<EOL>)<EOL>os.remove(incomplete_qpklf)<EOL>return None<EOL><DEDENT><DEDENT>LOGINFO('<STR_LIT>')<EOL>result_url_elem = resxml.getElementsByTagName(resultkeyword)[<NUM_LIT:0>]<EOL>result_url = result_url_elem.getAttribute('<STR_LIT>')<EOL>result_nrows = result_url_elem.getAttribute('<STR_LIT>')<EOL>try:<EOL><INDENT>resreq = requests.get(result_url, timeout=timeout)<EOL>resreq.raise_for_status()<EOL>if cachefname.endswith('<STR_LIT>'):<EOL><INDENT>with gzip.open(cachefname,'<STR_LIT:wb>') as outfd:<EOL><INDENT>for chunk in resreq.iter_content(chunk_size=<NUM_LIT>):<EOL><INDENT>outfd.write(chunk)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>with open(cachefname,'<STR_LIT:wb>') as outfd:<EOL><INDENT>for chunk in resreq.iter_content(chunk_size=<NUM_LIT>):<EOL><INDENT>outfd.write(chunk)<EOL><DEDENT><DEDENT><DEDENT>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>' % result_nrows)<EOL><DEDENT>tablefname = cachefname<EOL>provenance = '<STR_LIT>'<EOL>resdict = {'<STR_LIT>':inputparams,<EOL>'<STR_LIT>':provenance,<EOL>'<STR_LIT:result>':tablefname}<EOL>os.remove(incomplete_qpklf)<EOL>return resdict<EOL><DEDENT>except requests.exceptions.Timeout as e:<EOL><INDENT>LOGEXCEPTION(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(repr(inputparams), result_url)<EOL>)<EOL>return None<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(repr(inputparams),<EOL>result_url,<EOL>resreq.status_code)<EOL>)<EOL>os.remove(incomplete_qpklf)<EOL>return None<EOL><DEDENT><DEDENT>if forcefetch or (not os.path.exists(cachefname)):<EOL><INDENT>provenance = '<STR_LIT>'<EOL>time.sleep(random.randint(<NUM_LIT:1>,jitter))<EOL>jobid = '<STR_LIT>' % time.time()<EOL>inputparams['<STR_LIT>'] = jobid<EOL>inputparams['<STR_LIT>'] = '<STR_LIT>'<EOL>try:<EOL><INDENT>waitdone = False<EOL>timeelapsed = <NUM_LIT:0.0><EOL>if simbad_mirror is not None and simbad_mirror in SIMBAD_URLS:<EOL><INDENT>tapurl = SIMBAD_URLS[simbad_mirror]['<STR_LIT:url>']<EOL>resultkeyword = SIMBAD_URLS[simbad_mirror]['<STR_LIT>']<EOL>phasekeyword = SIMBAD_URLS[simbad_mirror]['<STR_LIT>']<EOL>randkey = simbad_mirror<EOL>if '<STR_LIT>' in querystr:<EOL><INDENT>inputparams['<STR_LIT>'] = (<EOL>querystr.format(<EOL>table=SIMBAD_URLS[simbad_mirror]['<STR_LIT>']<EOL>)<EOL>)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>randkey = random.choice(list(SIMBAD_URLS.keys()))<EOL>tapurl = SIMBAD_URLS[randkey]['<STR_LIT:url>']<EOL>resultkeyword = SIMBAD_URLS[randkey]['<STR_LIT>']<EOL>phasekeyword = SIMBAD_URLS[randkey]['<STR_LIT>']<EOL>if '<STR_LIT>' in querystr:<EOL><INDENT>inputparams['<STR_LIT>'] = (<EOL>querystr.format(<EOL>table=SIMBAD_URLS[randkey]['<STR_LIT>']<EOL>)<EOL>)<EOL><DEDENT>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>' % tapurl)<EOL><DEDENT><DEDENT>if verbose:<EOL><INDENT>LOGINFO(<EOL>'<STR_LIT>'<EOL>% repr(inputparams)<EOL>)<EOL><DEDENT>mirrorok = False<EOL>ntries = <NUM_LIT:1><EOL>while (not mirrorok):<EOL><INDENT>if ntries > maxtries:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>maxtries)<EOL>return None<EOL><DEDENT>try:<EOL><INDENT>req = requests.post(tapurl,<EOL>data=inputparams,<EOL>timeout=timeout)<EOL>resp_status = req.status_code<EOL>req.raise_for_status()<EOL>mirrorok = True<EOL><DEDENT>except requests.exceptions.HTTPError as e:<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% tapurl<EOL>)<EOL>mirrorok = False<EOL>remainingmirrors = list(SIMBAD_URLS.keys())<EOL>waittime = random.choice(range(<NUM_LIT:1>,<NUM_LIT:6>))<EOL>time.sleep(waittime)<EOL>randkey = remainingmirrors[<NUM_LIT:0>]<EOL>tapurl = SIMBAD_URLS[randkey]['<STR_LIT:url>']<EOL>resultkeyword = SIMBAD_URLS[randkey]['<STR_LIT>']<EOL>phasekeyword = SIMBAD_URLS[randkey]['<STR_LIT>']<EOL>if '<STR_LIT>' in querystr:<EOL><INDENT>inputparams['<STR_LIT>'] = (<EOL>querystr.format(<EOL>table=SIMBAD_URLS[randkey]['<STR_LIT>']<EOL>)<EOL>)<EOL><DEDENT><DEDENT>except requests.exceptions.Timeout as e:<EOL><INDENT>LOGWARNING(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>)<EOL>mirrorok = False<EOL>remainingmirrors = list(SIMBAD_URLS.keys())<EOL>waittime = random.choice(range(<NUM_LIT:1>,<NUM_LIT:6>))<EOL>time.sleep(waittime)<EOL>randkey = remainingmirrors[<NUM_LIT:0>]<EOL>tapurl = SIMBAD_URLS[randkey]['<STR_LIT:url>']<EOL>resultkeyword = SIMBAD_URLS[randkey]['<STR_LIT>']<EOL>phasekeyword = SIMBAD_URLS[randkey]['<STR_LIT>']<EOL>if '<STR_LIT>' in querystr:<EOL><INDENT>inputparams['<STR_LIT>'] = (<EOL>querystr.format(<EOL>table=SIMBAD_URLS[randkey]['<STR_LIT>']<EOL>)<EOL>)<EOL><DEDENT><DEDENT>ntries = ntries + <NUM_LIT:1><EOL><DEDENT>status_url = req.url<EOL>resxml = parseString(req.text)<EOL>jobstatuselem = resxml.getElementsByTagName(phasekeyword)<EOL>if jobstatuselem:<EOL><INDENT>jobstatuselem = jobstatuselem[<NUM_LIT:0>]<EOL><DEDENT>else:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>' % phasekeyword)<EOL>LOGERROR('<STR_LIT:%s>' % req.txt)<EOL>req.close()<EOL>return None<EOL><DEDENT>jobstatus = jobstatuselem.firstChild.toxml()<EOL>if jobstatus == '<STR_LIT>':<EOL><INDENT>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>')<EOL>waitdone = True<EOL><DEDENT><DEDENT>elif jobstatus == '<STR_LIT>':<EOL><INDENT>if verbose:<EOL><INDENT>LOGERROR(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(repr(inputparams),<EOL>status_url,<EOL>req.text)<EOL>)<EOL>return None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>LOGINFO(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' % jobstatus<EOL>)<EOL><DEDENT>while not waitdone:<EOL><INDENT>if timeelapsed > maxtimeout:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(maxtimeout,<EOL>repr(inputparams),<EOL>status_url,<EOL>jobstatus))<EOL>if complete_query_later and jobstatus in ('<STR_LIT>',<EOL>'<STR_LIT>'):<EOL><INDENT>incomplete_qpklf = os.path.join(<EOL>cachedir,<EOL>'<STR_LIT>' % cachekey<EOL>)<EOL>with open(incomplete_qpklf, '<STR_LIT:wb>') as outfd:<EOL><INDENT>savedict = inputparams.copy()<EOL>savedict['<STR_LIT>'] = status_url<EOL>savedict['<STR_LIT>'] = jobstatus<EOL>savedict['<STR_LIT>'] = simbad_mirror<EOL>savedict['<STR_LIT>'] = phasekeyword<EOL>savedict['<STR_LIT>'] = resultkeyword<EOL>pickle.dump(savedict,<EOL>outfd,<EOL>pickle.HIGHEST_PROTOCOL)<EOL><DEDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>jobstatus)<EOL><DEDENT>return None<EOL><DEDENT>time.sleep(refresh)<EOL>timeelapsed = timeelapsed + refresh<EOL>try:<EOL><INDENT>resreq = requests.get(status_url, timeout=timeout)<EOL>resreq.raise_for_status()<EOL>resxml = parseString(resreq.text)<EOL>jobstatuselem = (<EOL>resxml.getElementsByTagName(phasekeyword)[<NUM_LIT:0>]<EOL>)<EOL>jobstatus = jobstatuselem.firstChild.toxml()<EOL>if jobstatus == '<STR_LIT>':<EOL><INDENT>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>waitdone = True<EOL><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>% (timeelapsed, jobstatus, status_url))<EOL><DEDENT>continue<EOL><DEDENT><DEDENT>except requests.exceptions.Timeout as e:<EOL><INDENT>LOGEXCEPTION(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(repr(inputparams), status_url)<EOL>)<EOL>return None<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(repr(inputparams),<EOL>status_url,<EOL>resreq.text)<EOL>)<EOL>return None<EOL><DEDENT><DEDENT><DEDENT>result_url_elem = resxml.getElementsByTagName(resultkeyword)[<NUM_LIT:0>]<EOL>result_url = result_url_elem.getAttribute('<STR_LIT>')<EOL>result_nrows = result_url_elem.getAttribute('<STR_LIT>')<EOL>try:<EOL><INDENT>resreq = requests.get(result_url, timeout=timeout)<EOL>resreq.raise_for_status()<EOL>if cachefname.endswith('<STR_LIT>'):<EOL><INDENT>with gzip.open(cachefname,'<STR_LIT:wb>') as outfd:<EOL><INDENT>for chunk in resreq.iter_content(chunk_size=<NUM_LIT>):<EOL><INDENT>outfd.write(chunk)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>with open(cachefname,'<STR_LIT:wb>') as outfd:<EOL><INDENT>for chunk in resreq.iter_content(chunk_size=<NUM_LIT>):<EOL><INDENT>outfd.write(chunk)<EOL><DEDENT><DEDENT><DEDENT>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>' % result_nrows)<EOL><DEDENT>tablefname = cachefname<EOL><DEDENT>except requests.exceptions.Timeout as e:<EOL><INDENT>LOGEXCEPTION(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(repr(inputparams), result_url)<EOL>)<EOL>return None<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION(<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(repr(inputparams),<EOL>result_url,<EOL>resreq.status_code)<EOL>)<EOL>return None<EOL><DEDENT><DEDENT>except requests.exceptions.HTTPError as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>'<EOL>'<STR_LIT>' % (resp_status,<EOL>repr(inputparams)))<EOL>return None<EOL><DEDENT>except requests.exceptions.Timeout as e:<EOL><INDENT>LOGERROR('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT:%s>' % repr(inputparams))<EOL>return None<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>'<EOL>'<STR_LIT:%s>' % repr(inputparams))<EOL>if '<STR_LIT>' in locals():<EOL><INDENT>LOGERROR('<STR_LIT>' % req.text)<EOL><DEDENT>return None<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if verbose:<EOL><INDENT>LOGINFO('<STR_LIT>'<EOL>'<STR_LIT>' %<EOL>(repr(inputparams)))<EOL><DEDENT>tablefname = cachefname<EOL>try:<EOL><INDENT>infd = gzip.open(cachefname,'<STR_LIT:rb>')<EOL>simbad_objectnames = np.genfromtxt(<EOL>infd,<EOL>names=True,<EOL>delimiter='<STR_LIT:U+002C>',<EOL>dtype='<STR_LIT>',<EOL>usecols=(<NUM_LIT:0>,<NUM_LIT:1>,<NUM_LIT:2>,<NUM_LIT:3>,<NUM_LIT:4>,<NUM_LIT:5>,<NUM_LIT:6>,<NUM_LIT:7>,<NUM_LIT:8>),<EOL>comments='<STR_LIT:?>',  <EOL>)<EOL>infd.close()<EOL><DEDENT>except Exception as e:<EOL><INDENT>LOGEXCEPTION('<STR_LIT>'<EOL>'<STR_LIT>' % cachefname)<EOL>return tap_query(querystr,<EOL>simbad_mirror=simbad_mirror,<EOL>returnformat=returnformat,<EOL>forcefetch=True,<EOL>cachedir=cachedir,<EOL>verbose=verbose,<EOL>timeout=timeout,<EOL>refresh=refresh,<EOL>maxtimeout=maxtimeout)<EOL><DEDENT><DEDENT>resdict = {'<STR_LIT>':inputparams,<EOL>'<STR_LIT>':provenance,<EOL>'<STR_LIT:result>':tablefname}<EOL>return resdict<EOL>
This queries the SIMBAD TAP service using the ADQL query string provided.

    Parameters
    ----------

    querystr : str
        This is the ADQL query string. See:
        http://www.ivoa.net/documents/ADQL/2.0 for the specification.

    simbad_mirror : str
        This is the key used to select a SIMBAD mirror from the
        `SIMBAD_URLS` dict above. If set, the specified mirror will be used. If
        None, a random mirror chosen from that dict will be used.

    returnformat : {'csv','votable','json'}
        The returned file format to request from the GAIA catalog service.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our initial request.

    refresh : float
        This sets the amount of time in seconds to wait before checking if the
        result file is available. If the results file isn't available after
        `refresh` seconds have elapsed, the function will wait for `refresh`
        seconds continuously, until `maxtimeout` is reached or the results file
        becomes available.

    maxtimeout : float
        The maximum amount of time in seconds to wait for a result to become
        available after submitting our query request.

    maxtries : int
        The maximum number of tries (across all mirrors tried) to make to either
        submit the request or download the results, before giving up.

    complete_query_later : bool
        If set to True, a submitted query that does not return a result before
        `maxtimeout` has passed will be cancelled but its input request
        parameters and the result URL provided by the service will be saved. If
        this function is then called later with these same input request
        parameters, it will check if the query finally finished and a result is
        available. If so, will download the results instead of submitting a new
        query. If it's not done yet, will start waiting for results again. To
        force launch a new query with the same request parameters, set the
        `forcefetch` kwarg to True.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    Returns
    -------

    dict
        This returns a dict of the following form::

            {'params':dict of the input params used for the query,
             'provenance':'cache' or 'new download',
             'result':path to the file on disk with the downloaded data table}