text: /* Copyright (c) 2001-2004, Roger Dingledine.
   * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
   * Copyright (c) 2007-2018, The Tor Project, Inc. */
  /* See LICENSE for licensing information */
  
  #define DIRCLIENT_PRIVATE
  
  #include "core/or/or.h"
  
  #include "app/config/config.h"
  #include "core/mainloop/connection.h"
  #include "core/mainloop/mainloop.h"
  #include "core/or/connection_edge.h"
  #include "core/or/policies.h"
  #include "feature/client/bridges.h"
  #include "feature/client/entrynodes.h"
  #include "feature/control/control.h"
  #include "feature/dirauth/authmode.h"
  #include "feature/dirauth/dirvote.h"
  #include "feature/dirauth/shared_random.h"
  #include "feature/dircache/dirserv.h"
  #include "feature/dirclient/dirclient.h"
  #include "feature/dirclient/dlstatus.h"
  #include "feature/dircommon/consdiff.h"
  #include "feature/dircommon/directory.h"
  #include "feature/dircommon/fp_pair.h"
  #include "feature/hs/hs_cache.h"
  #include "feature/hs/hs_client.h"
  #include "feature/hs/hs_control.h"
  #include "feature/nodelist/authcert.h"
  #include "feature/nodelist/describe.h"
  #include "feature/nodelist/dirlist.h"
  #include "feature/nodelist/microdesc.h"
  #include "feature/nodelist/networkstatus.h"
  #include "feature/nodelist/node_select.h"
  #include "feature/nodelist/nodelist.h"
  #include "feature/nodelist/routerinfo.h"
  #include "feature/nodelist/routerlist.h"
  #include "feature/nodelist/routerset.h"
  #include "feature/relay/routermode.h"
  #include "feature/relay/selftest.h"
  #include "feature/rend/rendcache.h"
  #include "feature/rend/rendclient.h"
  #include "feature/rend/rendcommon.h"
  #include "feature/rend/rendservice.h"
  #include "feature/stats/predict_ports.h"
  
  #include "lib/compress/compress.h"
  #include "lib/crypt_ops/crypto_format.h"
  #include "lib/crypt_ops/crypto_util.h"
  #include "lib/encoding/confline.h"
  #include "lib/err/backtrace.h"
  
  #include "core/or/entry_connection_st.h"
  #include "feature/dircache/cached_dir_st.h"
  #include "feature/dirclient/dir_server_st.h"
  #include "feature/dircommon/dir_connection_st.h"
  #include "feature/nodelist/networkstatus_st.h"
  #include "feature/nodelist/node_st.h"
  #include "feature/nodelist/routerinfo_st.h"
  #include "feature/rend/rend_service_descriptor_st.h"
  
  /** Maximum size, in bytes, for any directory object that we've downloaded. */
  #define MAX_DIR_DL_SIZE ((1<<24)-1) /* 16 MB - 1 */
  
  /** How far in the future do we allow a directory server to tell us it is
   * before deciding that one of us has the wrong time? */
  #define ALLOW_DIRECTORY_TIME_SKEW (30*60)
  
  static int body_is_plausible(const char *body, size_t body_len, int purpose);
  static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
  static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn);
  static void connection_dir_download_cert_failed(
                                 dir_connection_t *conn, int status_code);
  static void connection_dir_retry_bridges(smartlist_t *descs);
  static void dir_routerdesc_download_failed(smartlist_t *failed,
                                             int status_code,
                                             int router_purpose,
                                             int was_extrainfo,
                                             int was_descriptor_digests);
  static void dir_microdesc_download_failed(smartlist_t *failed,
                                            int status_code,
                                            const char *dir_id);
  static void directory_send_command(dir_connection_t *conn,
                                     const int direct,
                                     const directory_request_t *req);
  static void connection_dir_close_consensus_fetches(
                     dir_connection_t *except_this_one, const char *resource);
  
  /** Return a string describing a given directory connection purpose. */
  STATIC const char *
  dir_conn_purpose_to_string(int purpose)
  {
    switch (purpose)
      {
      case DIR_PURPOSE_UPLOAD_DIR:
        return "server descriptor upload";
      case DIR_PURPOSE_UPLOAD_VOTE:
        return "server vote upload";
      case DIR_PURPOSE_UPLOAD_SIGNATURES:
        return "consensus signature upload";
      case DIR_PURPOSE_FETCH_SERVERDESC:
        return "server descriptor fetch";
      case DIR_PURPOSE_FETCH_EXTRAINFO:
        return "extra-info fetch";
      case DIR_PURPOSE_FETCH_CONSENSUS:
        return "consensus network-status fetch";
      case DIR_PURPOSE_FETCH_CERTIFICATE:
        return "authority cert fetch";
      case DIR_PURPOSE_FETCH_STATUS_VOTE:
        return "status vote fetch";
      case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
        return "consensus signature fetch";
      case DIR_PURPOSE_FETCH_RENDDESC_V2:
        return "hidden-service v2 descriptor fetch";
      case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
        return "hidden-service v2 descriptor upload";
      case DIR_PURPOSE_FETCH_HSDESC:
        return "hidden-service descriptor fetch";
      case DIR_PURPOSE_UPLOAD_HSDESC:
        return "hidden-service descriptor upload";
      case DIR_PURPOSE_FETCH_MICRODESC:
        return "microdescriptor fetch";
      }
  
    log_warn(LD_BUG, "Called with unknown purpose %d", purpose);
    return "(unknown)";
  }
  
  /** Return the requisite directory information types. */
  STATIC dirinfo_type_t
  dir_fetch_type(int dir_purpose, int router_purpose, const char *resource)
  {
    dirinfo_type_t type;
    switch (dir_purpose) {
      case DIR_PURPOSE_FETCH_EXTRAINFO:
        type = EXTRAINFO_DIRINFO;
        if (router_purpose == ROUTER_PURPOSE_BRIDGE)
          type |= BRIDGE_DIRINFO;
        else
          type |= V3_DIRINFO;
        break;
      case DIR_PURPOSE_FETCH_SERVERDESC:
        if (router_purpose == ROUTER_PURPOSE_BRIDGE)
          type = BRIDGE_DIRINFO;
        else
          type = V3_DIRINFO;
        break;
      case DIR_PURPOSE_FETCH_STATUS_VOTE:
      case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
      case DIR_PURPOSE_FETCH_CERTIFICATE:
        type = V3_DIRINFO;
        break;
      case DIR_PURPOSE_FETCH_CONSENSUS:
        type = V3_DIRINFO;
        if (resource && !strcmp(resource, "microdesc"))
          type |= MICRODESC_DIRINFO;
        break;
      case DIR_PURPOSE_FETCH_MICRODESC:
        type = MICRODESC_DIRINFO;
        break;
      default:
        log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose);
        type = NO_DIRINFO;
        break;
    }
    return type;
  }
  
  /** Return true iff <b>identity_digest</b> is the digest of a router which
   * says that it caches extrainfos.  (If <b>is_authority</b> we always
   * believe that to be true.) */
  int
  router_supports_extrainfo(const char *identity_digest, int is_authority)
  {
    const node_t *node = node_get_by_id(identity_digest);
  
    if (node && node->ri) {
      if (node->ri->caches_extra_info)
        return 1;
    }
    if (is_authority) {
      return 1;
    }
    return 0;
  }
  
  /** Return true iff any trusted directory authority has accepted our
   * server descriptor.
   *
   * We consider any authority sufficient because waiting for all of
   * them means it never happens while any authority is down; we don't
   * go for something more complex in the middle (like \>1/3 or \>1/2 or
   * \>=1/2) because that doesn't seem necessary yet.
   */
  int
  directories_have_accepted_server_descriptor(void)
  {
    const smartlist_t *servers = router_get_trusted_dir_servers();
    const or_options_t *options = get_options();
    SMARTLIST_FOREACH(servers, dir_server_t *, d, {
      if ((d->type & options->PublishServerDescriptor_) &&
          d->has_accepted_serverdesc) {
        return 1;
      }
    });
    return 0;
  }
  
  /** Start a connection to every suitable directory authority, using
   * connection purpose <b>dir_purpose</b> and uploading <b>payload</b>
   * (of length <b>payload_len</b>). The dir_purpose should be one of
   * 'DIR_PURPOSE_UPLOAD_{DIR|VOTE|SIGNATURES}'.
   *
   * <b>router_purpose</b> describes the type of descriptor we're
   * publishing, if we're publishing a descriptor -- e.g. general or bridge.
   *
   * <b>type</b> specifies what sort of dir authorities (V3,
   * BRIDGE, etc) we should upload to.
   *
   * If <b>extrainfo_len</b> is nonzero, the first <b>payload_len</b> bytes of
   * <b>payload</b> hold a router descriptor, and the next <b>extrainfo_len</b>
   * bytes of <b>payload</b> hold an extra-info document.  Upload the descriptor
   * to all authorities, and the extra-info document to all authorities that
   * support it.
   */
  void
  directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
                               dirinfo_type_t type,
                               const char *payload,
                               size_t payload_len, size_t extrainfo_len)
  {
    const or_options_t *options = get_options();
    dir_indirection_t indirection;
    const smartlist_t *dirservers = router_get_trusted_dir_servers();
    int found = 0;
    const int exclude_self = (dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
                              dir_purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
    tor_assert(dirservers);
    /* This tries dirservers which we believe to be down, but ultimately, that's
     * harmless, and we may as well err on the side of getting things uploaded.
     */
    SMARTLIST_FOREACH_BEGIN(dirservers, dir_server_t *, ds) {
        routerstatus_t *rs = &(ds->fake_status);
        size_t upload_len = payload_len;
  
        if ((type & ds->type) == 0)
          continue;
  
        if (exclude_self && router_digest_is_me(ds->digest)) {
          /* we don't upload to ourselves, but at least there's now at least
           * one authority of this type that has what we wanted to upload. */
          found = 1;
          continue;
        }
  
        if (options->StrictNodes &&
            routerset_contains_routerstatus(options->ExcludeNodes, rs, -1)) {
          log_warn(LD_DIR, "Wanted to contact authority '%s' for %s, but "
                   "it's in our ExcludedNodes list and StrictNodes is set. "
                   "Skipping.",
                   ds->nickname,
                   dir_conn_purpose_to_string(dir_purpose));
          continue;
        }
  
        found = 1; /* at least one authority of this type was listed */
        if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR)
          ds->has_accepted_serverdesc = 0;
  
        if (extrainfo_len && router_supports_extrainfo(ds->digest, 1)) {
          upload_len += extrainfo_len;
          log_info(LD_DIR, "Uploading an extrainfo too (length %d)",
                   (int) extrainfo_len);
        }
        if (purpose_needs_anonymity(dir_purpose, router_purpose, NULL)) {
          indirection = DIRIND_ANONYMOUS;
        } else if (!fascist_firewall_allows_dir_server(ds,
                                                       FIREWALL_DIR_CONNECTION,
                                                       0)) {
          if (fascist_firewall_allows_dir_server(ds, FIREWALL_OR_CONNECTION, 0))
            indirection = DIRIND_ONEHOP;
          else
            indirection = DIRIND_ANONYMOUS;
        } else {
          indirection = DIRIND_DIRECT_CONN;
        }
  
        directory_request_t *req = directory_request_new(dir_purpose);
        directory_request_set_routerstatus(req, rs);
        directory_request_set_router_purpose(req, router_purpose);
        directory_request_set_indirection(req, indirection);
        directory_request_set_payload(req, payload, upload_len);
        directory_initiate_request(req);
        directory_request_free(req);
    } SMARTLIST_FOREACH_END(ds);
    if (!found) {
      char *s = authdir_type_to_string(type);
      log_warn(LD_DIR, "Publishing server descriptor to directory authorities "
               "of type '%s', but no authorities of that type listed!", s);
      tor_free(s);
    }
  }
  
  /** Return true iff, according to the values in <b>options</b>, we should be
   * using directory guards for direct downloads of directory information. */
  STATIC int
  should_use_directory_guards(const or_options_t *options)
  {
    /* Public (non-bridge) servers never use directory guards. */
    if (public_server_mode(options))
      return 0;
    /* If guards are disabled, we can't use directory guards.
     */
    if (!options->UseEntryGuards)
      return 0;
    /* If we're configured to fetch directory info aggressively or of a
     * nonstandard type, don't use directory guards. */
    if (options->DownloadExtraInfo || options->FetchDirInfoEarly ||
        options->FetchDirInfoExtraEarly || options->FetchUselessDescriptors)
      return 0;
    return 1;
  }
  
  /** Pick an unconstrained directory server from among our guards, the latest
   * networkstatus, or the fallback dirservers, for use in downloading
   * information of type <b>type</b>, and return its routerstatus. */
  static const routerstatus_t *
  directory_pick_generic_dirserver(dirinfo_type_t type, int pds_flags,
                                   uint8_t dir_purpose,
                                   circuit_guard_state_t **guard_state_out)
  {
    const routerstatus_t *rs = NULL;
    const or_options_t *options = get_options();
  
    if (options->UseBridges)
      log_warn(LD_BUG, "Called when we have UseBridges set.");
  
    if (should_use_directory_guards(options)) {
      const node_t *node = guards_choose_dirguard(dir_purpose, guard_state_out);
      if (node)
        rs = node->rs;
    } else {
      /* anybody with a non-zero dirport will do */
      rs = router_pick_directory_server(type, pds_flags);
    }
    if (!rs) {
      log_info(LD_DIR, "No router found for %s; falling back to "
               "dirserver list.", dir_conn_purpose_to_string(dir_purpose));
      rs = router_pick_fallback_dirserver(type, pds_flags);
    }
  
    return rs;
  }
  
  /**
   * Set the extra fields in <b>req</b> that are used when requesting a
   * consensus of type <b>resource</b>.
   *
   * Right now, these fields are if-modified-since and x-or-diff-from-consensus.
   */
  static void
  dir_consensus_request_set_additional_headers(directory_request_t *req,
                                               const char *resource)
  {
    time_t if_modified_since = 0;
    uint8_t or_diff_from[DIGEST256_LEN];
    int or_diff_from_is_set = 0;
  
    /* DEFAULT_IF_MODIFIED_SINCE_DELAY is 1/20 of the default consensus
     * period of 1 hour.
     */
    const int DEFAULT_IF_MODIFIED_SINCE_DELAY = 180;
    const int32_t DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER = 72;
    const int32_t MIN_TRY_DIFF_FOR_CONSENSUS_NEWER = 0;
    const int32_t MAX_TRY_DIFF_FOR_CONSENSUS_NEWER = 8192;
    const char TRY_DIFF_FOR_CONSENSUS_NEWER_NAME[] =
      "try-diff-for-consensus-newer-than";
  
    int flav = FLAV_NS;
    if (resource)
      flav = networkstatus_parse_flavor_name(resource);
  
    int32_t max_age_for_diff = 3600 *
      networkstatus_get_param(NULL,
                              TRY_DIFF_FOR_CONSENSUS_NEWER_NAME,
                              DEFAULT_TRY_DIFF_FOR_CONSENSUS_NEWER,
                              MIN_TRY_DIFF_FOR_CONSENSUS_NEWER,
                              MAX_TRY_DIFF_FOR_CONSENSUS_NEWER);
  
    if (flav != -1) {
      /* IF we have a parsed consensus of this type, we can do an
       * if-modified-time based on it. */
      networkstatus_t *v;
      v = networkstatus_get_latest_consensus_by_flavor(flav);
      if (v) {
        /* In networks with particularly short V3AuthVotingIntervals,
         * ask for the consensus if it's been modified since half the
         * V3AuthVotingInterval of the most recent consensus. */
        time_t ims_delay = DEFAULT_IF_MODIFIED_SINCE_DELAY;
        if (v->fresh_until > v->valid_after
            && ims_delay > (v->fresh_until - v->valid_after)/2) {
          ims_delay = (v->fresh_until - v->valid_after)/2;
        }
        if_modified_since = v->valid_after + ims_delay;
        if (v->valid_after >= approx_time() - max_age_for_diff) {
          memcpy(or_diff_from, v->digest_sha3_as_signed, DIGEST256_LEN);
          or_diff_from_is_set = 1;
        }
      }
    } else {
      /* Otherwise it might be a consensus we don't parse, but which we
       * do cache.  Look at the cached copy, perhaps. */
      cached_dir_t *cd = dirserv_get_consensus(resource);
      /* We have no method of determining the voting interval from an
       * unparsed consensus, so we use the default. */
      if (cd) {
        if_modified_since = cd->published + DEFAULT_IF_MODIFIED_SINCE_DELAY;
        if (cd->published >= approx_time() - max_age_for_diff) {
          memcpy(or_diff_from, cd->digest_sha3_as_signed, DIGEST256_LEN);
          or_diff_from_is_set = 1;
        }
      }
    }
  
    if (if_modified_since > 0)
      directory_request_set_if_modified_since(req, if_modified_since);
    if (or_diff_from_is_set) {
      char hex[HEX_DIGEST256_LEN + 1];
      base16_encode(hex, sizeof(hex),
                    (const char*)or_diff_from, sizeof(or_diff_from));
      directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex);
    }
  }
  /** Start a connection to a random running directory server, using
   * connection purpose <b>dir_purpose</b>, intending to fetch descriptors
   * of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
   * Use <b>pds_flags</b> as arguments to router_pick_directory_server()
   * or router_pick_trusteddirserver().
   */
  MOCK_IMPL(void,
  directory_get_from_dirserver,(
                              uint8_t dir_purpose,
                              uint8_t router_purpose,
                              const char *resource,
                              int pds_flags,
                              download_want_authority_t want_authority))
  {
    const routerstatus_t *rs = NULL;
    const or_options_t *options = get_options();
    int prefer_authority = (directory_fetches_from_authorities(options)
                            || want_authority == DL_WANT_AUTHORITY);
    int require_authority = 0;
    int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose,
                                              resource);
    dirinfo_type_t type = dir_fetch_type(dir_purpose, router_purpose, resource);
  
    if (type == NO_DIRINFO)
      return;
  
    if (!options->FetchServerDescriptors)
      return;
  
    circuit_guard_state_t *guard_state = NULL;
    if (!get_via_tor) {
      if (options->UseBridges && !(type & BRIDGE_DIRINFO)) {
        /* We want to ask a running bridge for which we have a descriptor.
         *
         * When we ask choose_random_entry() for a bridge, we specify what
         * sort of dir fetch we'll be doing, so it won't return a bridge
         * that can't answer our question.
         */
        const node_t *node = guards_choose_dirguard(dir_purpose, &guard_state);
        if (node && node->ri) {
          /* every bridge has a routerinfo. */
          routerinfo_t *ri = node->ri;
          /* clients always make OR connections to bridges */
          tor_addr_port_t or_ap;
          directory_request_t *req = directory_request_new(dir_purpose);
          /* we are willing to use a non-preferred address if we need to */
          fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
                                               &or_ap);
          directory_request_set_or_addr_port(req, &or_ap);
          directory_request_set_directory_id_digest(req,
                                              ri->cache_info.identity_digest);
          directory_request_set_router_purpose(req, router_purpose);
          directory_request_set_resource(req, resource);
          if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS)
            dir_consensus_request_set_additional_headers(req, resource);
          directory_request_set_guard_state(req, guard_state);
          directory_initiate_request(req);
          directory_request_free(req);
        } else {
          if (guard_state) {
            entry_guard_cancel(&guard_state);
          }
          log_notice(LD_DIR, "Ignoring directory request, since no bridge "
                             "nodes are available yet.");
        }
  
        return;
      } else {
        if (prefer_authority || (type & BRIDGE_DIRINFO)) {
          /* only ask authdirservers, and don't ask myself */
          rs = router_pick_trusteddirserver(type, pds_flags);
          if (rs == NULL && (pds_flags & (PDS_NO_EXISTING_SERVERDESC_FETCH|
                                          PDS_NO_EXISTING_MICRODESC_FETCH))) {
            /* We don't want to fetch from any authorities that we're currently
             * fetching server descriptors from, and we got no match.  Did we
             * get no match because all the authorities have connections
             * fetching server descriptors (in which case we should just
             * return,) or because all the authorities are down or on fire or
             * unreachable or something (in which case we should go on with
             * our fallback code)? */
            pds_flags &= ~(PDS_NO_EXISTING_SERVERDESC_FETCH|
                           PDS_NO_EXISTING_MICRODESC_FETCH);
            rs = router_pick_trusteddirserver(type, pds_flags);
            if (rs) {
              log_debug(LD_DIR, "Deferring serverdesc fetch: all authorities "
                        "are in use.");
              return;
            }
          }
          if (rs == NULL && require_authority) {
            log_info(LD_DIR, "No authorities were available for %s: will try "
                     "later.", dir_conn_purpose_to_string(dir_purpose));
            return;
          }
        }
        if (!rs && !(type & BRIDGE_DIRINFO)) {
          rs = directory_pick_generic_dirserver(type, pds_flags,
                                                dir_purpose,
                                                &guard_state);
          if (!rs)
            get_via_tor = 1; /* last resort: try routing it via Tor */
        }
      }
    }
  
    if (get_via_tor) {
      /* Never use fascistfirewall; we're going via Tor. */
      pds_flags |= PDS_IGNORE_FASCISTFIREWALL;
      rs = router_pick_directory_server(type, pds_flags);
    }
  
    /* If we have any hope of building an indirect conn, we know some router
     * descriptors.  If (rs==NULL), we can't build circuits anyway, so
     * there's no point in falling back to the authorities in this case. */
    if (rs) {
      const dir_indirection_t indirection =
        get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP;
      directory_request_t *req = directory_request_new(dir_purpose);
      directory_request_set_routerstatus(req, rs);
      directory_request_set_router_purpose(req, router_purpose);
      directory_request_set_indirection(req, indirection);
      directory_request_set_resource(req, resource);
      if (dir_purpose == DIR_PURPOSE_FETCH_CONSENSUS)
        dir_consensus_request_set_additional_headers(req, resource);
      if (guard_state)
        directory_request_set_guard_state(req, guard_state);
      directory_initiate_request(req);
      directory_request_free(req);
    } else {
      log_notice(LD_DIR,
                 "While fetching directory info, "
                 "no running dirservers known. Will try again later. "
                 "(purpose %d)", dir_purpose);
      if (!purpose_needs_anonymity(dir_purpose, router_purpose, resource)) {
        /* remember we tried them all and failed. */
        directory_all_unreachable(time(NULL));
      }
    }
  }
  
  /** As directory_get_from_dirserver, but initiates a request to <i>every</i>
   * directory authority other than ourself.  Only for use by authorities when
   * searching for missing information while voting. */
  void
  directory_get_from_all_authorities(uint8_t dir_purpose,
                                     uint8_t router_purpose,
                                     const char *resource)
  {
    tor_assert(dir_purpose == DIR_PURPOSE_FETCH_STATUS_VOTE ||
               dir_purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
  
    SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
                            dir_server_t *, ds) {
        if (router_digest_is_me(ds->digest))
          continue;
        if (!(ds->type & V3_DIRINFO))
          continue;
        const routerstatus_t *rs = &ds->fake_status;
        directory_request_t *req = directory_request_new(dir_purpose);
        directory_request_set_routerstatus(req, rs);
        directory_request_set_router_purpose(req, router_purpose);
        directory_request_set_resource(req, resource);
        directory_initiate_request(req);
        directory_request_free(req);
    } SMARTLIST_FOREACH_END(ds);
  }
  
  /** Return true iff <b>ind</b> requires a multihop circuit. */
  static int
  dirind_is_anon(dir_indirection_t ind)
  {
    return ind == DIRIND_ANON_DIRPORT || ind == DIRIND_ANONYMOUS;
  }
  
  /* Choose reachable OR and Dir addresses and ports from status, copying them
   * into use_or_ap and use_dir_ap. If indirection is anonymous, then we're
   * connecting via another relay, so choose the primary IPv4 address and ports.
   *
   * status should have at least one reachable address, if we can't choose a
   * reachable address, warn and return -1. Otherwise, return 0.
   */
  static int
  directory_choose_address_routerstatus(const routerstatus_t *status,
                                        dir_indirection_t indirection,
                                        tor_addr_port_t *use_or_ap,
                                        tor_addr_port_t *use_dir_ap)
  {
    tor_assert(status != NULL);
    tor_assert(use_or_ap != NULL);
    tor_assert(use_dir_ap != NULL);
  
    const or_options_t *options = get_options();
    int have_or = 0, have_dir = 0;
  
    /* We expect status to have at least one reachable address if we're
     * connecting to it directly.
     *
     * Therefore, we can simply use the other address if the one we want isn't
     * allowed by the firewall.
     *
     * (When Tor uploads and downloads a hidden service descriptor, it uses
     * DIRIND_ANONYMOUS. Even Single Onion Servers (NYI) use DIRIND_ANONYMOUS,
     * to avoid HSDirs denying service by rejecting descriptors.)
     */
  
    /* Initialise the OR / Dir addresses */
    tor_addr_make_null(&use_or_ap->addr, AF_UNSPEC);
    use_or_ap->port = 0;
    tor_addr_make_null(&use_dir_ap->addr, AF_UNSPEC);
    use_dir_ap->port = 0;
  
    /* ORPort connections */
    if (indirection == DIRIND_ANONYMOUS) {
      if (status->addr) {
        /* Since we're going to build a 3-hop circuit and ask the 2nd relay
         * to extend to this address, always use the primary (IPv4) OR address */
        tor_addr_from_ipv4h(&use_or_ap->addr, status->addr);
        use_or_ap->port = status->or_port;
        have_or = 1;
      }
    } else if (indirection == DIRIND_ONEHOP) {
      /* We use an IPv6 address if we have one and we prefer it.
       * Use the preferred address and port if they are reachable, otherwise,
       * use the alternate address and port (if any).
       */
      fascist_firewall_choose_address_rs(status, FIREWALL_OR_CONNECTION, 0,
                                         use_or_ap);
      have_or = tor_addr_port_is_valid_ap(use_or_ap, 0);
    }
  
    /* DirPort connections
     * DIRIND_ONEHOP uses ORPort, but may fall back to the DirPort on relays */
    if (indirection == DIRIND_DIRECT_CONN ||
        indirection == DIRIND_ANON_DIRPORT ||
        (indirection == DIRIND_ONEHOP
         && !directory_must_use_begindir(options))) {
      fascist_firewall_choose_address_rs(status, FIREWALL_DIR_CONNECTION, 0,
                                         use_dir_ap);
      have_dir = tor_addr_port_is_valid_ap(use_dir_ap, 0);
    }
  
    /* We rejected all addresses in the relay's status. This means we can't
     * connect to it. */
    if (!have_or && !have_dir) {
      static int logged_backtrace = 0;
      log_info(LD_BUG, "Rejected all OR and Dir addresses from %s when "
               "launching an outgoing directory connection to: IPv4 %s OR %d "
               "Dir %d IPv6 %s OR %d Dir %d", routerstatus_describe(status),
               fmt_addr32(status->addr), status->or_port,
               status->dir_port, fmt_addr(&status->ipv6_addr),
               status->ipv6_orport, status->dir_port);
      if (!logged_backtrace) {
        log_backtrace(LOG_INFO, LD_BUG, "Addresses came from");
        logged_backtrace = 1;
      }
      return -1;
    }
  
    return 0;
  }
  
  /** Return true iff <b>conn</b> is the client side of a directory connection
   * we launched to ourself in order to determine the reachability of our
   * dir_port. */
  static int
  directory_conn_is_self_reachability_test(dir_connection_t *conn)
  {
    if (conn->requested_resource &&
        !strcmpstart(conn->requested_resource,"authority")) {
      const routerinfo_t *me = router_get_my_routerinfo();
      if (me &&
          router_digest_is_me(conn->identity_digest) &&
          tor_addr_eq_ipv4h(&conn->base_.addr, me->addr) && /*XXXX prop 118*/
          me->dir_port == conn->base_.port)
        return 1;
    }
    return 0;
  }
  
  /** Called when we are unable to complete the client's request to a directory
   * server due to a network error: Mark the router as down and try again if
   * possible.
   */
  void
  connection_dir_client_request_failed(dir_connection_t *conn)
  {
    if (conn->guard_state) {
      /* We haven't seen a success on this guard state, so consider it to have
       * failed. */
      entry_guard_failed(&conn->guard_state);
    }
    if (directory_conn_is_self_reachability_test(conn)) {
      return; /* this was a test fetch. don't retry. */
    }
    if (!entry_list_is_constrained(get_options()))
      router_set_status(conn->identity_digest, 0); /* don't try this one again */
    if (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
               conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
      log_info(LD_DIR, "Giving up on serverdesc/extrainfo fetch from "
               "directory server at '%s'; retrying",
               conn->base_.address);
      if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE)
        connection_dir_bridge_routerdesc_failed(conn);
      connection_dir_download_routerdesc_failed(conn);
    } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
      if (conn->requested_resource)
        networkstatus_consensus_download_failed(0, conn->requested_resource);
    } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE) {
      log_info(LD_DIR, "Giving up on certificate fetch from directory server "
               "at '%s'; retrying",
               conn->base_.address);
      connection_dir_download_cert_failed(conn, 0);
    } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
      log_info(LD_DIR, "Giving up downloading detached signatures from '%s'",
               conn->base_.address);
    } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) {
      log_info(LD_DIR, "Giving up downloading votes from '%s'",
               conn->base_.address);
    } else if (conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC) {
      log_info(LD_DIR, "Giving up on downloading microdescriptors from "
               "directory server at '%s'; will retry", conn->base_.address);
      connection_dir_download_routerdesc_failed(conn);
    }
  }
  
  /** Helper: Attempt to fetch directly the descriptors of each bridge
   * listed in <b>failed</b>.
   */
  static void
  connection_dir_retry_bridges(smartlist_t *descs)
  {
    char digest[DIGEST_LEN];
    SMARTLIST_FOREACH(descs, const char *, cp,
    {
      if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) {
        log_warn(LD_BUG, "Malformed fingerprint in list: %s",
                escaped(cp));
        continue;
      }
      retry_bridge_descriptor_fetch_directly(digest);
    });
  }
  
  /** Called when an attempt to download one or more router descriptors
   * or extra-info documents on connection <b>conn</b> failed.
   */
  static void
  connection_dir_download_routerdesc_failed(dir_connection_t *conn)
  {
    /* No need to increment the failure count for routerdescs, since
     * it's not their fault. */
  
    /* No need to relaunch descriptor downloads here: we already do it
     * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
    tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
               conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
               conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
  
    (void) conn;
  }
  
  /** Called when an attempt to download a bridge's routerdesc from
   * one of the authorities failed due to a network error. If
   * possible attempt to download descriptors from the bridge directly.
   */
  static void
  connection_dir_bridge_routerdesc_failed(dir_connection_t *conn)
  {
    smartlist_t *which = NULL;
  
    /* Requests for bridge descriptors are in the form 'fp/', so ignore
       anything else. */
    if (!conn->requested_resource || strcmpstart(conn->requested_resource,"fp/"))
      return;
  
    which = smartlist_new();
    dir_split_resource_into_fingerprints(conn->requested_resource
                                          + strlen("fp/"),
                                         which, NULL, 0);
  
    tor_assert(conn->base_.purpose != DIR_PURPOSE_FETCH_EXTRAINFO);
    if (smartlist_len(which)) {
      connection_dir_retry_bridges(which);
      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
    }
    smartlist_free(which);
  }
  
  /** Called when an attempt to fetch a certificate fails. */
  static void
  connection_dir_download_cert_failed(dir_connection_t *conn, int status)
  {
    const char *fp_pfx = "fp/";
    const char *fpsk_pfx = "fp-sk/";
    smartlist_t *failed;
    tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);
  
    if (!conn->requested_resource)
      return;
    failed = smartlist_new();
    /*
     * We have two cases download by fingerprint (resource starts
     * with "fp/") or download by fingerprint/signing key pair
     * (resource starts with "fp-sk/").
     */
    if (!strcmpstart(conn->requested_resource, fp_pfx)) {
      /* Download by fingerprint case */
      dir_split_resource_into_fingerprints(conn->requested_resource +
                                           strlen(fp_pfx),
                                           failed, NULL, DSR_HEX);
      SMARTLIST_FOREACH_BEGIN(failed, char *, cp) {
        /* Null signing key digest indicates download by fp only */
        authority_cert_dl_failed(cp, NULL, status);
        tor_free(cp);
      } SMARTLIST_FOREACH_END(cp);
    } else if (!strcmpstart(conn->requested_resource, fpsk_pfx)) {
      /* Download by (fp,sk) pairs */
      dir_split_resource_into_fingerprint_pairs(conn->requested_resource +
                                                strlen(fpsk_pfx), failed);
      SMARTLIST_FOREACH_BEGIN(failed, fp_pair_t *, cp) {
        authority_cert_dl_failed(cp->first, cp->second, status);
        tor_free(cp);
      } SMARTLIST_FOREACH_END(cp);
    } else {
      log_warn(LD_DIR,
               "Don't know what to do with failure for cert fetch %s",
               conn->requested_resource);
    }
  
    smartlist_free(failed);
  
    update_certificate_downloads(time(NULL));
  }
  
  /* Should this tor instance only use begindir for all its directory requests?
   */
  int
  directory_must_use_begindir(const or_options_t *options)
  {
    /* Clients, onion services, and bridges must use begindir,
     * relays and authorities do not have to */
    return !public_server_mode(options);
  }
  
  /** Evaluate the situation and decide if we should use an encrypted
   * "begindir-style" connection for this directory request.
   * 0) If there is no DirPort, yes.
   * 1) If or_port is 0, or it's a direct conn and or_port is firewalled
   *    or we're a dir mirror, no.
   * 2) If we prefer to avoid begindir conns, and we're not fetching or
   *    publishing a bridge relay descriptor, no.
   * 3) Else yes.
   * If returning 0, return in *reason why we can't use begindir.
   * reason must not be NULL.
   */
  static int
  directory_command_should_use_begindir(const or_options_t *options,
                                        const directory_request_t *req,
                                        const char **reason)
  {
    const tor_addr_t *or_addr = &req->or_addr_port.addr;
    //const tor_addr_t *dir_addr = &req->dir_addr_port.addr;
    const int or_port = req->or_addr_port.port;
    const int dir_port = req->dir_addr_port.port;
  
    const dir_indirection_t indirection = req->indirection;
  
    tor_assert(reason);
    *reason = NULL;
  
    /* Reasons why we must use begindir */
    if (!dir_port) {
      *reason = "(using begindir - directory with no DirPort)";
      return 1; /* We don't know a DirPort -- must begindir. */
    }
    /* Reasons why we can't possibly use begindir */
    if (!or_port) {
      *reason = "directory with unknown ORPort";
      return 0; /* We don't know an ORPort -- no chance. */
    }
    if (indirection == DIRIND_DIRECT_CONN ||
        indirection == DIRIND_ANON_DIRPORT) {
      *reason = "DirPort connection";
      return 0;
    }
    if (indirection == DIRIND_ONEHOP) {
      /* We're firewalled and want a direct OR connection */
      if (!fascist_firewall_allows_address_addr(or_addr, or_port,
                                                FIREWALL_OR_CONNECTION, 0, 0)) {
        *reason = "ORPort not reachable";
        return 0;
      }
    }
    /* Reasons why we want to avoid using begindir */
    if (indirection == DIRIND_ONEHOP) {
      if (!directory_must_use_begindir(options)) {
        *reason = "in relay mode";
        return 0;
      }
    }
    /* DIRIND_ONEHOP on a client, or DIRIND_ANONYMOUS
     */
    *reason = "(using begindir)";
    return 1;
  }
  
  /**
   * Create and return a new directory_request_t with purpose
   * <b>dir_purpose</b>.
   */
  directory_request_t *
  directory_request_new(uint8_t dir_purpose)
  {
    tor_assert(dir_purpose >= DIR_PURPOSE_MIN_);
    tor_assert(dir_purpose <= DIR_PURPOSE_MAX_);
    tor_assert(dir_purpose != DIR_PURPOSE_SERVER);
    tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2);
    tor_assert(dir_purpose != DIR_PURPOSE_HAS_FETCHED_HSDESC);
  
    directory_request_t *result = tor_malloc_zero(sizeof(*result));
    tor_addr_make_null(&result->or_addr_port.addr, AF_INET);
    result->or_addr_port.port = 0;
    tor_addr_make_null(&result->dir_addr_port.addr, AF_INET);
    result->dir_addr_port.port = 0;
    result->dir_purpose = dir_purpose;
    result->router_purpose = ROUTER_PURPOSE_GENERAL;
    result->indirection = DIRIND_ONEHOP;
    return result;
  }
  /**
   * Release all resources held by <b>req</b>.
   */
  void
  directory_request_free_(directory_request_t *req)
  {
    if (req == NULL)
      return;
    config_free_lines(req->additional_headers);
    tor_free(req);
  }
  /**
   * Set the address and OR port to use for this directory request.  If there is
   * no OR port, we'll have to connect over the dirport.  (If there are both,
   * the indirection setting determines which to use.)
   */
  void
  directory_request_set_or_addr_port(directory_request_t *req,
                                     const tor_addr_port_t *p)
  {
    memcpy(&req->or_addr_port, p, sizeof(*p));
  }
  /**
   * Set the address and dirport to use for this directory request.  If there
   * is no dirport, we'll have to connect over the OR port.  (If there are both,
   * the indirection setting determines which to use.)
   */
  void
  directory_request_set_dir_addr_port(directory_request_t *req,
                                      const tor_addr_port_t *p)
  {
    memcpy(&req->dir_addr_port, p, sizeof(*p));
  }
  /**
   * Set the RSA identity digest of the directory to use for this directory
   * request.
   */
  void
  directory_request_set_directory_id_digest(directory_request_t *req,
                                            const char *digest)
  {
    memcpy(req->digest, digest, DIGEST_LEN);
  }
  /**
   * Set the router purpose associated with uploaded and downloaded router
   * descriptors and extrainfo documents in this directory request.  The purpose
   * must be one of ROUTER_PURPOSE_GENERAL (the default) or
   * ROUTER_PURPOSE_BRIDGE.
   */
  void
  directory_request_set_router_purpose(directory_request_t *req,
                                       uint8_t router_purpose)
  {
    tor_assert(router_purpose == ROUTER_PURPOSE_GENERAL ||
               router_purpose == ROUTER_PURPOSE_BRIDGE);
    // assert that it actually makes sense to set this purpose, given
    // the dir_purpose.
    req->router_purpose = router_purpose;
  }
  /**
   * Set the indirection to be used for the directory request.  The indirection
   * parameter configures whether to connect to a DirPort or ORPort, and whether
   * to anonymize the connection.  DIRIND_ONEHOP (use ORPort, don't anonymize)
   * is the default.  See dir_indirection_t for more information.
   */
  void
  directory_request_set_indirection(directory_request_t *req,
                                    dir_indirection_t indirection)
  {
    req->indirection = indirection;
  }
  
  /**
   * Set a pointer to the resource to request from a directory.  Different
   * request types use resources to indicate different components of their URL.
   * Note that only an alias to <b>resource</b> is stored, so the
   * <b>resource</b> must outlive the request.
   */
  void
  directory_request_set_resource(directory_request_t *req,
                                 const char *resource)
  {
    req->resource = resource;
  }
  /**
   * Set a pointer to the payload to include with this directory request, along
   * with its length.  Note that only an alias to <b>payload</b> is stored, so
   * the <b>payload</b> must outlive the request.
   */
  void
  directory_request_set_payload(directory_request_t *req,
                                const char *payload,
                                size_t payload_len)
  {
    tor_assert(DIR_PURPOSE_IS_UPLOAD(req->dir_purpose));
  
    req->payload = payload;
    req->payload_len = payload_len;
  }
  /**
   * Set an if-modified-since date to send along with the request.  The
   * default is 0 (meaning, send no if-modified-since header).
   */
  void
  directory_request_set_if_modified_since(directory_request_t *req,
                                          time_t if_modified_since)
  {
    req->if_modified_since = if_modified_since;
  }
  
  /** Include a header of name <b>key</b> with content <b>val</b> in the
   * request. Neither may include newlines or other odd characters. Their
   * ordering is not currently guaranteed.
   *
   * Note that, as elsewhere in this module, header keys include a trailing
   * colon and space.
   */
  void
  directory_request_add_header(directory_request_t *req,
                               const char *key,
                               const char *val)
  {
    config_line_prepend(&req->additional_headers, key, val);
  }
  /**
   * Set an object containing HS data to be associated with this request.  Note
   * that only an alias to <b>query</b> is stored, so the <b>query</b> object
   * must outlive the request.
   */
  void
  directory_request_set_rend_query(directory_request_t *req,
                                   const rend_data_t *query)
  {
    if (query) {
      tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 ||
                 req->dir_purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2);
    }
    req->rend_query = query;
  }
  /**
   * Set an object containing HS connection identifier to be associated with
   * this request. Note that only an alias to <b>ident</b> is stored, so the
   * <b>ident</b> object must outlive the request.
   */
  void
  directory_request_upload_set_hs_ident(directory_request_t *req,
                                        const hs_ident_dir_conn_t *ident)
  {
    if (ident) {
      tor_assert(req->dir_purpose == DIR_PURPOSE_UPLOAD_HSDESC);
    }
    req->hs_ident = ident;
  }
  /**
   * Set an object containing HS connection identifier to be associated with
   * this fetch request. Note that only an alias to <b>ident</b> is stored, so
   * the <b>ident</b> object must outlive the request.
   */
  void
  directory_request_fetch_set_hs_ident(directory_request_t *req,
                                       const hs_ident_dir_conn_t *ident)
  {
    if (ident) {
      tor_assert(req->dir_purpose == DIR_PURPOSE_FETCH_HSDESC);
    }
    req->hs_ident = ident;
  }
  /** Set a static circuit_guard_state_t object to affliate with the request in
   * <b>req</b>.  This object will receive notification when the attempt to
   * connect to the guard either succeeds or fails. */
  void
  directory_request_set_guard_state(directory_request_t *req,
                                    circuit_guard_state_t *state)
  {
    req->guard_state = state;
  }
  
  /**
   * Internal: Return true if any information for contacting the directory in
   * <b>req</b> has been set, other than by the routerstatus. */
  static int
  directory_request_dir_contact_info_specified(const directory_request_t *req)
  {
    /* We only check for ports here, since we don't use an addr unless the port
     * is set */
    return (req->or_addr_port.port ||
            req->dir_addr_port.port ||
            ! tor_digest_is_zero(req->digest));
  }
  
  /**
   * Set the routerstatus to use for the directory associated with this
   * request.  If this option is set, then no other function to set the
   * directory's address or identity should be called.
   */
  void
  directory_request_set_routerstatus(directory_request_t *req,
                                     const routerstatus_t *status)
  {
    req->routerstatus = status;
  }
  /**
   * Helper: update the addresses, ports, and identities in <b>req</b>
   * from the routerstatus object in <b>req</b>.  Return 0 on success.
   * On failure, warn and return -1.
   */
  static int
  directory_request_set_dir_from_routerstatus(directory_request_t *req)
  
  {
    const routerstatus_t *status = req->routerstatus;
    if (BUG(status == NULL))
      return -1;
    const or_options_t *options = get_options();
    const node_t *node;
    tor_addr_port_t use_or_ap, use_dir_ap;
    const int anonymized_connection = dirind_is_anon(req->indirection);
  
    tor_assert(status != NULL);
  
    node = node_get_by_id(status->identity_digest);
  
    /* XXX The below check is wrong: !node means it's not in the consensus,
     * but we haven't checked if we have a descriptor for it -- and also,
     * we only care about the descriptor if it's a begindir-style anonymized
     * connection. */
    if (!node && anonymized_connection) {
      log_info(LD_DIR, "Not sending anonymized request to directory '%s'; we "
               "don't have its router descriptor.",
               routerstatus_describe(status));
      return -1;
    }
  
    if (options->ExcludeNodes && options->StrictNodes &&
        routerset_contains_routerstatus(options->ExcludeNodes, status, -1)) {
      log_warn(LD_DIR, "Wanted to contact directory mirror %s for %s, but "
               "it's in our ExcludedNodes list and StrictNodes is set. "
               "Skipping. This choice might make your Tor not work.",
               routerstatus_describe(status),
               dir_conn_purpose_to_string(req->dir_purpose));
      return -1;
    }
  
      /* At this point, if we are a client making a direct connection to a
     * directory server, we have selected a server that has at least one address
     * allowed by ClientUseIPv4/6 and Reachable{"",OR,Dir}Addresses. This
     * selection uses the preference in ClientPreferIPv6{OR,Dir}Port, if
     * possible. (If UseBridges is set, clients always use IPv6, and prefer it
     * by default.)
     *
     * Now choose an address that we can use to connect to the directory server.
     */
    if (directory_choose_address_routerstatus(status,
                                              req->indirection, &use_or_ap,
                                              &use_dir_ap) < 0) {
      return -1;
    }
  
    directory_request_set_or_addr_port(req, &use_or_ap);
    directory_request_set_dir_addr_port(req, &use_dir_ap);
    directory_request_set_directory_id_digest(req, status->identity_digest);
    return 0;
  }
  
  /**
   * Launch the provided directory request, configured in <b>request</b>.
   * After this function is called, you can free <b>request</b>.
   */
  MOCK_IMPL(void,
  directory_initiate_request,(directory_request_t *request))
  {
    tor_assert(request);
    if (request->routerstatus) {
      tor_assert_nonfatal(
                 ! directory_request_dir_contact_info_specified(request));
      if (directory_request_set_dir_from_routerstatus(request) < 0) {
        return;
      }
    }
  
    const tor_addr_port_t *or_addr_port = &request->or_addr_port;
    const tor_addr_port_t *dir_addr_port = &request->dir_addr_port;
    const char *digest = request->digest;
    const uint8_t dir_purpose = request->dir_purpose;
    const uint8_t router_purpose = request->router_purpose;
    const dir_indirection_t indirection = request->indirection;
    const char *resource = request->resource;
    const rend_data_t *rend_query = request->rend_query;
    const hs_ident_dir_conn_t *hs_ident = request->hs_ident;
    circuit_guard_state_t *guard_state = request->guard_state;
  
    tor_assert(or_addr_port->port || dir_addr_port->port);
    tor_assert(digest);
  
    dir_connection_t *conn;
    const or_options_t *options = get_options();
    int socket_error = 0;
    const char *begindir_reason = NULL;
    /* Should the connection be to a relay's OR port (and inside that we will
     * send our directory request)? */
    const int use_begindir =
      directory_command_should_use_begindir(options, request, &begindir_reason);
  
    /* Will the connection go via a three-hop Tor circuit? Note that this
     * is separate from whether it will use_begindir. */
    const int anonymized_connection = dirind_is_anon(indirection);
  
    /* What is the address we want to make the directory request to? If
     * we're making a begindir request this is the ORPort of the relay
     * we're contacting; if not a begindir request, this is its DirPort.
     * Note that if anonymized_connection is true, we won't be initiating
     * a connection directly to this address. */
    tor_addr_t addr;
    tor_addr_copy(&addr, &(use_begindir ? or_addr_port : dir_addr_port)->addr);
    uint16_t port = (use_begindir ? or_addr_port : dir_addr_port)->port;
  
    log_debug(LD_DIR, "anonymized %d, use_begindir %d.",
              anonymized_connection, use_begindir);
  
    log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
  
    if (purpose_needs_anonymity(dir_purpose, router_purpose, resource)) {
      tor_assert(anonymized_connection ||
                 rend_non_anonymous_mode_enabled(options));
    }
  
    /* use encrypted begindir connections for everything except relays
     * this provides better protection for directory fetches */
    if (!use_begindir && directory_must_use_begindir(options)) {
      log_warn(LD_BUG, "Client could not use begindir connection: %s",
               begindir_reason ? begindir_reason : "(NULL)");
      return;
    }
  
    /* ensure that we don't make direct connections when a SOCKS server is
     * configured. */
    if (!anonymized_connection && !use_begindir && !options->HTTPProxy &&
        (options->Socks4Proxy || options->Socks5Proxy)) {
      log_warn(LD_DIR, "Cannot connect to a directory server through a "
                       "SOCKS proxy!");
      return;
    }
  
    /* Make sure that the destination addr and port we picked is viable. */
    if (!port || tor_addr_is_null(&addr)) {
      static int logged_backtrace = 0;
      log_warn(LD_DIR,
               "Cannot make an outgoing %sconnection without a remote %sPort.",
               use_begindir ? "begindir " : "",
               use_begindir ? "OR" : "Dir");
      if (!logged_backtrace) {
        log_backtrace(LOG_INFO, LD_BUG, "Address came from");
        logged_backtrace = 1;
      }
      return;
    }
  
    conn = dir_connection_new(tor_addr_family(&addr));
  
    /* set up conn so it's got all the data we need to remember */
    tor_addr_copy(&conn->base_.addr, &addr);
    conn->base_.port = port;
    conn->base_.address = tor_addr_to_str_dup(&addr);
    memcpy(conn->identity_digest, digest, DIGEST_LEN);
  
    conn->base_.purpose = dir_purpose;
    conn->router_purpose = router_purpose;
  
    /* give it an initial state */
    conn->base_.state = DIR_CONN_STATE_CONNECTING;
  
    /* decide whether we can learn our IP address from this conn */
    /* XXXX This is a bad name for this field now. */
    conn->dirconn_direct = !anonymized_connection;
  
    /* copy rendezvous data, if any */
    if (rend_query) {
      /* We can't have both v2 and v3+ identifier. */
      tor_assert_nonfatal(!hs_ident);
      conn->rend_data = rend_data_dup(rend_query);
    }
    if (hs_ident) {
      /* We can't have both v2 and v3+ identifier. */
      tor_assert_nonfatal(!rend_query);
      conn->hs_ident = hs_ident_dir_conn_dup(hs_ident);
    }
  
    if (!anonymized_connection && !use_begindir) {
      /* then we want to connect to dirport directly */
  
      if (options->HTTPProxy) {
        tor_addr_copy(&addr, &options->HTTPProxyAddr);
        port = options->HTTPProxyPort;
      }
  
      // In this case we should not have picked a directory guard.
      if (BUG(guard_state)) {
        entry_guard_cancel(&guard_state);
      }
  
      switch (connection_connect(TO_CONN(conn), conn->base_.address, &addr,
                                 port, &socket_error)) {
        case -1:
          connection_mark_for_close(TO_CONN(conn));
          return;
        case 1:
          /* start flushing conn */
          conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
          /* fall through */
        case 0:
          /* queue the command on the outbuf */
          directory_send_command(conn, 1, request);
          connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT);
          /* writable indicates finish, readable indicates broken link,
             error indicates broken link in windowsland. */
      }
    } else {
      /* We will use a Tor circuit (maybe 1-hop, maybe 3-hop, maybe with
       * begindir, maybe not with begindir) */
  
      entry_connection_t *linked_conn;
  
      /* Anonymized tunneled connections can never share a circuit.
       * One-hop directory connections can share circuits with each other
       * but nothing else. */
      int iso_flags = anonymized_connection ? ISO_STREAM : ISO_SESSIONGRP;
  
      /* If it's an anonymized connection, remember the fact that we
       * wanted it for later: maybe we'll want it again soon. */
      if (anonymized_connection && use_begindir)
        rep_hist_note_used_internal(time(NULL), 0, 1);
      else if (anonymized_connection && !use_begindir)
        rep_hist_note_used_port(time(NULL), conn->base_.port);
  
      // In this case we should not have a directory guard; we'll
      // get a regular guard later when we build the circuit.
      if (BUG(anonymized_connection && guard_state)) {
        entry_guard_cancel(&guard_state);
      }
  
      conn->guard_state = guard_state;
  
      /* make an AP connection
       * populate it and add it at the right state
       * hook up both sides
       */
      linked_conn =
        connection_ap_make_link(TO_CONN(conn),
                                conn->base_.address, conn->base_.port,
                                digest,
                                SESSION_GROUP_DIRCONN, iso_flags,
                                use_begindir, !anonymized_connection);
      if (!linked_conn) {
        log_warn(LD_NET,"Making tunnel to dirserver failed.");
        connection_mark_for_close(TO_CONN(conn));
        return;
      }
  
      if (connection_add(TO_CONN(conn)) < 0) {
        log_warn(LD_NET,"Unable to add connection for link to dirserver.");
        connection_mark_for_close(TO_CONN(conn));
        return;
      }
      conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
      /* queue the command on the outbuf */
      directory_send_command(conn, 0, request);
  
      connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
      connection_start_reading(ENTRY_TO_CONN(linked_conn));
    }
  }
  
  /** Helper for sorting
   *
   * sort strings alphabetically
   *
   * XXXX we have a smartlist_sort_strings() function, right?
   */
  static int
  compare_strs_(const void **a, const void **b)
  {
    const char *s1 = *a, *s2 = *b;
    return strcmp(s1, s2);
  }
  
  #define CONDITIONAL_CONSENSUS_FPR_LEN 3
  #if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN)
  #error "conditional consensus fingerprint length is larger than digest length"
  #endif
  
  /** Return the URL we should use for a consensus download.
   *
   * Use the "conditional consensus downloading" feature described in
   * dir-spec.txt, i.e.
   * GET .../consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>
   *
   * If 'resource' is provided, it is the name of a consensus flavor to request.
   */
  static char *
  directory_get_consensus_url(const char *resource)
  {
    char *url = NULL;
    const char *hyphen, *flavor;
    if (resource==NULL || strcmp(resource, "ns")==0) {
      flavor = ""; /* Request ns consensuses as "", so older servers will work*/
      hyphen = "";
    } else {
      flavor = resource;
      hyphen = "-";
    }
  
    {
      char *authority_id_list;
      smartlist_t *authority_digests = smartlist_new();
  
      SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
                              dir_server_t *, ds) {
          char *hex;
          if (!(ds->type & V3_DIRINFO))
            continue;
  
          hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1);
          base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1,
                        ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN);
          smartlist_add(authority_digests, hex);
      } SMARTLIST_FOREACH_END(ds);
      smartlist_sort(authority_digests, compare_strs_);
      authority_id_list = smartlist_join_strings(authority_digests,
                                                 "+", 0, NULL);
  
      tor_asprintf(&url, "/tor/status-vote/current/consensus%s%s/%s.z",
                   hyphen, flavor, authority_id_list);
  
      SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp));
      smartlist_free(authority_digests);
      tor_free(authority_id_list);
    }
    return url;
  }
  
  /**
   * Copies the ipv6 from source to destination, subject to buffer size limit
   * size. If decorate is true, makes sure the copied address is decorated.
   */
  static void
  copy_ipv6_address(char* destination, const char* source, size_t len,
                    int decorate) {
    tor_assert(destination);
    tor_assert(source);
  
    if (decorate && source[0] != '[') {
      tor_snprintf(destination, len, "[%s]", source);
    } else {
      strlcpy(destination, source, len);
    }
  }
  
  /** Queue an appropriate HTTP command for <b>request</b> on
   * <b>conn</b>-\>outbuf.  If <b>direct</b> is true, we're making a
   * non-anonymized connection to the dirport.
   */
  static void
  directory_send_command(dir_connection_t *conn,
                         const int direct,
                         const directory_request_t *req)
  {
    tor_assert(req);
    const int purpose = req->dir_purpose;
    const char *resource = req->resource;
    const char *payload = req->payload;
    const size_t payload_len = req->payload_len;
    const time_t if_modified_since = req->if_modified_since;
    const int anonymized_connection = dirind_is_anon(req->indirection);
  
    char proxystring[256];
    char hoststring[128];
    /* NEEDS to be the same size hoststring.
     Will be decorated with brackets around it if it is ipv6. */
    char decorated_address[128];
    smartlist_t *headers = smartlist_new();
    char *url;
    char *accept_encoding;
    size_t url_len;
    char request[8192];
    size_t request_len, total_request_len = 0;
    const char *httpcommand = NULL;
  
    tor_assert(conn);
    tor_assert(conn->base_.type == CONN_TYPE_DIR);
  
    tor_free(conn->requested_resource);
    if (resource)
      conn->requested_resource = tor_strdup(resource);
  
    /* decorate the ip address if it is ipv6 */
    if (strchr(conn->base_.address, ':')) {
      copy_ipv6_address(decorated_address, conn->base_.address,
                        sizeof(decorated_address), 1);
    } else {
      strlcpy(decorated_address, conn->base_.address, sizeof(decorated_address));
    }
  
    /* come up with a string for which Host: we want */
    if (conn->base_.port == 80) {
      strlcpy(hoststring, decorated_address, sizeof(hoststring));
    } else {
      tor_snprintf(hoststring, sizeof(hoststring), "%s:%d",
                   decorated_address, conn->base_.port);
    }
  
    /* Format if-modified-since */
    if (if_modified_since) {
      char b[RFC1123_TIME_LEN+1];
      format_rfc1123_time(b, if_modified_since);
      smartlist_add_asprintf(headers, "If-Modified-Since: %s\r\n", b);
    }
  
    /* come up with some proxy lines, if we're using one. */
    if (direct && get_options()->HTTPProxy) {
      char *base64_authenticator=NULL;
      const char *authenticator = get_options()->HTTPProxyAuthenticator;
  
      tor_snprintf(proxystring, sizeof(proxystring),"http://%s", hoststring);
      if (authenticator) {
        base64_authenticator = alloc_http_authenticator(authenticator);
        if (!base64_authenticator)
          log_warn(LD_BUG, "Encoding http authenticator failed");
      }
      if (base64_authenticator) {
        smartlist_add_asprintf(headers,
                     "Proxy-Authorization: Basic %s\r\n",
                     base64_authenticator);
        tor_free(base64_authenticator);
      }
    } else {
      proxystring[0] = 0;
    }
  
    if (! anonymized_connection) {
      /* Add Accept-Encoding. */
      accept_encoding = accept_encoding_header();
      smartlist_add_asprintf(headers, "Accept-Encoding: %s\r\n",
                             accept_encoding);
      tor_free(accept_encoding);
    }
  
    /* Add additional headers, if any */
    {
      config_line_t *h;
      for (h = req->additional_headers; h; h = h->next) {
        smartlist_add_asprintf(headers, "%s%s\r\n", h->key, h->value);
      }
    }
  
    switch (purpose) {
      case DIR_PURPOSE_FETCH_CONSENSUS:
        /* resource is optional.  If present, it's a flavor name */
        tor_assert(!payload);
        httpcommand = "GET";
        url = directory_get_consensus_url(resource);
        log_info(LD_DIR, "Downloading consensus from %s using %s",
                 hoststring, url);
        break;
      case DIR_PURPOSE_FETCH_CERTIFICATE:
        tor_assert(resource);
        tor_assert(!payload);
        httpcommand = "GET";
        tor_asprintf(&url, "/tor/keys/%s", resource);
        break;
      case DIR_PURPOSE_FETCH_STATUS_VOTE:
        tor_assert(resource);
        tor_assert(!payload);
        httpcommand = "GET";
        tor_asprintf(&url, "/tor/status-vote/next/%s.z", resource);
        break;
      case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
        tor_assert(!resource);
        tor_assert(!payload);
        httpcommand = "GET";
        url = tor_strdup("/tor/status-vote/next/consensus-signatures.z");
        break;
      case DIR_PURPOSE_FETCH_SERVERDESC:
        tor_assert(resource);
        httpcommand = "GET";
        tor_asprintf(&url, "/tor/server/%s", resource);
        break;
      case DIR_PURPOSE_FETCH_EXTRAINFO:
        tor_assert(resource);
        httpcommand = "GET";
        tor_asprintf(&url, "/tor/extra/%s", resource);
        break;
      case DIR_PURPOSE_FETCH_MICRODESC:
        tor_assert(resource);
        httpcommand = "GET";
        tor_asprintf(&url, "/tor/micro/%s", resource);
        break;
      case DIR_PURPOSE_UPLOAD_DIR: {
        const char *why = router_get_descriptor_gen_reason();
        tor_assert(!resource);
        tor_assert(payload);
        httpcommand = "POST";
        url = tor_strdup("/tor/");
        if (!why) {
          why = "for no reason at all";
        }
        smartlist_add_asprintf(headers, "X-Desc-Gen-Reason: %s\r\n", why);
        break;
      }
      case DIR_PURPOSE_UPLOAD_VOTE:
        tor_assert(!resource);
        tor_assert(payload);
        httpcommand = "POST";
        url = tor_strdup("/tor/post/vote");
        break;
      case DIR_PURPOSE_UPLOAD_SIGNATURES:
        tor_assert(!resource);
        tor_assert(payload);
        httpcommand = "POST";
        url = tor_strdup("/tor/post/consensus-signature");
        break;
      case DIR_PURPOSE_FETCH_RENDDESC_V2:
        tor_assert(resource);
        tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32);
        tor_assert(!payload);
        httpcommand = "GET";
        tor_asprintf(&url, "/tor/rendezvous2/%s", resource);
        break;
      case DIR_PURPOSE_FETCH_HSDESC:
        tor_assert(resource);
        tor_assert(strlen(resource) <= ED25519_BASE64_LEN);
        tor_assert(!payload);
        httpcommand = "GET";
        tor_asprintf(&url, "/tor/hs/3/%s", resource);
        break;
      case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
        tor_assert(!resource);
        tor_assert(payload);
        httpcommand = "POST";
        url = tor_strdup("/tor/rendezvous2/publish");
        break;
      case DIR_PURPOSE_UPLOAD_HSDESC:
        tor_assert(resource);
        tor_assert(payload);
        httpcommand = "POST";
        tor_asprintf(&url, "/tor/hs/%s/publish", resource);
        break;
      default:
        tor_assert(0);
        return;
    }
  
    /* warn in the non-tunneled case */
    if (direct && (strlen(proxystring) + strlen(url) >= 4096)) {
      log_warn(LD_BUG,
               "Squid does not like URLs longer than 4095 bytes, and this "
               "one is %d bytes long: %s%s",
               (int)(strlen(proxystring) + strlen(url)), proxystring, url);
    }
  
    tor_snprintf(request, sizeof(request), "%s %s", httpcommand, proxystring);
  
    request_len = strlen(request);
    total_request_len += request_len;
    connection_buf_add(request, request_len, TO_CONN(conn));
  
    url_len = strlen(url);
    total_request_len += url_len;
    connection_buf_add(url, url_len, TO_CONN(conn));
    tor_free(url);
  
    if (!strcmp(httpcommand, "POST") || payload) {
      smartlist_add_asprintf(headers, "Content-Length: %lu\r\n",
                   payload ? (unsigned long)payload_len : 0);
    }
  
    {
      char *header = smartlist_join_strings(headers, "", 0, NULL);
      tor_snprintf(request, sizeof(request), " HTTP/1.0\r\nHost: %s\r\n%s\r\n",
                   hoststring, header);
      tor_free(header);
    }
  
    request_len = strlen(request);
    total_request_len += request_len;
    connection_buf_add(request, request_len, TO_CONN(conn));
  
    if (payload) {
      /* then send the payload afterwards too */
      connection_buf_add(payload, payload_len, TO_CONN(conn));
      total_request_len += payload_len;
    }
  
    SMARTLIST_FOREACH(headers, char *, h, tor_free(h));
    smartlist_free(headers);
  
    log_debug(LD_DIR,
              "Sent request to directory server '%s:%d': "
              "(purpose: %d, request size: %"TOR_PRIuSZ", "
              "payload size: %"TOR_PRIuSZ")",
              conn->base_.address, conn->base_.port,
              conn->base_.purpose,
              (total_request_len),
              (payload ? payload_len : 0));
  }
  
  /** Return true iff <b>body</b> doesn't start with a plausible router or
   * network-status or microdescriptor opening.  This is a sign of possible
   * compression. */
  static int
  body_is_plausible(const char *body, size_t len, int purpose)
  {
    int i;
    if (len == 0)
      return 1; /* empty bodies don't need decompression */
    if (len < 32)
      return 0;
    if (purpose == DIR_PURPOSE_FETCH_MICRODESC) {
      return (!strcmpstart(body,"onion-key"));
    }
  
    if (!strcmpstart(body,"router") ||
        !strcmpstart(body,"network-status"))
      return 1;
    for (i=0;i<32;++i) {
      if (!TOR_ISPRINT(body[i]) && !TOR_ISSPACE(body[i]))
        return 0;
    }
  
    return 1;
  }
  
  /** Called when we've just fetched a bunch of router descriptors in
   * <b>body</b>.  The list <b>which</b>, if present, holds digests for
   * descriptors we requested: descriptor digests if <b>descriptor_digests</b>
   * is true, or identity digests otherwise.  Parse the descriptors, validate
   * them, and annotate them as having purpose <b>purpose</b> and as having been
   * downloaded from <b>source</b>.
   *
   * Return the number of routers actually added. */
  static int
  load_downloaded_routers(const char *body, smartlist_t *which,
                          int descriptor_digests,
                          int router_purpose,
                          const char *source)
  {
    char buf[256];
    char time_buf[ISO_TIME_LEN+1];
    int added = 0;
    int general = router_purpose == ROUTER_PURPOSE_GENERAL;
    format_iso_time(time_buf, time(NULL));
    tor_assert(source);
  
    if (tor_snprintf(buf, sizeof(buf),
                     "@downloaded-at %s\n"
                     "@source %s\n"
                     "%s%s%s", time_buf, escaped(source),
                     !general ? "@purpose " : "",
                     !general ? router_purpose_to_string(router_purpose) : "",
                     !general ? "\n" : "")<0)
      return added;
  
    added = router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
                                    descriptor_digests, buf);
    if (added && general)
      control_event_boot_dir(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
                             count_loading_descriptors_progress());
    return added;
  }
  
  static int handle_response_fetch_certificate(dir_connection_t *,
                                               const response_handler_args_t *);
  static int handle_response_fetch_status_vote(dir_connection_t *,
                                               const response_handler_args_t *);
  static int handle_response_fetch_detached_signatures(dir_connection_t *,
                                               const response_handler_args_t *);
  static int handle_response_fetch_desc(dir_connection_t *,
                                               const response_handler_args_t *);
  static int handle_response_upload_dir(dir_connection_t *,
                                        const response_handler_args_t *);
  static int handle_response_upload_vote(dir_connection_t *,
                                         const response_handler_args_t *);
  static int handle_response_upload_signatures(dir_connection_t *,
                                               const response_handler_args_t *);
  static int handle_response_fetch_renddesc_v2(dir_connection_t *,
                                               const response_handler_args_t *);
  static int handle_response_upload_renddesc_v2(dir_connection_t *,
                                                const response_handler_args_t *);
  static int handle_response_upload_hsdesc(dir_connection_t *,
                                           const response_handler_args_t *);
  
  static int
  dir_client_decompress_response_body(char **bodyp, size_t *bodylenp,
                                      dir_connection_t *conn,
                                      compress_method_t compression,
                                      int anonymized_connection)
  {
    int rv = 0;
    const char *body = *bodyp;
    size_t body_len = *bodylenp;
    int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
                         conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
                         conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
  
    int plausible = body_is_plausible(body, body_len, conn->base_.purpose);
  
    if (plausible && compression == NO_METHOD) {
      return 0;
    }
  
    int severity = LOG_DEBUG;
    char *new_body = NULL;
    size_t new_len = 0;
    const char *description1, *description2;
    int want_to_try_both = 0;
    int tried_both = 0;
    compress_method_t guessed = detect_compression_method(body, body_len);
  
    description1 = compression_method_get_human_name(compression);
  
    if (BUG(description1 == NULL))
      description1 = compression_method_get_human_name(UNKNOWN_METHOD);
  
    if (guessed == UNKNOWN_METHOD && !plausible)
      description2 = "confusing binary junk";
    else
      description2 = compression_method_get_human_name(guessed);
  
    /* Tell the user if we don't believe what we're told about compression.*/
    want_to_try_both = (compression == UNKNOWN_METHOD ||
                        guessed != compression);
    if (want_to_try_both) {
      severity = LOG_PROTOCOL_WARN;
    }
  
    tor_log(severity, LD_HTTP,
            "HTTP body from server '%s:%d' was labeled as %s, "
            "%s it seems to be %s.%s",
            conn->base_.address, conn->base_.port, description1,
            guessed != compression?"but":"and",
            description2,
            (compression>0 && guessed>0 && want_to_try_both)?
            "  Trying both.":"");
  
    /* Try declared compression first if we can.
     * tor_compress_supports_method() also returns true for NO_METHOD.
     * Ensure that the server is not sending us data compressed using a
     * compression method that is not allowed for anonymous connections. */
    if (anonymized_connection &&
        ! allowed_anonymous_connection_compression_method(compression)) {
      warn_disallowed_anonymous_compression_method(compression);
      rv = -1;
      goto done;
    }
  
    if (tor_compress_supports_method(compression)) {
      tor_uncompress(&new_body, &new_len, body, body_len, compression,
                     !allow_partial, LOG_PROTOCOL_WARN);
      if (new_body) {
        /* We succeeded with the declared compression method. Great! */
        rv = 0;
        goto done;
      }
    }
  
    /* Okay, if that didn't work, and we think that it was compressed
     * differently, try that. */
    if (anonymized_connection &&
        ! allowed_anonymous_connection_compression_method(guessed)) {
      warn_disallowed_anonymous_compression_method(guessed);
      rv = -1;
      goto done;
    }
  
    if (tor_compress_supports_method(guessed) &&
        compression != guessed) {
      tor_uncompress(&new_body, &new_len, body, body_len, guessed,
                     !allow_partial, LOG_INFO);
      tried_both = 1;
    }
    /* If we're pretty sure that we have a compressed directory, and
     * we didn't manage to uncompress it, then warn and bail. */
    if (!plausible && !new_body) {
      log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
             "Unable to decompress HTTP body (tried %s%s%s, server '%s:%d').",
             description1,
             tried_both?" and ":"",
             tried_both?description2:"",
             conn->base_.address, conn->base_.port);
      rv = -1;
      goto done;
    }
  
   done:
    if (new_body) {
      if (rv == 0) {
        /* success! */
        tor_free(*bodyp);
        *bodyp = new_body;
        *bodylenp = new_len;
      } else {
        tor_free(new_body);
      }
    }
  
    return rv;
  }
  
  /** We are a client, and we've finished reading the server's
   * response. Parse it and act appropriately.
   *
   * If we're still happy with using this directory server in the future, return
   * 0. Otherwise return -1; and the caller should consider trying the request
   * again.
   *
   * The caller will take care of marking the connection for close.
   */
  static int
  connection_dir_client_reached_eof(dir_connection_t *conn)
  {
    char *body = NULL;
    char *headers = NULL;
    char *reason = NULL;
    size_t body_len = 0;
    int status_code;
    time_t date_header = 0;
    long apparent_skew;
    compress_method_t compression;
    int skewed = 0;
    int rv;
    int allow_partial = (conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
                         conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO ||
                         conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
    size_t received_bytes;
    const int anonymized_connection =
      purpose_needs_anonymity(conn->base_.purpose,
                              conn->router_purpose,
                              conn->requested_resource);
  
    received_bytes = connection_get_inbuf_len(TO_CONN(conn));
  
    switch (connection_fetch_from_buf_http(TO_CONN(conn),
                                &headers, MAX_HEADERS_SIZE,
                                &body, &body_len, MAX_DIR_DL_SIZE,
                                allow_partial)) {
      case -1: /* overflow */
        log_warn(LD_PROTOCOL,
                 "'fetch' response too large (server '%s:%d'). Closing.",
                 conn->base_.address, conn->base_.port);
        return -1;
      case 0:
        log_info(LD_HTTP,
                 "'fetch' response not all here, but we're at eof. Closing.");
        return -1;
      /* case 1, fall through */
    }
  
    if (parse_http_response(headers, &status_code, &date_header,
                            &compression, &reason) < 0) {
      log_warn(LD_HTTP,"Unparseable headers (server '%s:%d'). Closing.",
               conn->base_.address, conn->base_.port);
  
      rv = -1;
      goto done;
    }
    if (!reason) reason = tor_strdup("[no reason given]");
  
    tor_log(LOG_DEBUG, LD_DIR,
              "Received response from directory server '%s:%d': %d %s "
              "(purpose: %d, response size: %"TOR_PRIuSZ
  #ifdef MEASUREMENTS_21206
              ", data cells received: %d, data cells sent: %d"
  #endif
              ", compression: %d)",
              conn->base_.address, conn->base_.port, status_code,
              escaped(reason), conn->base_.purpose,
              (received_bytes),
  #ifdef MEASUREMENTS_21206
              conn->data_cells_received, conn->data_cells_sent,
  #endif
              compression);
  
    if (conn->guard_state) {
      /* we count the connection as successful once we can read from it.  We do
       * not, however, delay use of the circuit here, since it's just for a
       * one-hop directory request. */
      /* XXXXprop271 note that this will not do the right thing for other
       * waiting circuits that would be triggered by this circuit becoming
       * complete/usable. But that's ok, I think.
       */
      entry_guard_succeeded(&conn->guard_state);
      circuit_guard_state_free(conn->guard_state);
      conn->guard_state = NULL;
    }
  
    /* now check if it's got any hints for us about our IP address. */
    if (conn->dirconn_direct) {
      char *guess = http_get_header(headers, X_ADDRESS_HEADER);
      if (guess) {
        router_new_address_suggestion(guess, conn);
        tor_free(guess);
      }
    }
  
    if (date_header > 0) {
      /* The date header was written very soon after we sent our request,
       * so compute the skew as the difference between sending the request
       * and the date header.  (We used to check now-date_header, but that's
       * inaccurate if we spend a lot of time downloading.)
       */
      apparent_skew = conn->base_.timestamp_last_write_allowed - date_header;
      if (labs(apparent_skew)>ALLOW_DIRECTORY_TIME_SKEW) {
        int trusted = router_digest_is_trusted_dir(conn->identity_digest);
        clock_skew_warning(TO_CONN(conn), apparent_skew, trusted, LD_HTTP,
                           "directory", "DIRSERV");
        skewed = 1; /* don't check the recommended-versions line */
      } else {
        log_debug(LD_HTTP, "Time on received directory is within tolerance; "
                  "we are %ld seconds skewed.  (That's okay.)", apparent_skew);
      }
    }
    (void) skewed; /* skewed isn't used yet. */
  
    if (status_code == 503) {
      routerstatus_t *rs;
      dir_server_t *ds;
      const char *id_digest = conn->identity_digest;
      log_info(LD_DIR,"Received http status code %d (%s) from server "
               "'%s:%d'. I'll try again soon.",
               status_code, escaped(reason), conn->base_.address,
               conn->base_.port);
      time_t now = approx_time();
      if ((rs = router_get_mutable_consensus_status_by_id(id_digest)))
        rs->last_dir_503_at = now;
      if ((ds = router_get_fallback_dirserver_by_digest(id_digest)))
        ds->fake_status.last_dir_503_at = now;
  
      rv = -1;
      goto done;
    }
  
    if (dir_client_decompress_response_body(&body, &body_len,
                               conn, compression, anonymized_connection) < 0) {
      rv = -1;
      goto done;
    }
  
    response_handler_args_t args;
    memset(&args, 0, sizeof(args));
    args.status_code = status_code;
    args.reason = reason;
    args.body = body;
    args.body_len = body_len;
    args.headers = headers;
  
    switch (conn->base_.purpose) {
      case DIR_PURPOSE_FETCH_CONSENSUS:
        rv = handle_response_fetch_consensus(conn, &args);
        break;
      case DIR_PURPOSE_FETCH_CERTIFICATE:
        rv = handle_response_fetch_certificate(conn, &args);
        break;
      case DIR_PURPOSE_FETCH_STATUS_VOTE:
        rv = handle_response_fetch_status_vote(conn, &args);
        break;
      case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
        rv = handle_response_fetch_detached_signatures(conn, &args);
        break;
      case DIR_PURPOSE_FETCH_SERVERDESC:
      case DIR_PURPOSE_FETCH_EXTRAINFO:
        rv = handle_response_fetch_desc(conn, &args);
        break;
      case DIR_PURPOSE_FETCH_MICRODESC:
        rv = handle_response_fetch_microdesc(conn, &args);
        break;
      case DIR_PURPOSE_FETCH_RENDDESC_V2:
        rv = handle_response_fetch_renddesc_v2(conn, &args);
        break;
      case DIR_PURPOSE_UPLOAD_DIR:
        rv = handle_response_upload_dir(conn, &args);
        break;
      case DIR_PURPOSE_UPLOAD_SIGNATURES:
        rv = handle_response_upload_signatures(conn, &args);
        break;
      case DIR_PURPOSE_UPLOAD_VOTE:
        rv = handle_response_upload_vote(conn, &args);
        break;
      case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
        rv = handle_response_upload_renddesc_v2(conn, &args);
        break;
      case DIR_PURPOSE_UPLOAD_HSDESC:
        rv = handle_response_upload_hsdesc(conn, &args);
        break;
      case DIR_PURPOSE_FETCH_HSDESC:
        rv = handle_response_fetch_hsdesc_v3(conn, &args);
        break;
      default:
        tor_assert_nonfatal_unreached();
        rv = -1;
        break;
    }
  
   done:
    tor_free(body);
    tor_free(headers);
    tor_free(reason);
    return rv;
  }
  
  /**
   * Handler function: processes a response to a request for a networkstatus
   * consensus document by checking the consensus, storing it, and marking
   * router requests as reachable.
   **/
  STATIC int
  handle_response_fetch_consensus(dir_connection_t *conn,
                                  const response_handler_args_t *args)
  {
    tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CONSENSUS);
    const int status_code = args->status_code;
    const char *body = args->body;
    const size_t body_len = args->body_len;
    const char *reason = args->reason;
    const time_t now = approx_time();
  
    const char *consensus;
    char *new_consensus = NULL;
    const char *sourcename;
  
    int r;
    const char *flavname = conn->requested_resource;
    if (status_code != 200) {
      int severity = (status_code == 304) ? LOG_INFO : LOG_WARN;
      tor_log(severity, LD_DIR,
              "Received http status code %d (%s) from server "
              "'%s:%d' while fetching consensus directory.",
              status_code, escaped(reason), conn->base_.address,
              conn->base_.port);
      networkstatus_consensus_download_failed(status_code, flavname);
      return -1;
    }
  
    if (looks_like_a_consensus_diff(body, body_len)) {
      /* First find our previous consensus. Maybe it's in ram, maybe not. */
      cached_dir_t *cd = dirserv_get_consensus(flavname);
      const char *consensus_body = NULL;
      size_t consensus_body_len;
      tor_mmap_t *mapped_consensus = NULL;
      if (cd) {
        consensus_body = cd->dir;
        consensus_body_len = cd->dir_len;
      } else {
        mapped_consensus = networkstatus_map_cached_consensus(flavname);
        if (mapped_consensus) {
          consensus_body = mapped_consensus->data;
          consensus_body_len = mapped_consensus->size;
        }
      }
      if (!consensus_body) {
        log_warn(LD_DIR, "Received a consensus diff, but we can't find "
                 "any %s-flavored consensus in our current cache.",flavname);
        tor_munmap_file(mapped_consensus);
        networkstatus_consensus_download_failed(0, flavname);
        // XXXX if this happens too much, see below
        return -1;
      }
  
      new_consensus = consensus_diff_apply(consensus_body, consensus_body_len,
                                           body, body_len);
      tor_munmap_file(mapped_consensus);
      if (new_consensus == NULL) {
        log_warn(LD_DIR, "Could not apply consensus diff received from server "
                 "'%s:%d'", conn->base_.address, conn->base_.port);
        // XXXX If this happens too many times, we should maybe not use
        // XXXX this directory for diffs any more?
        networkstatus_consensus_download_failed(0, flavname);
        return -1;
      }
      log_info(LD_DIR, "Applied consensus diff (size %d) from server "
               "'%s:%d', resulting in a new consensus document (size %d).",
               (int)body_len, conn->base_.address, conn->base_.port,
               (int)strlen(new_consensus));
      consensus = new_consensus;
      sourcename = "generated based on a diff";
    } else {
      log_info(LD_DIR,"Received consensus directory (body size %d) from server "
               "'%s:%d'", (int)body_len, conn->base_.address, conn->base_.port);
      consensus = body;
      sourcename = "downloaded";
    }
  
    if ((r=networkstatus_set_current_consensus(consensus,
                                               strlen(consensus),
                                               flavname, 0,
                                               conn->identity_digest))<0) {
      log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
             "Unable to load %s consensus directory %s from "
             "server '%s:%d'. I'll try again soon.",
             flavname, sourcename, conn->base_.address, conn->base_.port);
      networkstatus_consensus_download_failed(0, flavname);
      tor_free(new_consensus);
      return -1;
    }
  
    /* If we launched other fetches for this consensus, cancel them. */
    connection_dir_close_consensus_fetches(conn, flavname);
  
    /* update the list of routers and directory guards */
    routers_update_all_from_networkstatus(now, 3);
    update_microdescs_from_networkstatus(now);
    directory_info_has_arrived(now, 0, 0);
  
    if (authdir_mode_v3(get_options())) {
      sr_act_post_consensus(
                       networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
    }
    log_info(LD_DIR, "Successfully loaded consensus.");
  
    tor_free(new_consensus);
    return 0;
  }
  
  /**
   * Handler function: processes a response to a request for one or more
   * authority certificates
   **/
  static int
  handle_response_fetch_certificate(dir_connection_t *conn,
                                    const response_handler_args_t *args)
  {
    tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_CERTIFICATE);
    const int status_code = args->status_code;
    const char *reason = args->reason;
    const char *body = args->body;
    const size_t body_len = args->body_len;
  
    if (status_code != 200) {
      log_warn(LD_DIR,
               "Received http status code %d (%s) from server "
               "'%s:%d' while fetching \"/tor/keys/%s\".",
               status_code, escaped(reason), conn->base_.address,
               conn->base_.port, conn->requested_resource);
      connection_dir_download_cert_failed(conn, status_code);
      return -1;
    }
    log_info(LD_DIR,"Received authority certificates (body size %d) from "
             "server '%s:%d'",
             (int)body_len, conn->base_.address, conn->base_.port);
  
    /*
     * Tell trusted_dirs_load_certs_from_string() whether it was by fp
     * or fp-sk pair.
     */
    int src_code = -1;
    if (!strcmpstart(conn->requested_resource, "fp/")) {
      src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST;
    } else if (!strcmpstart(conn->requested_resource, "fp-sk/")) {
      src_code = TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST;
    }
  
    if (src_code != -1) {
      if (trusted_dirs_load_certs_from_string(body, src_code, 1,
                                              conn->identity_digest)<0) {
        log_warn(LD_DIR, "Unable to parse fetched certificates");
        /* if we fetched more than one and only some failed, the successful
         * ones got flushed to disk so it's safe to call this on them */
        connection_dir_download_cert_failed(conn, status_code);
      } else {
        time_t now = approx_time();
        directory_info_has_arrived(now, 0, 0);
        log_info(LD_DIR, "Successfully loaded certificates from fetch.");
      }
    } else {
      log_warn(LD_DIR,
               "Couldn't figure out what to do with fetched certificates for "
               "unknown resource %s",
               conn->requested_resource);
      connection_dir_download_cert_failed(conn, status_code);
    }
    return 0;
  }
  
  /**
   * Handler function: processes a response to a request for an authority's
   * current networkstatus vote.
   **/
  static int
  handle_response_fetch_status_vote(dir_connection_t *conn,
                                    const response_handler_args_t *args)
  {
    tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_STATUS_VOTE);
    const int status_code = args->status_code;
    const char *reason = args->reason;
    const char *body = args->body;
    const size_t body_len = args->body_len;
  
    const char *msg;
    int st;
    log_info(LD_DIR,"Got votes (body size %d) from server %s:%d",
             (int)body_len, conn->base_.address, conn->base_.port);
    if (status_code != 200) {
      log_warn(LD_DIR,
               "Received http status code %d (%s) from server "
               "'%s:%d' while fetching \"/tor/status-vote/next/%s.z\".",
               status_code, escaped(reason), conn->base_.address,
               conn->base_.port, conn->requested_resource);
      return -1;
    }
    dirvote_add_vote(body, &msg, &st);
    if (st > 299) {
      log_warn(LD_DIR, "Error adding retrieved vote: %s", msg);
    } else {
      log_info(LD_DIR, "Added vote(s) successfully [msg: %s]", msg);
    }
  
    return 0;
  }
  
  /**
   * Handler function: processes a response to a request for the signatures
   * that an authority knows about on a given consensus.
   **/
  static int
  handle_response_fetch_detached_signatures(dir_connection_t *conn,
                                            const response_handler_args_t *args)
  {
    tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES);
    const int status_code = args->status_code;
    const char *reason = args->reason;
    const char *body = args->body;
    const size_t body_len = args->body_len;
  
    const char *msg = NULL;
    log_info(LD_DIR,"Got detached signatures (body size %d) from server %s:%d",
             (int)body_len, conn->base_.address, conn->base_.port);
    if (status_code != 200) {
      log_warn(LD_DIR,
          "Received http status code %d (%s) from server '%s:%d' while fetching "
          "\"/tor/status-vote/next/consensus-signatures.z\".",
          status_code, escaped(reason), conn->base_.address,
          conn->base_.port);
      return -1;
    }
    if (dirvote_add_signatures(body, conn->base_.address, &msg)<0) {
      log_warn(LD_DIR, "Problem adding detached signatures from %s:%d: %s",
               conn->base_.address, conn->base_.port, msg?msg:"???");
    }
  
    return 0;
  }
  
  /**
   * Handler function: processes a response to a request for a group of server
   * descriptors or an extrainfo documents.
   **/
  static int
  handle_response_fetch_desc(dir_connection_t *conn,
                             const response_handler_args_t *args)
  {
    tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_SERVERDESC ||
               conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO);
    const int status_code = args->status_code;
    const char *reason = args->reason;
    const char *body = args->body;
    const size_t body_len = args->body_len;
  
    int was_ei = conn->base_.purpose == DIR_PURPOSE_FETCH_EXTRAINFO;
    smartlist_t *which = NULL;
    int n_asked_for = 0;
    int descriptor_digests = conn->requested_resource &&
      !strcmpstart(conn->requested_resource,"d/");
    log_info(LD_DIR,"Received %s (body size %d) from server '%s:%d'",
             was_ei ? "extra server info" : "server info",
             (int)body_len, conn->base_.address, conn->base_.port);
    if (conn->requested_resource &&
        (!strcmpstart(conn->requested_resource,"d/") ||
         !strcmpstart(conn->requested_resource,"fp/"))) {
      which = smartlist_new();
      dir_split_resource_into_fingerprints(conn->requested_resource +
                                           (descriptor_digests ? 2 : 3),
                                           which, NULL, 0);
      n_asked_for = smartlist_len(which);
    }
    if (status_code != 200) {
      int dir_okay = status_code == 404 ||
        (status_code == 400 && !strcmp(reason, "Servers unavailable."));
      /* 404 means that it didn't have them; no big deal.
       * Older (pre-0.1.1.8) servers said 400 Servers unavailable instead. */
      log_fn(dir_okay ? LOG_INFO : LOG_WARN, LD_DIR,
             "Received http status code %d (%s) from server '%s:%d' "
             "while fetching \"/tor/server/%s\". I'll try again soon.",
             status_code, escaped(reason), conn->base_.address,
             conn->base_.port, conn->requested_resource);
      if (!which) {
        connection_dir_download_routerdesc_failed(conn);
      } else {
        dir_routerdesc_download_failed(which, status_code,
                                       conn->router_purpose,
                                       was_ei, descriptor_digests);
        SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
        smartlist_free(which);
      }
      return dir_okay ? 0 : -1;
    }
    /* Learn the routers, assuming we requested by fingerprint or "all"
     * or "authority".
     *
     * We use "authority" to fetch our own descriptor for
     * testing, and to fetch bridge descriptors for bootstrapping. Ignore
     * the output of "authority" requests unless we are using bridges,
     * since otherwise they'll be the response from reachability tests,
     * and we don't really want to add that to our routerlist. */
    if (which || (conn->requested_resource &&
                  (!strcmpstart(conn->requested_resource, "all") ||
                   (!strcmpstart(conn->requested_resource, "authority") &&
                    get_options()->UseBridges)))) {
      /* as we learn from them, we remove them from 'which' */
      if (was_ei) {
        router_load_extrainfo_from_string(body, NULL, SAVED_NOWHERE, which,
                                          descriptor_digests);
      } else {
        //router_load_routers_from_string(body, NULL, SAVED_NOWHERE, which,
        //                       descriptor_digests, conn->router_purpose);
        if (load_downloaded_routers(body, which, descriptor_digests,
                                    conn->router_purpose,
                                    conn->base_.address)) {
          time_t now = approx_time();
          directory_info_has_arrived(now, 0, 1);
        }
      }
    }
    if (which) { /* mark remaining ones as failed */
      log_info(LD_DIR, "Received %d/%d %s requested from %s:%d",
               n_asked_for-smartlist_len(which), n_asked_for,
               was_ei ? "extra-info documents" : "router descriptors",
               conn->base_.address, (int)conn->base_.port);
      if (smartlist_len(which)) {
        dir_routerdesc_download_failed(which, status_code,
                                       conn->router_purpose,
                                       was_ei, descriptor_digests);
      }
      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
      smartlist_free(which);
    }
    if (directory_conn_is_self_reachability_test(conn))
      router_dirport_found_reachable();
  
    return 0;
  }
  
  /**
   * Handler function: processes a response to a request for a group of
   * microdescriptors
   **/
  STATIC int
  handle_response_fetch_microdesc(dir_connection_t *conn,
                                  const response_handler_args_t *args)
  {
    tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_MICRODESC);
    const int status_code = args->status_code;
    const char *reason = args->reason;
    const char *body = args->body;
    const size_t body_len = args->body_len;
  
    smartlist_t *which = NULL;
    log_info(LD_DIR,"Received answer to microdescriptor request (status %d, "
             "body size %d) from server '%s:%d'",
             status_code, (int)body_len, conn->base_.address,
             conn->base_.port);
    tor_assert(conn->requested_resource &&
               !strcmpstart(conn->requested_resource, "d/"));
    tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN));
    which = smartlist_new();
    dir_split_resource_into_fingerprints(conn->requested_resource+2,
                                         which, NULL,
                                         DSR_DIGEST256|DSR_BASE64);
    if (status_code != 200) {
      log_info(LD_DIR, "Received status code %d (%s) from server "
               "'%s:%d' while fetching \"/tor/micro/%s\".  I'll try again "
               "soon.",
               status_code, escaped(reason), conn->base_.address,
               (int)conn->base_.port, conn->requested_resource);
      dir_microdesc_download_failed(which, status_code, conn->identity_digest);
      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
      smartlist_free(which);
      return 0;
    } else {
      smartlist_t *mds;
      time_t now = approx_time();
      mds = microdescs_add_to_cache(get_microdesc_cache(),
                                    body, body+body_len, SAVED_NOWHERE, 0,
                                    now, which);
      if (smartlist_len(which)) {
        /* Mark remaining ones as failed. */
        dir_microdesc_download_failed(which, status_code, conn->identity_digest);
      }
      if (mds && smartlist_len(mds)) {
        control_event_boot_dir(BOOTSTRAP_STATUS_LOADING_DESCRIPTORS,
                               count_loading_descriptors_progress());
        directory_info_has_arrived(now, 0, 1);
      }
      SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
      smartlist_free(which);
      smartlist_free(mds);
    }
  
    return 0;
  }
  
  /**
   * Handler function: processes a response to a POST request to upload our
   * router descriptor.
   **/
  static int
  handle_response_upload_dir(dir_connection_t *conn,
                             const response_handler_args_t *args)
  {
    tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_DIR);
    const int status_code = args->status_code;
    const char *reason = args->reason;
    const char *headers = args->headers;
  
    switch (status_code) {
    case 200: {
      dir_server_t *ds =
        router_get_trusteddirserver_by_digest(conn->identity_digest);
      char *rejected_hdr = http_get_header(headers,
                                           "X-Descriptor-Not-New: ");
      if (rejected_hdr) {
        if (!strcmp(rejected_hdr, "Yes")) {
          log_info(LD_GENERAL,
                   "Authority '%s' declined our descriptor (not new)",
                   ds->nickname);
          /* XXXX use this information; be sure to upload next one
           * sooner. -NM */
          /* XXXX++ On further thought, the task above implies that we're
           * basing our regenerate-descriptor time on when we uploaded the
           * last descriptor, not on the published time of the last
           * descriptor.  If those are different, that's a bad thing to
           * do. -NM */
        }
        tor_free(rejected_hdr);
      }
      log_info(LD_GENERAL,"eof (status 200) after uploading server "
               "descriptor: finished.");
      control_event_server_status(
                     LOG_NOTICE, "ACCEPTED_SERVER_DESCRIPTOR DIRAUTH=%s:%d",
                     conn->base_.address, conn->base_.port);
  
      ds->has_accepted_serverdesc = 1;
      if (directories_have_accepted_server_descriptor())
        control_event_server_status(LOG_NOTICE, "GOOD_SERVER_DESCRIPTOR");
    }
      break;
    case 400:
      log_warn(LD_GENERAL,"http status 400 (%s) response from "
               "dirserver '%s:%d'. Please correct.",
               escaped(reason), conn->base_.address, conn->base_.port);
      control_event_server_status(LOG_WARN,
                      "BAD_SERVER_DESCRIPTOR DIRAUTH=%s:%d REASON=\"%s\"",
                      conn->base_.address, conn->base_.port, escaped(reason));
      break;
    default:
      log_warn(LD_GENERAL,
               "HTTP status %d (%s) was unexpected while uploading "
               "descriptor to server '%s:%d'. Possibly the server is "
               "misconfigured?",
               status_code, escaped(reason), conn->base_.address,
               conn->base_.port);
      break;
    }
    /* return 0 in all cases, since we don't want to mark any
     * dirservers down just because they don't like us. */
  
    return 0;
  }
  
  /**
   * Handler function: processes a response to POST request to upload our
   * own networkstatus vote.
   **/
  static int
  handle_response_upload_vote(dir_connection_t *conn,
                              const response_handler_args_t *args)
  {
    tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_VOTE);
    const int status_code = args->status_code;
    const char *reason = args->reason;
  
    switch (status_code) {
    case 200: {
      log_notice(LD_DIR,"Uploaded a vote to dirserver %s:%d",
                 conn->base_.address, conn->base_.port);
    }
      break;
    case 400:
      log_warn(LD_DIR,"http status 400 (%s) response after uploading "
               "vote to dirserver '%s:%d'. Please correct.",
               escaped(reason), conn->base_.address, conn->base_.port);
      break;
    default:
      log_warn(LD_GENERAL,
               "HTTP status %d (%s) was unexpected while uploading "
               "vote to server '%s:%d'.",
               status_code, escaped(reason), conn->base_.address,
               conn->base_.port);
      break;
    }
    /* return 0 in all cases, since we don't want to mark any
     * dirservers down just because they don't like us. */
    return 0;
  }
  
  /**
   * Handler function: processes a response to POST request to upload our
   * view of the signatures on the current consensus.
   **/
  static int
  handle_response_upload_signatures(dir_connection_t *conn,
                                    const response_handler_args_t *args)
  {
    tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_SIGNATURES);
    const int status_code = args->status_code;
    const char *reason = args->reason;
  
    switch (status_code) {
    case 200: {
      log_notice(LD_DIR,"Uploaded signature(s) to dirserver %s:%d",
                 conn->base_.address, conn->base_.port);
    }
      break;
    case 400:
      log_warn(LD_DIR,"http status 400 (%s) response after uploading "
               "signatures to dirserver '%s:%d'. Please correct.",
               escaped(reason), conn->base_.address, conn->base_.port);
      break;
    default:
      log_warn(LD_GENERAL,
               "HTTP status %d (%s) was unexpected while uploading "
               "signatures to server '%s:%d'.",
               status_code, escaped(reason), conn->base_.address,
               conn->base_.port);
      break;
    }
    /* return 0 in all cases, since we don't want to mark any
     * dirservers down just because they don't like us. */
  
    return 0;
  }
  
  /**
   * Handler function: processes a response to a request for a v3 hidden service
   * descriptor.
   **/
  STATIC int
  handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
                                  const response_handler_args_t *args)
  {
    const int status_code = args->status_code;
    const char *reason = args->reason;
    const char *body = args->body;
    const size_t body_len = args->body_len;
  
    tor_assert(conn->hs_ident);
  
    log_info(LD_REND,"Received v3 hsdesc (body size %d, status %d (%s))",
             (int)body_len, status_code, escaped(reason));
  
    switch (status_code) {
    case 200:
      /* We got something: Try storing it in the cache. */
      if (hs_cache_store_as_client(body, &conn->hs_ident->identity_pk) < 0) {
        log_info(LD_REND, "Failed to store hidden service descriptor");
        /* Fire control port FAILED event. */
        hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
                                     "BAD_DESC");
        hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
                                      NULL);
      } else {
        log_info(LD_REND, "Stored hidden service descriptor successfully.");
        TO_CONN(conn)->purpose = DIR_PURPOSE_HAS_FETCHED_HSDESC;
        hs_client_desc_has_arrived(conn->hs_ident);
        /* Fire control port RECEIVED event. */
        hs_control_desc_event_received(conn->hs_ident, conn->identity_digest);
        hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
                                      body);
      }
      break;
    case 404:
      /* Not there. We'll retry when connection_about_to_close_connection()
       * tries to clean this conn up. */
      log_info(LD_REND, "Fetching hidden service v3 descriptor not found: "
                        "Retrying at another directory.");
      /* Fire control port FAILED event. */
      hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
                                   "NOT_FOUND");
      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
                                    NULL);
      break;
    case 400:
      log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
                        "http status 400 (%s). Dirserver didn't like our "
                        "query? Retrying at another directory.",
               escaped(reason));
      /* Fire control port FAILED event. */
      hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
                                   "QUERY_REJECTED");
      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
                                    NULL);
      break;
    default:
      log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
               "http status %d (%s) response unexpected from HSDir server "
               "'%s:%d'. Retrying at another directory.",
               status_code, escaped(reason), TO_CONN(conn)->address,
               TO_CONN(conn)->port);
      /* Fire control port FAILED event. */
      hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
                                   "UNEXPECTED");
      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
                                    NULL);
      break;
    }
  
    return 0;
  }
  
  /**
   * Handler function: processes a response to a request for a v2 hidden service
   * descriptor.
   **/
  static int
  handle_response_fetch_renddesc_v2(dir_connection_t *conn,
                                    const response_handler_args_t *args)
  {
    tor_assert(conn->base_.purpose == DIR_PURPOSE_FETCH_RENDDESC_V2);
    const int status_code = args->status_code;
    const char *reason = args->reason;
    const char *body = args->body;
    const size_t body_len = args->body_len;
  
  #define SEND_HS_DESC_FAILED_EVENT(reason)                               \
    (control_event_hsv2_descriptor_failed(conn->rend_data,                \
                                          conn->identity_digest,          \
                                          reason))
  #define SEND_HS_DESC_FAILED_CONTENT()                                   \
    (control_event_hs_descriptor_content(                                 \
                                  rend_data_get_address(conn->rend_data), \
                                  conn->requested_resource,               \
                                  conn->identity_digest,                  \
                                  NULL))
  
    tor_assert(conn->rend_data);
    log_info(LD_REND,"Received rendezvous descriptor (body size %d, status %d "
             "(%s))",
             (int)body_len, status_code, escaped(reason));
    switch (status_code) {
    case 200:
      {
        rend_cache_entry_t *entry = NULL;
  
        if (rend_cache_store_v2_desc_as_client(body,
                                               conn->requested_resource,
                                               conn->rend_data, &entry) < 0) {
          log_warn(LD_REND,"Fetching v2 rendezvous descriptor failed. "
                   "Retrying at another directory.");
          /* We'll retry when connection_about_to_close_connection()
           * cleans this dir conn up. */
          SEND_HS_DESC_FAILED_EVENT("BAD_DESC");
          SEND_HS_DESC_FAILED_CONTENT();
        } else {
          char service_id[REND_SERVICE_ID_LEN_BASE32 + 1];
          /* Should never be NULL here if we found the descriptor. */
          tor_assert(entry);
          rend_get_service_id(entry->parsed->pk, service_id);
  
          /* success. notify pending connections about this. */
          log_info(LD_REND, "Successfully fetched v2 rendezvous "
                   "descriptor.");
          control_event_hsv2_descriptor_received(service_id,
                                                 conn->rend_data,
                                                 conn->identity_digest);
          control_event_hs_descriptor_content(service_id,
                                              conn->requested_resource,
                                              conn->identity_digest,
                                              body);
          conn->base_.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2;
          rend_client_desc_trynow(service_id);
          memwipe(service_id, 0, sizeof(service_id));
        }
        break;
      }
    case 404:
      /* Not there. We'll retry when
       * connection_about_to_close_connection() cleans this conn up. */
      log_info(LD_REND,"Fetching v2 rendezvous descriptor failed: "
               "Retrying at another directory.");
      SEND_HS_DESC_FAILED_EVENT("NOT_FOUND");
      SEND_HS_DESC_FAILED_CONTENT();
      break;
    case 400:
      log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
               "http status 400 (%s). Dirserver didn't like our "
               "v2 rendezvous query? Retrying at another directory.",
               escaped(reason));
      SEND_HS_DESC_FAILED_EVENT("QUERY_REJECTED");
      SEND_HS_DESC_FAILED_CONTENT();
      break;
    default:
      log_warn(LD_REND, "Fetching v2 rendezvous descriptor failed: "
               "http status %d (%s) response unexpected while "
               "fetching v2 hidden service descriptor (server '%s:%d'). "
               "Retrying at another directory.",
               status_code, escaped(reason), conn->base_.address,
               conn->base_.port);
      SEND_HS_DESC_FAILED_EVENT("UNEXPECTED");
      SEND_HS_DESC_FAILED_CONTENT();
      break;
    }
  
    return 0;
  }
  
  /**
   * Handler function: processes a response to a POST request to upload a v2
   * hidden service descriptor.
   **/
  static int
  handle_response_upload_renddesc_v2(dir_connection_t *conn,
                                     const response_handler_args_t *args)
  {
    tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_RENDDESC_V2);
    const int status_code = args->status_code;
    const char *reason = args->reason;
  
  #define SEND_HS_DESC_UPLOAD_FAILED_EVENT(reason)                        \
      (control_event_hs_descriptor_upload_failed(                         \
                                  conn->identity_digest,                  \
                                  rend_data_get_address(conn->rend_data), \
                                  reason))
  
    log_info(LD_REND,"Uploaded rendezvous descriptor (status %d "
             "(%s))",
             status_code, escaped(reason));
    /* Without the rend data, we'll have a problem identifying what has been
     * uploaded for which service. */
    tor_assert(conn->rend_data);
    switch (status_code) {
    case 200:
      log_info(LD_REND,
               "Uploading rendezvous descriptor: finished with status "
               "200 (%s)", escaped(reason));
      control_event_hs_descriptor_uploaded(conn->identity_digest,
                                     rend_data_get_address(conn->rend_data));
      rend_service_desc_has_uploaded(conn->rend_data);
      break;
    case 400:
      log_warn(LD_REND,"http status 400 (%s) response from dirserver "
               "'%s:%d'. Malformed rendezvous descriptor?",
               escaped(reason), conn->base_.address, conn->base_.port);
      SEND_HS_DESC_UPLOAD_FAILED_EVENT("UPLOAD_REJECTED");
      break;
    default:
      log_warn(LD_REND,"http status %d (%s) response unexpected (server "
               "'%s:%d').",
               status_code, escaped(reason), conn->base_.address,
               conn->base_.port);
      SEND_HS_DESC_UPLOAD_FAILED_EVENT("UNEXPECTED");
      break;
    }
  
    return 0;
  }
  
  /**
   * Handler function: processes a response to a POST request to upload an
   * hidden service descriptor.
   **/
  static int
  handle_response_upload_hsdesc(dir_connection_t *conn,
                                const response_handler_args_t *args)
  {
    const int status_code = args->status_code;
    const char *reason = args->reason;
  
    tor_assert(conn);
    tor_assert(conn->base_.purpose == DIR_PURPOSE_UPLOAD_HSDESC);
  
    log_info(LD_REND, "Uploaded hidden service descriptor (status %d "
                      "(%s))",
             status_code, escaped(reason));
    /* For this directory response, it MUST have an hidden service identifier on
     * this connection. */
    tor_assert(conn->hs_ident);
    switch (status_code) {
    case 200:
      log_info(LD_REND, "Uploading hidden service descriptor: "
                        "finished with status 200 (%s)", escaped(reason));
      hs_control_desc_event_uploaded(conn->hs_ident, conn->identity_digest);
      break;
    case 400:
      log_fn(LOG_PROTOCOL_WARN, LD_REND,
             "Uploading hidden service descriptor: http "
             "status 400 (%s) response from dirserver "
             "'%s:%d'. Malformed hidden service descriptor?",
             escaped(reason), conn->base_.address, conn->base_.port);
      hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
                                   "UPLOAD_REJECTED");
      break;
    default:
      log_warn(LD_REND, "Uploading hidden service descriptor: http "
                        "status %d (%s) response unexpected (server "
                        "'%s:%d').",
               status_code, escaped(reason), conn->base_.address,
               conn->base_.port);
      hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
                                   "UNEXPECTED");
      break;
    }
  
    return 0;
  }
  
  /** Called when a directory connection reaches EOF. */
  int
  connection_dir_reached_eof(dir_connection_t *conn)
  {
    int retval;
    if (conn->base_.state != DIR_CONN_STATE_CLIENT_READING) {
      log_info(LD_HTTP,"conn reached eof, not reading. [state=%d] Closing.",
               conn->base_.state);
      connection_close_immediate(TO_CONN(conn)); /* error: give up on flushing */
      connection_mark_for_close(TO_CONN(conn));
      return -1;
    }
  
    retval = connection_dir_client_reached_eof(conn);
    if (retval == 0) /* success */
      conn->base_.state = DIR_CONN_STATE_CLIENT_FINISHED;
    connection_mark_for_close(TO_CONN(conn));
    return retval;
  }
  /** We are closing a dir connection: If <b>dir_conn</b> is a dir connection
   *  that tried to fetch an HS descriptor, check if it successfully fetched it,
   *  or if we need to try again. */
  void
  connection_dir_client_refetch_hsdesc_if_needed(dir_connection_t *dir_conn)
  {
    connection_t *conn = TO_CONN(dir_conn);
  
    /* If we were trying to fetch a v2 rend desc and did not succeed, retry as
     * needed. (If a fetch is successful, the connection state is changed to
     * DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 or DIR_PURPOSE_HAS_FETCHED_HSDESC to
     * mark that refetching is unnecessary.) */
    if (conn->purpose == DIR_PURPOSE_FETCH_RENDDESC_V2 &&
        dir_conn->rend_data &&
        rend_valid_v2_service_id(
             rend_data_get_address(dir_conn->rend_data))) {
      rend_client_refetch_v2_renddesc(dir_conn->rend_data);
    }
  
    /* Check for v3 rend desc fetch */
    if (conn->purpose == DIR_PURPOSE_FETCH_HSDESC &&
        dir_conn->hs_ident &&
        !ed25519_public_key_is_zero(&dir_conn->hs_ident->identity_pk)) {
      hs_client_refetch_hsdesc(&dir_conn->hs_ident->identity_pk);
    }
  }
  
  /** Array of compression methods to use (if supported) for requesting
   * compressed data, ordered from best to worst. */
  static compress_method_t client_meth_pref[] = {
    LZMA_METHOD,
    ZSTD_METHOD,
    ZLIB_METHOD,
    GZIP_METHOD,
    NO_METHOD
  };
  
  /** Array of allowed compression methods to use (if supported) when receiving a
   * response from a request that was required to be anonymous. */
  static compress_method_t client_meth_allowed_anonymous_compression[] = {
    ZLIB_METHOD,
    GZIP_METHOD,
    NO_METHOD
  };
  
  /** Return a newly allocated string containing a comma separated list of
   * supported encodings. */
  STATIC char *
  accept_encoding_header(void)
  {
    smartlist_t *methods = smartlist_new();
    char *header = NULL;
    compress_method_t method;
    unsigned i;
  
    for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) {
      method = client_meth_pref[i];
      if (tor_compress_supports_method(method))
        smartlist_add(methods, (char *)compression_method_get_name(method));
    }
  
    header = smartlist_join_strings(methods, ", ", 0, NULL);
    smartlist_free(methods);
  
    return header;
  }
  
  /** Check if the given compression method is allowed for a connection that is
   * supposed to be anonymous. Returns 1 if the compression method is allowed,
   * otherwise 0. */
  STATIC int
  allowed_anonymous_connection_compression_method(compress_method_t method)
  {
    unsigned u;
  
    for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression);
         ++u) {
      compress_method_t allowed_method =
        client_meth_allowed_anonymous_compression[u];
  
      if (! tor_compress_supports_method(allowed_method))
        continue;
  
      if (method == allowed_method)
        return 1;
    }
  
    return 0;
  }
  
  /** Log a warning when a remote server has sent us a document using a
   * compression method that is not allowed for anonymous directory requests. */
  STATIC void
  warn_disallowed_anonymous_compression_method(compress_method_t method)
  {
    log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
           "Received a %s HTTP response, which is not "
           "allowed for anonymous directory requests.",
           compression_method_get_human_name(method));
  }
  
  /* We just got a new consensus! If there are other in-progress requests
   * for this consensus flavor (for example because we launched several in
   * parallel), cancel them.
   *
   * We do this check here (not just in
   * connection_ap_handshake_attach_circuit()) to handle the edge case where
   * a consensus fetch begins and ends before some other one tries to attach to
   * a circuit, in which case the other one won't know that we're all happy now.
   *
   * Don't mark the conn that just gave us the consensus -- otherwise we
   * would end up double-marking it when it cleans itself up.
   */
  static void
  connection_dir_close_consensus_fetches(dir_connection_t *except_this_one,
                                         const char *resource)
  {
    smartlist_t *conns_to_close =
      connection_dir_list_by_purpose_and_resource(DIR_PURPOSE_FETCH_CONSENSUS,
                                                  resource);
    SMARTLIST_FOREACH_BEGIN(conns_to_close, dir_connection_t *, d) {
      if (d == except_this_one)
        continue;
      log_info(LD_DIR, "Closing consensus fetch (to %s) since one "
               "has just arrived.", TO_CONN(d)->address);
      connection_mark_for_close(TO_CONN(d));
    } SMARTLIST_FOREACH_END(d);
    smartlist_free(conns_to_close);
  }
  /** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>)
   * fetches have failed (with uppercase fingerprints listed in <b>failed</b>,
   * either as descriptor digests or as identity digests based on
   * <b>was_descriptor_digests</b>).
   */
  static void
  dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
                                 int router_purpose,
                                 int was_extrainfo, int was_descriptor_digests)
  {
    char digest[DIGEST_LEN];
    time_t now = time(NULL);
    int server = directory_fetches_from_authorities(get_options());
    if (!was_descriptor_digests) {
      if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
        tor_assert(!was_extrainfo);
        connection_dir_retry_bridges(failed);
      }
      return; /* FFFF should implement for other-than-router-purpose someday */
    }
    SMARTLIST_FOREACH_BEGIN(failed, const char *, cp) {
      download_status_t *dls = NULL;
      if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp)) != DIGEST_LEN) {
        log_warn(LD_BUG, "Malformed fingerprint in list: %s", escaped(cp));
        continue;
      }
      if (was_extrainfo) {
        signed_descriptor_t *sd =
          router_get_by_extrainfo_digest(digest);
        if (sd)
          dls = &sd->ei_dl_status;
      } else {
        dls = router_get_dl_status_by_descriptor_digest(digest);
      }
      if (!dls)
        continue;
      download_status_increment_failure(dls, status_code, cp, server, now);
    } SMARTLIST_FOREACH_END(cp);
  
    /* No need to relaunch descriptor downloads here: we already do it
     * every 10 or 60 seconds (FOO_DESCRIPTOR_RETRY_INTERVAL) in main.c. */
  }
  
  /** Called when a connection to download microdescriptors from relay with
   * <b>dir_id</b> has failed in whole or in part. <b>failed</b> is a list
   * of every microdesc digest we didn't get. <b>status_code</b> is the http
   * status code we received. Reschedule the microdesc downloads as
   * appropriate. */
  static void
  dir_microdesc_download_failed(smartlist_t *failed,
                                int status_code, const char *dir_id)
  {
    networkstatus_t *consensus
      = networkstatus_get_latest_consensus_by_flavor(FLAV_MICRODESC);
    routerstatus_t *rs;
    download_status_t *dls;
    time_t now = time(NULL);
    int server = directory_fetches_from_authorities(get_options());
  
    if (! consensus)
      return;
  
    /* We failed to fetch a microdescriptor from 'dir_id', note it down
     * so that we don't try the same relay next time... */
    microdesc_note_outdated_dirserver(dir_id);
  
    SMARTLIST_FOREACH_BEGIN(failed, const char *, d) {
      rs = router_get_mutable_consensus_status_by_descriptor_digest(consensus,d);
      if (!rs)
        continue;
      dls = &rs->dl_status;
  
      { /* Increment the failure count for this md fetch */
        char buf[BASE64_DIGEST256_LEN+1];
        digest256_to_base64(buf, d);
        log_info(LD_DIR, "Failed to download md %s from %s",
                 buf, hex_str(dir_id, DIGEST_LEN));
        download_status_increment_failure(dls, status_code, buf,
                                          server, now);
      }
    } SMARTLIST_FOREACH_END(d);
  }
  