text: /*
     p0f - HTTP fingerprinting
     -------------------------
  
     Copyright (C) 2012 by Michal Zalewski <lcamtuf@coredump.cx>
  
     Distributed under the terms and conditions of GNU LGPL.
  
   */
  
  #define _FROM_FP_HTTP
  #define _GNU_SOURCE
  
  #include <stdio.h>
  #include <stdlib.h>
  #include <unistd.h>
  #include <ctype.h>
  #include <time.h>
  
  #include <netinet/in.h>
  #include <sys/types.h>
  
  #include "types.h"
  #include "config.h"
  #include "debug.h"
  #include "alloc-inl.h"
  #include "process.h"
  #include "readfp.h"
  #include "p0f.h"
  #include "tcp.h"
  #include "hash.h"
  
  #include "fp_http.h"
  #include "languages.h"
  
  static u8** hdr_names;                 /* List of header names by ID         */
  static u32  hdr_cnt;                   /* Number of headers registered       */
  
  static u32* hdr_by_hash[SIG_BUCKETS];  /* Hashed header names                */
  static u32  hbh_cnt[SIG_BUCKETS];      /* Number of headers in bucket        */
  
  /* Signatures aren't bucketed due to the complex matching used; but we use
     Bloom filters to go through them quickly. */
  
  static struct http_sig_record* sigs[2];
  static u32 sig_cnt[2];
  
  static struct ua_map_record* ua_map;   /* Mappings between U-A and OS        */
  static u32 ua_map_cnt;
  
  #define SLOF(_str) (u8*)_str, strlen((char*)_str)
  
  
  /* Ghetto Bloom filter 4-out-of-64 bitmask generator for adding 32-bit header
     IDs to a set. We expect around 10 members in a set. */
  
  static inline u64 bloom4_64(u32 val) {
    u32 hash = hash32(&val, 4, hash_seed);
    u64 ret;
    ret = (1LL << (hash & 63));
    ret ^= (1LL << ((hash >> 8) & 63));
    ret ^= (1LL << ((hash >> 16) & 63));
    ret ^= (1LL << ((hash >> 24) & 63));
    return ret;
  }
  
  
  /* Look up or register new header */
  
  static s32 lookup_hdr(u8* name, u32 len, u8 create) {
  
    u32  bucket = hash32(name, len, hash_seed) % SIG_BUCKETS;
  
    u32* p = hdr_by_hash[bucket];
    u32  i = hbh_cnt[bucket];
  
    while (i--) {
      if (!memcmp(hdr_names[*p], name, len) && 
          !hdr_names[*p][len]) return *p;
      p++;
    }
  
    /* Not found! */
  
    if (!create) return -1;
  
    hdr_names = DFL_ck_realloc(hdr_names, (hdr_cnt + 1) * sizeof(u8*));
    hdr_names[hdr_cnt] = DFL_ck_memdup_str(name, len);
  
    hdr_by_hash[bucket] = DFL_ck_realloc(hdr_by_hash[bucket],
      (hbh_cnt[bucket] + 1) * 4);
  
    hdr_by_hash[bucket][hbh_cnt[bucket]++] = hdr_cnt++;
  
    return hdr_cnt - 1;
  
  }
  
  
  /* Pre-register essential headers. */
  
  void http_init(void) {
    u32 i;
  
    /* Do not change - other code depends on the ordering of first 6 entries. */
  
    lookup_hdr(SLOF("User-Agent"), 1);      /* 0 */
    lookup_hdr(SLOF("Server"), 1);          /* 1 */
    lookup_hdr(SLOF("Accept-Language"), 1); /* 2 */
    lookup_hdr(SLOF("Via"), 1);             /* 3 */
    lookup_hdr(SLOF("X-Forwarded-For"), 1); /* 4 */
    lookup_hdr(SLOF("Date"), 1);            /* 5 */
  
  #define HDR_UA  0
  #define HDR_SRV 1
  #define HDR_AL  2
  #define HDR_VIA 3
  #define HDR_XFF 4
  #define HDR_DAT 5
  
    i = 0;
    while (req_optional[i].name) {
      req_optional[i].id = lookup_hdr(SLOF(req_optional[i].name), 1);
      i++;
    }
  
    i = 0;
    while (resp_optional[i].name) {
      resp_optional[i].id = lookup_hdr(SLOF(resp_optional[i].name), 1);
      i++;
    }
  
    i = 0;
    while (req_skipval[i].name) {
      req_skipval[i].id = lookup_hdr(SLOF(req_skipval[i].name), 1);
      i++;
    }
  
    i = 0;
    while (resp_skipval[i].name) {
      resp_skipval[i].id = lookup_hdr(SLOF(resp_skipval[i].name), 1);
      i++;
    }
  
    i = 0;
    while (req_common[i].name) {
      req_common[i].id = lookup_hdr(SLOF(req_common[i].name), 1);
      i++;
    }
  
    i = 0;
    while (resp_common[i].name) {
      resp_common[i].id = lookup_hdr(SLOF(resp_common[i].name), 1);
      i++;
    }
  
  }
  
  
  /* Find match for a signature. */
  
  static void http_find_match(u8 to_srv, struct http_sig* ts, u8 dupe_det) {
  
    struct http_sig_record* gmatch = NULL;
    struct http_sig_record* ref = sigs[to_srv];
    u32 cnt = sig_cnt[to_srv];
  
    while (cnt--) {
  
      struct http_sig* rs = ref->sig;
      u32 ts_hdr = 0, rs_hdr = 0;
  
      if (rs->http_ver != -1 && rs->http_ver != ts->http_ver) goto next_sig;
  
      /* Check that all the headers listed for the p0f.fp signature (probably)
         appear in the examined traffic. */
  
      if ((ts->hdr_bloom4 & rs->hdr_bloom4) != rs->hdr_bloom4) goto next_sig;
  
      /* Confirm the ordering and values of headers (this is relatively
         slow, hence the Bloom filter first). */
  
      while (rs_hdr < rs->hdr_cnt) {
  
        u32 orig_ts = ts_hdr;
  
        while (rs->hdr[rs_hdr].id != ts->hdr[ts_hdr].id &&
               ts_hdr < ts->hdr_cnt) ts_hdr++;
  
        if (ts_hdr == ts->hdr_cnt) {
  
          if (!rs->hdr[rs_hdr].optional) goto next_sig;
  
          /* If this is an optional header, check that it doesn't appear
             anywhere else. */
  
          for (ts_hdr = 0; ts_hdr < ts->hdr_cnt; ts_hdr++)
            if (rs->hdr[rs_hdr].id == ts->hdr[ts_hdr].id) goto next_sig;
  
          ts_hdr = orig_ts;
          rs_hdr++;
          continue;
  
        }
  
        if (rs->hdr[rs_hdr].value &&
            (!ts->hdr[ts_hdr].value ||
            !strstr((char*)ts->hdr[ts_hdr].value,
            (char*)rs->hdr[rs_hdr].value))) goto next_sig;
  
        ts_hdr++;
        rs_hdr++;
  
      }
  
      /* Check that the headers forbidden in p0f.fp don't appear in the traffic.
         We first check if they seem to appear in ts->hdr_bloom4, and only if so,
         we do a full check. */
  
      for (rs_hdr = 0; rs_hdr < rs->miss_cnt; rs_hdr++) {
  
        u64 miss_bloom4 = bloom4_64(rs->miss[rs_hdr]);
  
        if ((ts->hdr_bloom4 & miss_bloom4) != miss_bloom4) continue;
  
        /* Okay, possible instance of a banned header - scan list... */
  
        for (ts_hdr = 0; ts_hdr < ts->hdr_cnt; ts_hdr++)
          if (rs->miss[rs_hdr] == ts->hdr[ts_hdr].id) goto next_sig;
  
      }
  
      /* When doing dupe detection, we want to allow a signature with additional
         banned headers to precede one with fewer, or with a different set. */
  
      if (dupe_det) {
  
        if (rs->miss_cnt > ts->miss_cnt) goto next_sig;
  
        for (rs_hdr = 0; rs_hdr < rs->miss_cnt; rs_hdr++) {
  
          for (ts_hdr = 0; ts_hdr < ts->miss_cnt; ts_hdr++) 
            if (rs->miss[rs_hdr] == ts->miss[ts_hdr]) break;
  
          /* One of the reference headers doesn't appear in current sig! */
  
          if (ts_hdr == ts->miss_cnt) goto next_sig;
  
        }
  
  
      }
  
      /* Whoa, a match. */
  
      if (!ref->generic) {
  
        ts->matched = ref;
  
        if (rs->sw && ts->sw && !strstr((char*)ts->sw, (char*)rs->sw))
          ts->dishonest = 1;
  
        return;
  
      } else if (!gmatch) gmatch = ref;
  
  next_sig:
  
      ref = ref + 1;
  
    }
  
    /* A generic signature is the best we could find. */
  
    if (!dupe_det && gmatch) {
  
      ts->matched = gmatch;
  
      if (gmatch->sig->sw && ts->sw && !strstr((char*)ts->sw,
          (char*)gmatch->sig->sw)) ts->dishonest = 1;
  
    }
  
  }
  
  
  /* Register new HTTP signature. */
  
  void http_register_sig(u8 to_srv, u8 generic, s32 sig_class, u32 sig_name,
                         u8* sig_flavor, u32 label_id, u32* sys, u32 sys_cnt,
                         u8* val, u32 line_no) {
  
    struct http_sig* hsig;
    struct http_sig_record* hrec;
  
    u8* nxt;
  
    hsig = DFL_ck_alloc(sizeof(struct http_sig));
  
    sigs[to_srv] = DFL_ck_realloc(sigs[to_srv], sizeof(struct http_sig_record) *
      (sig_cnt[to_srv] + 1));
  
    hrec = &sigs[to_srv][sig_cnt[to_srv]];
  
    if (val[1] != ':') FATAL("Malformed signature in line %u.", line_no);
  
    /* http_ver */
  
    switch (*val) {
      case '0': break;
      case '1': hsig->http_ver = 1; break;
      case '*': hsig->http_ver = -1; break;
      default: FATAL("Bad HTTP version in line %u.", line_no);
    }
  
    val += 2;
  
    /* horder */
  
    while (*val != ':') {
  
      u32 id;
      u8 optional = 0;
  
      if (hsig->hdr_cnt >= HTTP_MAX_HDRS)
        FATAL("Too many headers listed in line %u.", line_no);
  
      nxt = val;
  
      if (*nxt == '?') { optional = 1; val++; nxt++; }
  
      while (isalnum(*nxt) || *nxt == '-' || *nxt == '_') nxt++;
  
      if (val == nxt)
        FATAL("Malformed header name in line %u.", line_no);
  
      id = lookup_hdr(val, nxt - val, 1);
  
      hsig->hdr[hsig->hdr_cnt].id = id;
      hsig->hdr[hsig->hdr_cnt].optional = optional;
  
      if (!optional) hsig->hdr_bloom4 |= bloom4_64(id);
  
      val = nxt;
  
      if (*val == '=') {
  
        if (val[1] != '[') 
          FATAL("Missing '[' after '=' in line %u.", line_no);
  
        val += 2;
        nxt = val;
  
        while (*nxt && *nxt != ']') nxt++;
  
        if (val == nxt || !*nxt)
          FATAL("Malformed signature in line %u.", line_no);
  
        hsig->hdr[hsig->hdr_cnt].value = DFL_ck_memdup_str(val, nxt - val);
  
        val = nxt + 1;
  
      }
  
      hsig->hdr_cnt++;
  
      if (*val == ',') val++; else if (*val != ':')
        FATAL("Malformed signature in line %u.", line_no);
  
    }
  
    val++;
  
    /* habsent */
  
    while (*val != ':') {
  
      u32 id;
  
      if (hsig->miss_cnt >= HTTP_MAX_HDRS)
        FATAL("Too many headers listed in line %u.", line_no);
  
      nxt = val;
      while (isalnum(*nxt) || *nxt == '-' || *nxt == '_') nxt++;
  
      if (val == nxt)
        FATAL("Malformed header name in line %u.", line_no);
  
      id = lookup_hdr(val, nxt - val, 1);
  
      hsig->miss[hsig->miss_cnt] = id;
  
      val = nxt;
  
      hsig->miss_cnt++;
  
      if (*val == ',') val++; else if (*val != ':')
        FATAL("Malformed signature in line %u.", line_no);
  
    }
  
    val++;
  
    /* exp_sw */
  
    if (*val) {
  
      if (strchr((char*)val, ':'))
        FATAL("Malformed signature in line %u.", line_no);
  
      hsig->sw = DFL_ck_strdup(val);
  
    }
  
    http_find_match(to_srv, hsig, 1);
  
    if (hsig->matched)
      FATAL("Signature in line %u is already covered by line %u.",
            line_no, hsig->matched->line_no);
  
    hrec->class_id = sig_class;
    hrec->name_id  = sig_name;
    hrec->flavor   = sig_flavor;
    hrec->label_id = label_id;
    hrec->sys      = sys;
    hrec->sys_cnt  = sys_cnt;
    hrec->line_no  = line_no;
    hrec->generic  = generic;
  
    hrec->sig      = hsig;
  
    sig_cnt[to_srv]++;
  
  }
  
  
  /* Register new HTTP signature. */
  
  void http_parse_ua(u8* val, u32 line_no) {
  
    u8* nxt;
  
    while (*val) {
  
      u32 id;
      u8* name = NULL;
  
      nxt = val;
      while (*nxt && (isalnum(*nxt) || strchr(NAME_CHARS, *nxt))) nxt++;
  
      if (val == nxt)
        FATAL("Malformed system name in line %u.", line_no);
  
      id = lookup_name_id(val, nxt - val);
  
      val = nxt;
  
      if (*val == '=') {
  
        if (val[1] != '[') 
          FATAL("Missing '[' after '=' in line %u.", line_no);
  
        val += 2;
        nxt = val;
  
        while (*nxt && *nxt != ']') nxt++;
  
        if (val == nxt || !*nxt)
          FATAL("Malformed signature in line %u.", line_no);
  
        name = DFL_ck_memdup_str(val, nxt - val);
  
        val = nxt + 1;
  
      }
  
      ua_map = DFL_ck_realloc(ua_map, (ua_map_cnt + 1) *
                          sizeof(struct ua_map_record));
  
      ua_map[ua_map_cnt].id = id;
     
      if (!name) ua_map[ua_map_cnt].name = fp_os_names[id];
        else ua_map[ua_map_cnt].name = name;
  
      ua_map_cnt++;
  
      if (*val == ',') val++;
  
    }
  
  }
  
  
  /* Dump a HTTP signature. */
  
  static u8* dump_sig(u8 to_srv, struct http_sig* hsig) {
  
    u32 i;
    u8 had_prev = 0;
    struct http_id* list;
  
    u8 tmp[HTTP_MAX_SHOW + 1];
    u32 tpos;
  
    static u8* ret;
    u32 rlen = 0;
  
    u8* val;
  
  #define RETF(_par...) do { \
      s32 _len = snprintf(NULL, 0, _par); \
      if (_len < 0) FATAL("Whoa, snprintf() fails?!"); \
      ret = DFL_ck_realloc_kb(ret, rlen + _len + 1); \
      snprintf((char*)ret + rlen, _len + 1, _par); \
      rlen += _len; \
    } while (0)
      
    RETF("%u:", hsig->http_ver);
  
    for (i = 0; i < hsig->hdr_cnt; i++) {
  
      if (hsig->hdr[i].id >= 0) {
  
        u8 optional = 0;
  
        /* Check the "optional" list. */
  
        list = to_srv ? req_optional : resp_optional;
  
        while (list->name) {
          if (list->id == hsig->hdr[i].id) break;
          list++;
        }
  
        if (list->name) optional = 1;
  
        RETF("%s%s%s", had_prev ? "," : "", optional ? "?" : "",
             hdr_names[hsig->hdr[i].id]);
        had_prev = 1;
  
        if (!(val = hsig->hdr[i].value)) continue;
  
        /* Next, make sure that the value is not on the ignore list. */
  
        if (optional) continue;
  
        list = to_srv ? req_skipval : resp_skipval;
  
        while (list->name) {
          if (list->id == hsig->hdr[i].id) break;
          list++;
        }
  
        if (list->name) continue;
  
        /* Looks like it's not on the list, so let's output a cleaned-up version
           up to HTTP_MAX_SHOW. */
  
        tpos = 0;
  
        while (tpos < HTTP_MAX_SHOW && val[tpos] >= 0x20 && val[tpos] < 0x80 &&
               val[tpos] != ']' && val[tpos] != '|') {
  
          tmp[tpos] = val[tpos];
          tpos++;
  
        }
  
        tmp[tpos] = 0;
  
        if (!tpos) continue;
  
        RETF("=[%s]", tmp);
  
      } else {
  
        RETF("%s%s", had_prev ? "," : "", hsig->hdr[i].name);
        had_prev = 1;
  
        if (!(val = hsig->hdr[i].value)) continue;
  
        tpos = 0;
  
        while (tpos < HTTP_MAX_SHOW && val[tpos] >= 0x20 && val[tpos] < 0x80 &&
               val[tpos] != ']') { tmp[tpos] = val[tpos]; tpos++; }
  
        tmp[tpos] = 0;
  
        if (!tpos) continue;
  
        RETF("=[%s]", tmp);
  
      }
  
    }
  
    RETF(":");
  
    list = to_srv ? req_common : resp_common;
    had_prev = 0;
  
    while (list->name) {
  
      for (i = 0; i < hsig->hdr_cnt; i++) 
        if (hsig->hdr[i].id == list->id) break;
  
      if (i == hsig->hdr_cnt) {
        RETF("%s%s", had_prev ? "," : "", list->name);
        had_prev = 1;
      }
  
      list++;
  
    }
  
    RETF(":");
  
    if ((val = hsig->sw)) {
  
      tpos = 0;
  
      while (tpos < HTTP_MAX_SHOW &&  val[tpos] >= 0x20 && val[tpos] < 0x80 &&
             val[tpos] != ']') { tmp[tpos] = val[tpos]; tpos++; }
  
      tmp[tpos] = 0;
  
      if (tpos) RETF("%s", tmp);
  
    }
  
    return ret;
  
  
  }
  
  
  /* Dump signature flags. */
  
  static u8* dump_flags(struct http_sig* hsig, struct http_sig_record* m) {
  
    static u8* ret;
    u32 rlen = 0;
  
    RETF("");
  
    if (hsig->dishonest) RETF(" dishonest");
    if (!hsig->sw) RETF(" anonymous");
    if (m && m->generic) RETF(" generic");
  
  #undef RETF
  
    if (*ret) return ret + 1; else return (u8*)"none";
  
  }
  
  
  /* Score signature differences. For unknown signatures, the presumption is that
     they identify apps, so the logic is quite different from TCP. */
  
  static void score_nat(u8 to_srv, struct packet_flow* f) {
  
    struct http_sig_record* m = f->http_tmp.matched;
    struct host_data* hd;
    struct http_sig* ref;
  
    u8  score = 0, diff_already = 0;
    u16 reason = 0;
  
    if (to_srv) {
  
      hd = f->client;
      ref = hd->http_req_os;
  
    } else {
  
      hd = f->server;
      ref = hd->http_resp;
  
      /* If the signature is for a different port, don't read too much into it. */
  
      if (hd->http_resp_port != f->srv_port) ref = NULL;
  
    }
  
    if (!m) {
  
      /* No match. The user is probably running another app; this is only of
         interest if a server progresses from known to unknown. We can't
         compare two unknown server sigs with that much confidence. */
  
      if (!to_srv && ref && ref->matched) {
  
        DEBUG("[#] HTTP server signature changed from known to unknown.\n");
        score  += 4;
        reason |= NAT_TO_UNK;
  
      }
  
      goto header_check;
  
    }
  
    if (m->class_id == -1) {
  
      /* Got a match for an application signature. Make sure it runs on the
         OS we have on file... */
  
      verify_tool_class(to_srv, f, m->sys, m->sys_cnt);
  
      /* ...and check for inconsistencies in server behavior. */
  
      if (!to_srv && ref && ref->matched) {
  
        if (ref->matched->name_id != m->name_id) {
  
          DEBUG("[#] Name on the matched HTTP server signature changes.\n");
          score  += 8;
          reason |= NAT_APP_LB;
  
        } else if (ref->matched->label_id != m->label_id) {
  
          DEBUG("[#] Label on the matched HTTP server signature changes.\n");
          score  += 2;
          reason |= NAT_APP_LB;
  
        }
  
      }
  
    } else {
  
      /* Ooh, spiffy: a match for an OS signature! There will be about two uses
         for this code, ever. */
  
      if (ref && ref->matched) {
  
        if (ref->matched->name_id != m->name_id) {
  
          DEBUG("[#] Name on the matched HTTP OS signature changes.\n");
          score  += 8;
          reason |= NAT_OS_SIG;
          diff_already = 1;
  
        } else if (ref->matched->name_id != m->label_id) {
  
          DEBUG("[#] Label on the matched HTTP OS signature changes.\n");
          score  += 2;
          reason |= NAT_OS_SIG;
  
        }
  
      } else if (ref) {
  
        DEBUG("[#] HTTP OS signature changed from unknown to known.\n");
        score  += 4;
        reason |= NAT_TO_UNK;
  
      }
  
      /* If we haven't pointed out anything major yet, also complain if the
         signature doesn't match host data. */
  
      if (!diff_already && hd->last_name_id != m->name_id) {
  
        DEBUG("[#] Matched HTTP OS signature different than host data.\n");
        score  += 4;
        reason |= NAT_OS_SIG;
  
      }
  
    }
  
    /* If we have determined that U-A looks legit, but the OS doesn't match,
       that's a clear sign of trouble. */
  
    if (to_srv && m->class_id == -1 && f->http_tmp.sw && !f->http_tmp.dishonest) {
  
      u32 i;
  
      for (i = 0; i < ua_map_cnt; i++)
        if (strstr((char*)f->http_tmp.sw, (char*)ua_map[i].name)) break;
  
      if (i != ua_map_cnt) {
  
        if (ua_map[i].id != hd->last_name_id) {
  
          DEBUG("[#] Otherwise plausible User-Agent points to another OS.\n");
          score  += 4;
          reason |= NAT_APP_UA;
  
          if (!hd->bad_sw) hd->bad_sw = 1;
  
        } else {
  
          DEBUG("[#] User-Agent OS value checks out.\n");
          hd->bad_sw = 0;
  
        }
  
      }
  
    }
  
  
  header_check:
  
    /* Okay, some last-resort checks. This is obviously concerning: */
  
    if (f->http_tmp.via) {
      DEBUG("[#] Explicit use of Via or X-Forwarded-For.\n");
      score  += 8;
      reason |= NAT_APP_VIA;
    }
  
    /* Last but not least, see what happened to 'Date': */
  
    if (ref && !to_srv && ref->date && f->http_tmp.date) {
  
      s64 recv_diff = ((s64)f->http_tmp.recv_date) - ref->recv_date;
      s64 hdr_diff  = ((s64)f->http_tmp.date) - ref->date;
  
      if (hdr_diff < -HTTP_MAX_DATE_DIFF || 
          hdr_diff > recv_diff + HTTP_MAX_DATE_DIFF) {
  
        DEBUG("[#] HTTP 'Date' distance too high (%lld in %lld sec).\n",
               hdr_diff, recv_diff);
        score  += 4;
        reason |= NAT_APP_DATE;
  
      } else {
  
        DEBUG("[#] HTTP 'Date' distance seems fine (%lld in %lld sec).\n",
               hdr_diff, recv_diff);
  
      }
  
    }
  
    add_nat_score(to_srv, f, reason, score);
  
  }
  
  
  /* Look up HTTP signature, create an observation. */
  
  static void fingerprint_http(u8 to_srv, struct packet_flow* f) {
  
    struct http_sig_record* m;
    u8* lang = NULL;
  
    http_find_match(to_srv, &f->http_tmp, 0);
  
    start_observation(to_srv ? "http request" : "http response", 4, to_srv, f);
  
    if ((m = f->http_tmp.matched)) {
  
      OBSERVF((m->class_id < 0) ? "app" : "os", "%s%s%s",
              fp_os_names[m->name_id], m->flavor ? " " : "",
              m->flavor ? m->flavor : (u8*)"");
  
    } else add_observation_field("app", NULL);
  
    if (f->http_tmp.lang && isalpha(f->http_tmp.lang[0]) &&
        isalpha(f->http_tmp.lang[1]) && !isalpha(f->http_tmp.lang[2])) {
  
      u8 lh = LANG_HASH(f->http_tmp.lang[0], f->http_tmp.lang[1]);
      u8 pos = 0;
  
      while (languages[lh][pos]) {
        if (f->http_tmp.lang[0] == languages[lh][pos][0] &&
            f->http_tmp.lang[1] == languages[lh][pos][1]) break;
        pos += 2;
      }
  
      if (!languages[lh][pos]) add_observation_field("lang", NULL);
        else add_observation_field("lang", 
             (lang = (u8*)languages[lh][pos + 1]));
  
    } else add_observation_field("lang", (u8*)"none");
  
    add_observation_field("params", dump_flags(&f->http_tmp, m));
  
    add_observation_field("raw_sig", dump_sig(to_srv, &f->http_tmp));
  
    score_nat(to_srv, f);
  
    /* Save observations needed to score future responses. */
  
    if (!to_srv) {
  
      /* For server response, always store the signature. */
  
      ck_free(f->server->http_resp);
      f->server->http_resp = ck_memdup(&f->http_tmp, sizeof(struct http_sig));
  
      f->server->http_resp->hdr_cnt = 0;
      f->server->http_resp->sw   = NULL;
      f->server->http_resp->lang = NULL;
      f->server->http_resp->via  = NULL;
  
      f->server->http_resp_port = f->srv_port;
  
      if (lang) f->server->language = lang;
  
      if (m) {
  
        if (m->class_id != -1) {
  
          /* If this is an OS signature, update host record. */
  
          f->server->last_class_id = m->class_id;
          f->server->last_name_id  = m->name_id;
          f->server->last_flavor   = m->flavor;
          f->server->last_quality  = (m->generic * P0F_MATCH_GENERIC);
  
        } else {
  
          /* Otherwise, record app data. */
  
          f->server->http_name_id = m->name_id;
          f->server->http_flavor  = m->flavor;
  
          if (f->http_tmp.dishonest) f->server->bad_sw = 2;
  
        }
  
      }
  
    } else {
  
      if (lang) f->client->language = lang;
  
      if (m) {
  
        if (m->class_id != -1) {
  
          /* Client request - only OS sig is of any note. */
  
          ck_free(f->client->http_req_os);
          f->client->http_req_os = ck_memdup(&f->http_tmp,
            sizeof(struct http_sig));
  
          f->client->http_req_os->hdr_cnt = 0;
          f->client->http_req_os->sw   = NULL;
          f->client->http_req_os->lang = NULL;
          f->client->http_req_os->via  = NULL;
  
          f->client->last_class_id = m->class_id;
          f->client->last_name_id  = m->name_id;
          f->client->last_flavor   = m->flavor;
  
          f->client->last_quality  = (m->generic * P0F_MATCH_GENERIC);
  
        } else {
  
          /* Record app data for the API. */
  
          f->client->http_name_id = m->name_id;
          f->client->http_flavor  = m->flavor;
  
          if (f->http_tmp.dishonest) f->client->bad_sw = 2;
  
        }
   
      }
  
    }
  
  }
  
  
  
  /* Free up any allocated strings in http_sig. */
  
  void free_sig_hdrs(struct http_sig* h) {
  
    u32 i;
  
    for (i = 0; i < h->hdr_cnt; i++) {
      if (h->hdr[i].name) ck_free(h->hdr[i].name);
      if (h->hdr[i].value) ck_free(h->hdr[i].value);
    }
  
  }
  
  
  /* Parse HTTP date field. */
  
  static u32 parse_date(u8* str) {
    struct tm t;
  
    if (!strptime((char*)str, "%a, %d %b %Y %H:%M:%S %Z", &t)) {
      DEBUG("[#] Invalid 'Date' field ('%s').\n", str);
      return 0;
    }
  
    return mktime(&t);
  
  }
  
  
  /* Parse name=value pairs into a signature. */
  
  static u8 parse_pairs(u8 to_srv, struct packet_flow* f, u8 can_get_more) {
  
    u32 plen = to_srv ? f->req_len : f->resp_len;
  
    u32 off;
  
    /* Try to parse name: value pairs. */
  
    while ((off = f->http_pos) < plen) {
  
      u8* pay = to_srv ? f->request : f->response;
  
      u32 nlen, vlen, vstart;
      s32 hid;
      u32 hcount;
  
      /* Empty line? Dispatch for fingerprinting! */
  
      if (pay[off] == '\r' || pay[off] == '\n') {
  
        f->http_tmp.recv_date = get_unix_time();
  
        fingerprint_http(to_srv, f);
  
        /* If this is a request, flush the collected signature and prepare
           for parsing the response. If it's a response, just shut down HTTP
           parsing on this flow. */
  
        if (to_srv) {
  
          f->http_req_done = 1;
          f->http_pos = 0;
  
          free_sig_hdrs(&f->http_tmp);
          memset(&f->http_tmp, 0, sizeof(struct http_sig));
  
          return 1;
  
        } else {
  
          f->in_http = -1;
          return 0;
  
        }
  
      }
  
      /* Looks like we're getting a header value. See if we have room for it. */
  
      if ((hcount = f->http_tmp.hdr_cnt) >= HTTP_MAX_HDRS) {
  
        DEBUG("[#] Too many HTTP headers in a %s.\n", to_srv ? "request" :
              "response");
  
        f->in_http = -1;
        return 0;
  
      }
  
      /* Try to extract header name. */
        
      nlen = 0;
  
      while ((isalnum(pay[off]) || pay[off] == '-' || pay[off] == '_') &&
             off < plen && nlen <= HTTP_MAX_HDR_NAME) {
  
        off++;
        nlen++;
  
      }
  
      if (off == plen) {
  
        if (!can_get_more) {
  
          DEBUG("[#] End of HTTP %s before end of headers.\n",
                to_srv ? "request" : "response");
          f->in_http = -1;
  
        }
  
        return can_get_more;
  
      }
  
      /* Empty, excessively long, or non-':'-followed header name? */
  
      if (!nlen || pay[off] != ':' || nlen > HTTP_MAX_HDR_NAME) {
  
        DEBUG("[#] Invalid HTTP header encountered (len = %u, char = 0x%02x).\n",
              nlen, pay[off]);
  
        f->in_http = -1;
        return 0;
  
      }
  
      /* At this point, header name starts at f->http_pos, and has nlen bytes.
         Skip ':' and a subsequent whitespace next. */
  
      off++;
  
      if (off < plen && isblank(pay[off])) off++;
  
      vstart = off;
      vlen = 0;
  
      /* Find the next \n. */
  
      while (off < plen && vlen <= HTTP_MAX_HDR_VAL && pay[off] != '\n') {
  
        off++;
        vlen++;
  
      }
  
      if (vlen > HTTP_MAX_HDR_VAL) {
        DEBUG("[#] HTTP %s header value length exceeded.\n",
              to_srv ? "request" : "response");
        f->in_http = -1;
        return -1;
      }
  
      if (off == plen) {
  
        if (!can_get_more) {
          DEBUG("[#] End of HTTP %s before end of headers.\n",
                to_srv ? "request" : "response");
          f->in_http = -1;
        }
  
        return can_get_more;
  
      }
  
      /* If party is using \r\n terminators, go back one char. */
  
      if (pay[off - 1] == '\r') vlen--;
   
      /* Header value starts at vstart, and has vlen bytes (may be zero). Record
         this in the signature. */
  
      hid = lookup_hdr(pay + f->http_pos, nlen, 0);
  
      f->http_tmp.hdr[hcount].id = hid;
  
      if (hid < 0) {
  
        /* Header ID not found, store literal value. */
  
        f->http_tmp.hdr[hcount].name = ck_memdup_str(pay + f->http_pos, nlen);
  
      } else {
  
        /* Found - update Bloom filter. */
  
        f->http_tmp.hdr_bloom4 |= bloom4_64(hid);
  
      }
  
      /* If there's a value, store that too. For U-A and Server, also update
         'sw'; and for requests, collect Accept-Language. */
  
      if (vlen) {
  
        u8* val = ck_memdup_str(pay + vstart, vlen);
  
        f->http_tmp.hdr[hcount].value = val;
  
        if (to_srv) {
  
          switch (hid) {
            case HDR_UA: f->http_tmp.sw = val; break;
            case HDR_AL: f->http_tmp.lang = val; break;
            case HDR_VIA:
            case HDR_XFF: f->http_tmp.via = val; break;
          }
  
        } else {
  
          switch (hid) {
  
            case HDR_SRV: f->http_tmp.sw = val; break;
            case HDR_DAT: f->http_tmp.date = parse_date(val); break;
            case HDR_VIA:
            case HDR_XFF: f->http_tmp.via = val; break;
  
          }
  
        }
  
      }
  
      /* Moving on... */
  
      f->http_tmp.hdr_cnt++;
      f->http_pos = off + 1; 
  
    }
  
    if (!can_get_more) {
      DEBUG("[#] End of HTTP %s before end of headers.\n",
            to_srv ? "request" : "response");
      f->in_http = -1;
    }
  
    return can_get_more;
  
  }
  
  
  /* Examine request or response; returns 1 if more data needed and plausibly can
     be read. Note that the buffer is always NUL-terminated. */
  
  u8 process_http(u8 to_srv, struct packet_flow* f) {
  
    /* Already decided this flow is not worth tracking? */
  
    if (f->in_http < 0) return 0;
  
    if (to_srv) {
  
      u8* pay = f->request;
      u8 can_get_more = (f->req_len < MAX_FLOW_DATA);
      u32 off;
  
      /* Request done, but pending response? */
  
      if (f->http_req_done) return 1;
  
      if (!f->in_http) {
  
        u8 chr;
        u8* sig_at;
  
        /* Ooh, new flow! */
  
        if (f->req_len < 15) return can_get_more;
  
        /* Scan until \n, or until binary data spotted. */
  
        off = f->http_pos;
  
        /* We only care about GET and HEAD requests at this point. */
  
        if (!off && strncmp((char*)pay, "GET /", 5) &&
            strncmp((char*)pay, "HEAD /", 6)) {
          DEBUG("[#] Does not seem like a GET / HEAD request.\n");
          f->in_http = -1;
          return 0;
        }
  
        while (off < f->req_len && off < HTTP_MAX_URL &&
               (chr = pay[off]) != '\n') {
  
          if (chr != '\r' && (chr < 0x20 || chr > 0x7f)) {
  
            DEBUG("[#] Not HTTP - character 0x%02x encountered.\n", chr);
  
            f->in_http = -1;
            return 0;
          }
  
          off++;
    
        }
  
        /* Newline too far or too close? */
  
        if (off == HTTP_MAX_URL || off < 14) {
  
          DEBUG("[#] Not HTTP - newline offset %u.\n", off);
  
          f->in_http = -1;
          return 0;
  
        }
  
        /* Not enough data yet? */
  
        if (off == f->req_len) {
  
          f->http_pos = off;
  
          if (!can_get_more) {
  
            DEBUG("[#] Not HTTP - no opening line found.\n");
            f->in_http = -1;
  
          }
  
          return can_get_more;
  
        }
  
        sig_at = pay + off - 8;
        if (pay[off - 1] == '\r') sig_at--;
  
        /* Bad HTTP/1.x signature? */
  
        if (strncmp((char*)sig_at, "HTTP/1.", 7)) {
  
          DEBUG("[#] Not HTTP - bad signature.\n");
  
          f->in_http = -1;
          return 0;
  
        }
  
        f->http_tmp.http_ver = (sig_at[7] == '1');
  
        f->in_http  = 1;
        f->http_pos = off + 1;
  
        DEBUG("[#] HTTP detected.\n");
  
      }
  
      return parse_pairs(1, f, can_get_more);
  
    } else {
  
      u8* pay = f->response;
      u8 can_get_more = (f->resp_len < MAX_FLOW_DATA);
      u32 off;
  
      /* Response before request? Bail out. */
  
      if (!f->in_http || !f->http_req_done) {
        f->in_http = -1;
        return 0;
      }
  
      if (!f->http_gotresp1) {
  
        u8 chr;
  
        if (f->resp_len < 13) return can_get_more;
  
        /* Scan until \n, or until binary data spotted. */
  
        off = f->http_pos;
  
        while (off < f->resp_len && off < HTTP_MAX_URL &&
               (chr = pay[off]) != '\n') {
  
          if (chr != '\r' && (chr < 0x20 || chr > 0x7f)) {
  
            DEBUG("[#] Invalid HTTP response - character 0x%02x encountered.\n",
                  chr);
            f->in_http = -1;
            return 0;
  
          }
  
          off++;
    
        }
  
        /* Newline too far or too close? */
  
        if (off == HTTP_MAX_URL || off < 13) {
  
          DEBUG("[#] Invalid HTTP response - newline offset %u.\n", off);
  
          f->in_http = -1;
          return 0;
  
        }
  
        /* Not enough data yet? */
  
        if (off == f->resp_len) {
  
          f->http_pos = off;
  
          if (!can_get_more) {
  
            DEBUG("[#] Invalid HTTP response - no opening line found.\n");
            f->in_http = -1;
  
          }
  
          return can_get_more;
  
        }
  
        /* Bad HTTP/1.x signature? */
  
        if (strncmp((char*)pay, "HTTP/1.", 7)) {
  
          DEBUG("[#] Invalid HTTP response - bad signature.\n");
  
          f->in_http = -1;
          return 0;
  
        }
  
        f->http_tmp.http_ver = (pay[7] == '1');
  
        f->http_pos = off + 1;
  
        DEBUG("[#] HTTP response starts correctly.\n");
  
  
      }
  
      return parse_pairs(0, f, can_get_more);
  
    }
  
  }
  