text: #!/usr/bin/env python
  # encoding: utf-8
  """
  identitycrisis.py
  
  Created by Sandro Gauci on 2014-04-23.
  Copyright (c) 2014 Enable Security ltd. All rights reserved.
  
  Inspired from Chris John Riley's excellent UATester:
  <http://blog.c22.cc/toolsscripts/>
  """
  
  from difflib import SequenceMatcher
  import threading
  
  from burp import IBurpExtender
  from burp import IContextMenuFactory
  from burp import IScanIssue
  from javax.swing import JMenuItem
  from javax.swing import JMenu
  from javax.swing import JFileChooser
  from java.util import ArrayList
  from java.io import PrintWriter
  
  
  
  EXTENSION_NAME = 'Identity Crisis (User-Agent Tester)'
  __VERSION__ = 0.2
  DEBUG = False
  
  class Similarity:
      """
      Train with strings/buffers to find a norm, then use is_different()
      to see if a particular content is different from the norm
      """
      def __init__(self, initdata, lower=False):
          self.lower = lower
          self.initdata = self.getsection(initdata)
          self.norm = 10.0
  
      def getsection(self, data):
          if self.lower:
              if len(data) > 1024:
                  data = data[-1024:]
              else:
                  data = ''
          else:
              data = data[:1024]
          return data
  
      def train(self, newdata):
          newratio = self.get_ratio(newdata)
          if self.norm > newratio:
              self.set_norm(newratio)
  
      def get_ratio(self, newdata):
          newdata = self.getsection(newdata)
          s = SequenceMatcher(lambda x: x == " ",
                              self.initdata,
                              newdata)
          return s.ratio()
  
      def set_norm(self, norm):
          self.norm = norm
  
      def is_different(self, newdata):
          ratio = self.get_ratio(newdata)
          if ratio < self.norm:
              return True
          return False
  
  
  class BurpExtender(IBurpExtender, IContextMenuFactory):
      banned_headers = ['expires', 'vtag', 'etag', 'date', 'time', 'set-cookie', 'x-transaction', 'x-cache', 'age']
      useragents = dict()
      useragents['Desktop Web Browsers'] = \
          [
              "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)",
              "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0)",
              "Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US)",
              "Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)",
              "Mozilla/4.0 (compatible;MSIE 5.5; Windows 98)",
              "Mozilla/5.0 (Windows; U; Windows NT 6.1; ru; rv:1.9.2.3) Gecko/20100401 Firefox/4.0 (.NET CLR 3.5.30729)",
              "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100922 Firefox/4.0.1",
              "Mozilla/5.0 (Windows; U; Windows NT 5.2; rv:1.9.2) Gecko/20100101 Firefox/3.6",
              "Mozilla/5.0 (X11; U; SunOS sun4v; en-US; rv:1.8.1.3) Gecko/20070321 Firefox/2.0.0.3",
              "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.514.0 Safari/534.7",
              "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13",
              "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8",
              "Opera/9.99 (Windows NT 5.1; U; pl) Presto/9.9.9",
              'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET4.0C; .NET4.0E;',
          ]
      useragents['Mobile Web Browsers'] = \
          [
              'Mozilla/5.0 (Linux; Android 4.4; Nexus 7 Build/KOT24) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.105 Safari/537.36',
              'Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16',
              'Nokia7650/1.0 Symbian-QP/6.1 Nokia/2.1',
          ]
      useragents['Crawlers and bots'] = \
          [
              "Googlebot/2.1 (+http://www.google.com/bot.html)",
              "Googlebot-Image/1.0",
              "Mediapartners-Google",
              "Mozilla/2.0 (compatible; Ask Jeeves)",
              "msnbot-Products/1.0 (+http://search.msn.com/msnbot.htm)",
              "mmcrawler",
              "TrackBack/1.02",
          ]
      useragents['Devices and non-browsers'] = \
          [
              "Windows-Media-Player/9.00.00.4503",
              "Mozilla/5.0 (PLAYSTATION 3; 2.00)",
              "wispr",
              "Wget 1.9cvs-stable",
              "Lynx (textmode)",
              'Mozilla/5.0',
          ]
      useragents['Attack strings'] = \
          [
              "<script>alert('123')</script>",
              "'",
              "' or 22=22'--",
              "%0d%0a",
              "../../../../../../etc/passwd",
              "../../../../../boot.ini",
              "Mozilla/4.75 (Nikto/2.01)",
              "curl/7.7.2 (powerpc-apple-darwin6.0) libcurl 7.7.2 (OpenSSL 0.9.6b)",
              "w3af.sourceforge.net",
              "HTTrack",
              ".nasl",
              "paros",
              "webinspect",
              "brutus",
              "java",
          ]
  
      def __init__(self, ):
          self.menuitems = dict()
          self.custom_file_text = 'Load custom user-agent strings from file ...'
          self.generate_menu_items()
  
      def generate_menu_items(self):
          all_tests = 'All tests'
          if all_tests not in self.useragents.keys():
              _tmparray = list()
              for v in self.useragents.values():
                  _tmparray.extend(v)
              self.useragents[all_tests] = _tmparray
          if self.custom_file_text not in self.useragents.keys():
              self.useragents[self.custom_file_text] = []
          for k in self.useragents.keys():
              menuitem = JMenuItem(k, actionPerformed=self.menuItemClicked)
              self.menuitems[menuitem] = k
  
  
      def registerExtenderCallbacks(self, callbacks):
          self.menuitems = dict()
          self._callbacks = callbacks
          self._helpers = callbacks.getHelpers()
          callbacks.setExtensionName(EXTENSION_NAME)
          callbacks.registerContextMenuFactory(self)
          self._contextMenuData = None
          self._stdout = PrintWriter(callbacks.getStdout(), True)
          self.generate_menu_items()
          return
  
      def createMenuItems(self, contextMenuInvocation):
          menuItemList = ArrayList()
          self._contextMenuData = contextMenuInvocation.getSelectedMessages()
          submenu = JMenu(EXTENSION_NAME)
          for menuitem in sorted(self.menuitems):
              submenu.add(menuitem)
          menuItemList.add(submenu)
          return menuItemList
  
      def menuItemClicked(self, event):
          if self._contextMenuData is None:
              return
          menutext = event.getSource().getText()
          if menutext == self.custom_file_text:
              self.get_custom_headers_file(event)
          useragents = self.useragents[menutext]
          if len(useragents) == 0:
              return
          if DEBUG:
              self.run_test(self._contextMenuData, useragents)
          else:
              t = threading.Thread(target=self.run_test, args=[self._contextMenuData, useragents])
              t.daemon = True
              t.start()
  
      def train(self, httpService, requestBytes):
          httpRequestResponse = self._callbacks.makeHttpRequest(httpService, requestBytes)
          httpResponseBytes = self._helpers.bytesToString(httpRequestResponse.getResponse())
          bodyOffset = self._helpers.analyzeRequest(httpResponseBytes).getBodyOffset()
          httpResponseBody = self._helpers.bytesToString(httpResponseBytes[bodyOffset:])
          self.similaritytest_headers.train(self.get_clean_headers(httpResponseBytes))
          self.similaritytest_upperbody.train(httpResponseBody)
          self.similaritytest_lowerbody.train(httpResponseBody)
  
      def initialise(self, baseRequestResponse):
          httpResponseBytes = self._helpers.bytesToString(baseRequestResponse.getResponse())
          bodyOffset = self._helpers.analyzeRequest(httpResponseBytes).getBodyOffset()
          httpResponseBody = self._helpers.bytesToString(httpResponseBytes[bodyOffset:])
          self.similaritytest_headers = Similarity(self.get_clean_headers(httpResponseBytes))
          self.similaritytest_upperbody = Similarity(httpResponseBody)
          self.similaritytest_lowerbody = Similarity(httpResponseBody, lower=True)
  
      def get_clean_headers_list(self, httpResponseBytes, banned_headers):
          httpResponseInfo = self._helpers.analyzeResponse(httpResponseBytes)
          httpResponseHeaders = httpResponseInfo.getHeaders()
          newHttpResponseHeaders = list()
          for hdr in httpResponseHeaders:
              _tmp = map(lambda x: x.strip().lower(), hdr.split(':', 1))
              if len(_tmp) == 2:
                  k, v = _tmp
                  if not k in banned_headers:
                      newHttpResponseHeaders.append(hdr)
              else:
                  newHttpResponseHeaders.append(hdr)
          return newHttpResponseHeaders
  
      def get_clean_headers(self, httpResponseBytes, banned_headers=banned_headers):
          newHttpResponseHeaders = self.get_clean_headers_list(httpResponseBytes, banned_headers)
          httpResponseInfo = self._helpers.analyzeResponse(httpResponseBytes)
          bodyOffset = httpResponseInfo.getBodyOffset()
          httpResponseBody = httpResponseBytes[bodyOffset:]
          newhttpResponseBytes = self._helpers.buildHttpMessage(newHttpResponseHeaders, httpResponseBody)[:bodyOffset]
          return newhttpResponseBytes
  
      def choose_file(self, event):
          chooseFile = JFileChooser()
          chooseFile.showOpenDialog(None)
          chosenFile = chooseFile.getSelectedFile()
          return str(chosenFile)
  
      def get_custom_headers_file(self, event):
          self.useragents[self.custom_file_text] = list()
          customheadersfn = self.choose_file(event)
          if customheadersfn is None or customheadersfn == 'None':
              return
          with open(customheadersfn,'r') as customheadersf:
              for line in customheadersf:
                  self.useragents[self.custom_file_text].append(line.strip())
  
      def run_test(self, _contextMenuData, useragents):
          for baseRequestResponse in _contextMenuData:
              httpService = baseRequestResponse.getHttpService()
              requestBytes = baseRequestResponse.getRequest()
              targeturl = self._helpers.analyzeRequest(baseRequestResponse).getUrl()
              self._stdout.println('Target request for this URL: ' + targeturl.toString())
              if baseRequestResponse.getResponse() == None:
                  self._stdout.println('No response for this request .. issuing a new request')
                  baseRequestResponse = self._callbacks.makeHttpRequest(httpService, requestBytes)
              if baseRequestResponse.getResponse() == None:
                  self._stdout.println('Did you get a response')
                  continue
              # first we do a test to see how stable the responses are
              # make a first request to compare to
              self.initialise(baseRequestResponse)
              # and we check if the response is totally way off from the one previously recorded
              # if it is then we have to issue a new request - might be an expired session
              self.train(httpService, requestBytes)
              if (self.similaritytest_upperbody.norm < 0.6) and \
                      (self.similaritytest_lowerbody.norm < 0.6) and \
                      (self.similaritytest_headers.norm < 0.6):
                  self._stdout.println('The previous request appears to grown fungii.. refreshing')
                  baseRequestResponse = self._callbacks.makeHttpRequest(httpService, requestBytes)
                  self.initialise(baseRequestResponse)
              # run the same request and analyse the responses to get a standard deviation
              # using the train() function which finds the lowest ratio
              self._stdout.println('training ...')
              for _ in range(10):
                  self.train(httpService, requestBytes)
              # reduce norm by 0.05 to account for future changes
              for obj in [self.similaritytest_upperbody, self.similaritytest_lowerbody, self.similaritytest_headers]:
                  obj.norm -= 0.05
              # now we check what is the lowest ratio number and use that as a normality tester
              if (self.similaritytest_upperbody.norm < 0.6) and (self.similaritytest_lowerbody.norm < 0.6) and (
                  self.similaritytest_headers.norm < 0.6):
                  self._stdout.println('The site does not appear stable enough for our tests')
                  continue
              dotest = list()
              if (self.similaritytest_lowerbody.norm >= 0.6):
                  dotest.append('lowerbody')
              if (self.similaritytest_upperbody.norm >= 0.6):
                  dotest.append('upperbody')
              if (self.similaritytest_headers.norm >= 0.6):
                  dotest.append('headers')
              self._stdout.println("Norm for headers: " + str(self.similaritytest_headers.norm))
              self._stdout.println("Norm for upper body: " + str(self.similaritytest_upperbody.norm))
              self._stdout.println("Norm for lower body: " + str(self.similaritytest_lowerbody.norm))
              requestInfo = self._helpers.analyzeRequest(requestBytes)
              origRequestBody = requestBytes[requestInfo.getBodyOffset():]
  
              # remove the user-agent header so that we can add our own later
              newRequestHeaders = self.get_clean_headers_list(requestBytes, banned_headers=['user-agent'])
  
              # get all reported issues for the target URL so that we later check if the issue is a dupe
              scanIssues = self._callbacks.getScanIssues(None)
              for ua in useragents:
                  if len(ua) > 25:
                      issueName = "User-Agent dependent response (%s...%s)" % (ua[:30], ua[-12:])
                  else:
                      issueName = "User-Agent dependent response (%s)" % (ua,)
                  issueDetail = "Web location responds differently when User-Agent header is set to \"%s\"." % ua
                  alreadyscanned = False
                  for scanIssue in scanIssues:
                      if (scanIssue.getIssueDetail() == issueDetail) and (scanIssue.getUrl() == targeturl):
                          alreadyscanned = True
                          continue
                  if alreadyscanned:
                      self._stdout.println("User-agent: %s is already an issue .. skipping" % ua)
                      continue
                  # add the new user agent                
                  _newRequestHeaders = newRequestHeaders[:]
                  _newRequestHeaders.append(u'User-Agent: ' + ua)
                  # build and send http request
                  httpMessage = self._helpers.buildHttpMessage(_newRequestHeaders, origRequestBody)
                  httpRequestResponse = self._callbacks.makeHttpRequest(httpService, httpMessage)
                  # get the response
                  httpResponseBytes = self._helpers.bytesToString(httpRequestResponse.getResponse())
                  # split up the head from the body
                  bodyOffset = self._helpers.analyzeRequest(httpResponseBytes).getBodyOffset()
                  httpResponseBody = self._helpers.bytesToString(httpResponseBytes[bodyOffset:])
                  change_hdr = None
                  change_upperbody = None
                  change_lowerbody = None
                  # only test stable ones
                  if 'headers' in dotest:
                      change_hdr = self.similaritytest_headers.is_different(self.get_clean_headers(httpResponseBytes))
                  if 'upperbody' in dotest:
                      change_upperbody = self.similaritytest_upperbody.is_different(httpResponseBody)
                  if 'lowerbody' in dotest:
                      change_lowerbody = self.similaritytest_lowerbody.is_different(httpResponseBody)
                  if change_lowerbody or change_upperbody or change_hdr:
                      self._stdout.println(
                          "%s - headers: %s upperbody: %s lowerbody: %s" \
                          % (ua, change_hdr, change_upperbody, change_lowerbody))
                      httpmsgs = [baseRequestResponse, httpRequestResponse]
                      issue = ScanIssue(httpService,
                                        self._helpers.analyzeRequest(httpRequestResponse).getUrl(),
                                        httpmsgs,
                                        issueName,
                                        issueDetail,
                                        "Information",
                                        "Firm",
                                        None,
                                        None,
                                        None,
                      )
                      self._callbacks.addScanIssue(issue)
                  else:
                      self._stdout.println('No changes detected with "%s"' % ua)
          self._stdout.println('Done')
  
  
  class ScanIssue(IScanIssue):
      """This is our custom IScanIssue class implementation."""
  
      def __init__(self, httpService, url, httpMessages, issueName, issueDetail, severity, confidence, issueBackground,
                   remediationDetail, remediationBackground):
          self._issueName = issueName
          self._httpService = httpService
          self._url = url
          self._httpMessages = httpMessages
          self._issueDetail = issueDetail
          self._severity = severity
          self._confidence = confidence
          self._remediationDetail = remediationDetail
          self._issueBackground = issueBackground
          self._remediationBackground = remediationBackground
  
      def getConfidence(self):
          return self._confidence
  
      def getHttpMessages(self):
          return self._httpMessages
  
      def getHttpService(self):
          return self._httpService
  
      def getIssueBackground(self):
          return self._issueBackground
  
      def getIssueDetail(self):
          return self._issueDetail
  
      def getIssueName(self):
          return self._issueName
  
      def getIssueType(self):
          return 0
  
      def getRemediationBackground(self):
          return self._remediationBackground
  
      def getRemediationDetail(self):
          return self._remediationDetail
  
      def getSeverity(self):
          return self._severity
  
      def getUrl(self):
          return self._url
  
      def getHost(self):
          return 'localhost'
  
      def getPort(self):
          return int(80)