grr/server/grr_response_server/databases/mysql_flows.py
def WriteFlowResponses(self, responses, cursor=None):
    """Writes a list of flow responses to the database."""

    if not responses:
      return

    # In addition to just writing responses, this function needs to also

    # - Update the expected nr of response for each request that received a
    #   status.
    # - Set the needs_processing flag for all requests that now have all
    #   responses.
    # - Send FlowProcessingRequests for all flows that are waiting on a request
    #   whose needs_processing flag was just set.

    # To achieve this, we need to get the next request each flow is waiting for.
    next_request_by_flow = {}

    # And the number of responses each affected request is waiting for (if
    # available).
    responses_expected_by_request = {}

    # As well as the ids of the currently available responses for each request.
    current_responses_by_request = {}

    # We also store all requests we have in the db so we can discard responses
    # for unknown requests right away.
    currently_available_requests = set()

    self._ReadCurrentFlowInfo(
        responses, currently_available_requests, next_request_by_flow,
        responses_expected_by_request, current_responses_by_request, cursor)

    # For some requests we will need to update the number of expected responses.
    needs_expected_update = {}

    # For some we will need to update the needs_processing flag.
    needs_processing_update = set()

    # Some completed requests will trigger a flow processing request, we collect
    # them in:
    flow_processing_requests = []

    task_ids_by_request = {}

    for r in responses:
      request_key = (r.client_id, r.flow_id, r.request_id)

      try:
        # If this is a response coming from a client, a task_id will be set. We
        # store it in case the request is complete and we can remove the client
        # messages.
        task_ids_by_request[request_key] = r.task_id
      except AttributeError:
        pass

      if not isinstance(r, rdf_flow_objects.FlowStatus):
        continue

      current = responses_expected_by_request.get(request_key)
      if current:
        logging.error("Got duplicate status message for request %s/%s/%d",
                      r.client_id, r.flow_id, r.request_id)
        # If there is already responses_expected information, we need to make
        # sure the current status doesn't disagree.
        if current != r.response_id:
          raise ValueError(
              "Got conflicting status information for request %s: %s" %
              (request_key, r))
      else:
        needs_expected_update[request_key] = r.response_id

      responses_expected_by_request[request_key] = r.response_id

    responses_to_write = []
    client_messages_to_delete = []
    for r in responses:
      request_key = (r.client_id, r.flow_id, r.request_id)

      if request_key not in currently_available_requests:
        logging.info("Dropping response for unknown request %s/%s/%d",
                     r.client_id, r.flow_id, r.request_id)
        continue

      responses_to_write.append(r)

      current_responses = current_responses_by_request.setdefault(
          request_key, set())
      if r.response_id in current_responses:
        # We have this response already, nothing further to do.
        continue

      current_responses.add(r.response_id)
      expected_responses = responses_expected_by_request.get(request_key, 0)
      if len(current_responses) == expected_responses:
        # This response was the one that was missing, time to set the
        # needs_processing flag.
        needs_processing_update.add(request_key)
        if r.request_id == next_request_by_flow[(r.client_id, r.flow_id)]:
          # The request that is now ready for processing was also the one the
          # flow was waiting for.
          req = rdf_flows.FlowProcessingRequest(
              client_id=r.client_id, flow_id=r.flow_id)
          flow_processing_requests.append(req)

        # Since this request is now complete, we can remove the corresponding
        # client messages if there are any.
        task_id = task_ids_by_request.get(request_key, None)
        if task_id is not None:
          client_messages_to_delete.append((r.client_id, task_id))

    now = rdfvalue.RDFDatetime.Now()
    now_str = mysql_utils.RDFDatetimeToMysqlString(now)

    if responses_to_write:
      self._WriteResponses(responses_to_write, now_str, cursor)

    self._UpdateRequests(needs_processing_update, needs_expected_update, cursor)

    if client_messages_to_delete:
      self._DeleteClientMessages(client_messages_to_delete, cursor)

    if flow_processing_requests:
      self._WriteFlowProcessingRequests(flow_processing_requests, cursor)
Writes a list of flow responses to the database.
Writes a list of flow responses to the database.
Writes a list of flow responses to the database .
def WriteFlowResponses(self, responses, cursor=None):
    if not responses:
      return
    next_request_by_flow = {}
    responses_expected_by_request = {}
    current_responses_by_request = {}
    currently_available_requests = set()
    self._ReadCurrentFlowInfo(
        responses, currently_available_requests, next_request_by_flow,
        responses_expected_by_request, current_responses_by_request, cursor)
    needs_expected_update = {}
    needs_processing_update = set()
    flow_processing_requests = []
    task_ids_by_request = {}
    for r in responses:
      request_key = (r.client_id, r.flow_id, r.request_id)
      try:
        task_ids_by_request[request_key] = r.task_id
      except AttributeError:
        pass
      if not isinstance(r, rdf_flow_objects.FlowStatus):
        continue
      current = responses_expected_by_request.get(request_key)
      if current:
        logging.error("Got duplicate status message for request %s/%s/%d",
                      r.client_id, r.flow_id, r.request_id)
        if current != r.response_id:
          raise ValueError(
              "Got conflicting status information for request %s: %s" %
              (request_key, r))
      else:
        needs_expected_update[request_key] = r.response_id
      responses_expected_by_request[request_key] = r.response_id
    responses_to_write = []
    client_messages_to_delete = []
    for r in responses:
      request_key = (r.client_id, r.flow_id, r.request_id)
      if request_key not in currently_available_requests:
        logging.info("Dropping response for unknown request %s/%s/%d",
                     r.client_id, r.flow_id, r.request_id)
        continue
      responses_to_write.append(r)
      current_responses = current_responses_by_request.setdefault(
          request_key, set())
      if r.response_id in current_responses:
        continue
      current_responses.add(r.response_id)
      expected_responses = responses_expected_by_request.get(request_key, 0)
      if len(current_responses) == expected_responses:
        needs_processing_update.add(request_key)
        if r.request_id == next_request_by_flow[(r.client_id, r.flow_id)]:
          req = rdf_flows.FlowProcessingRequest(
              client_id=r.client_id, flow_id=r.flow_id)
          flow_processing_requests.append(req)
        task_id = task_ids_by_request.get(request_key, None)
        if task_id is not None:
          client_messages_to_delete.append((r.client_id, task_id))
    now = rdfvalue.RDFDatetime.Now()
    now_str = mysql_utils.RDFDatetimeToMysqlString(now)
    if responses_to_write:
      self._WriteResponses(responses_to_write, now_str, cursor)
    self._UpdateRequests(needs_processing_update, needs_expected_update, cursor)
    if client_messages_to_delete:
      self._DeleteClientMessages(client_messages_to_delete, cursor)
    if flow_processing_requests:
      self._WriteFlowProcessingRequests(flow_processing_requests, cursor)
def WriteFlowResponses ( self , responses , cursor = None ) : if not responses : return next_request_by_flow = { } responses_expected_by_request = { } current_responses_by_request = { } currently_available_requests = set ( ) self . _ReadCurrentFlowInfo ( responses , currently_available_requests , next_request_by_flow , responses_expected_by_request , current_responses_by_request , cursor ) needs_expected_update = { } needs_processing_update = set ( ) flow_processing_requests = [ ] task_ids_by_request = { } for r in responses : request_key = ( r . client_id , r . flow_id , r . request_id ) try : task_ids_by_request [ request_key ] = r . task_id except AttributeError : pass if not isinstance ( r , rdf_flow_objects . FlowStatus ) : continue current = responses_expected_by_request . get ( request_key ) if current : logging . error ( "Got duplicate status message for request %s/%s/%d" , r . client_id , r . flow_id , r . request_id ) if current != r . response_id : raise ValueError ( "Got conflicting status information for request %s: %s" % ( request_key , r ) ) else : needs_expected_update [ request_key ] = r . response_id responses_expected_by_request [ request_key ] = r . response_id responses_to_write = [ ] client_messages_to_delete = [ ] for r in responses : request_key = ( r . client_id , r . flow_id , r . request_id ) if request_key not in currently_available_requests : logging . info ( "Dropping response for unknown request %s/%s/%d" , r . client_id , r . flow_id , r . request_id ) continue responses_to_write . append ( r ) current_responses = current_responses_by_request . setdefault ( request_key , set ( ) ) if r . response_id in current_responses : continue current_responses . add ( r . response_id ) expected_responses = responses_expected_by_request . get ( request_key , 0 ) if len ( current_responses ) == expected_responses : needs_processing_update . add ( request_key ) if r . request_id == next_request_by_flow [ ( r . client_id , r . flow_id ) ] : req = rdf_flows . FlowProcessingRequest ( client_id = r . client_id , flow_id = r . flow_id ) flow_processing_requests . append ( req ) task_id = task_ids_by_request . get ( request_key , None ) if task_id is not None : client_messages_to_delete . append ( ( r . client_id , task_id ) ) now = rdfvalue . RDFDatetime . Now ( ) now_str = mysql_utils . RDFDatetimeToMysqlString ( now ) if responses_to_write : self . _WriteResponses ( responses_to_write , now_str , cursor ) self . _UpdateRequests ( needs_processing_update , needs_expected_update , cursor ) if client_messages_to_delete : self . _DeleteClientMessages ( client_messages_to_delete , cursor ) if flow_processing_requests : self . _WriteFlowProcessingRequests ( flow_processing_requests , cursor )
Writes a list of flow responses to the database.
Writes a list of flow responses to the database .
"""Writes a list of flow responses to the database.""" # In addition to just writing responses, this function needs to also # - Update the expected nr of response for each request that received a #   status. # - Set the needs_processing flag for all requests that now have all #   responses. # - Send FlowProcessingRequests for all flows that are waiting on a request #   whose needs_processing flag was just set. # To achieve this, we need to get the next request each flow is waiting for. # And the number of responses each affected request is waiting for (if # available). # As well as the ids of the currently available responses for each request. # We also store all requests we have in the db so we can discard responses # for unknown requests right away. # For some requests we will need to update the number of expected responses. # For some we will need to update the needs_processing flag. # Some completed requests will trigger a flow processing request, we collect # them in: # If this is a response coming from a client, a task_id will be set. We # store it in case the request is complete and we can remove the client # messages. # If there is already responses_expected information, we need to make # sure the current status doesn't disagree. # We have this response already, nothing further to do. # This response was the one that was missing, time to set the # needs_processing flag. # The request that is now ready for processing was also the one the # flow was waiting for. # Since this request is now complete, we can remove the corresponding # client messages if there are any.
self
responses
cursor