parsedmarc/elastic.py
def save_forensic_report_to_elasticsearch(forensic_report,
                                          index_suffix=None,
                                          monthly_indexes=False):
    """
        Saves a parsed DMARC forensic report to ElasticSearch

        Args:
            forensic_report (OrderedDict): A parsed forensic report
            index_suffix (str): The suffix of the name of the index to save to
            monthly_indexes (bool): Use monthly indexes instead of daily
                                    indexes

        Raises:
            AlreadySaved

        """
    logger.debug("Saving forensic report to Elasticsearch")
    forensic_report = forensic_report.copy()
    sample_date = None
    if forensic_report["parsed_sample"]["date"] is not None:
        sample_date = forensic_report["parsed_sample"]["date"]
        sample_date = human_timestamp_to_datetime(sample_date)
    original_headers = forensic_report["parsed_sample"]["headers"]
    headers = OrderedDict()
    for original_header in original_headers:
        headers[original_header.lower()] = original_headers[original_header]

    arrival_date_human = forensic_report["arrival_date_utc"]
    arrival_date = human_timestamp_to_datetime(arrival_date_human)

    search = Search(index="dmarc_forensic*")
    arrival_query = {"match": {"arrival_date": arrival_date}}
    q = Q(arrival_query)

    from_ = None
    to_ = None
    subject = None
    if "from" in headers:
        from_ = headers["from"]
        from_query = {"match": {"sample.headers.from": from_}}
        q = q & Q(from_query)
    if "to" in headers:
        to_ = headers["to"]
        to_query = {"match": {"sample.headers.to": to_}}
        q = q & Q(to_query)
    if "subject" in headers:
        subject = headers["subject"]
        subject_query = {"match": {"sample.headers.subject": subject}}
        q = q & Q(subject_query)

    search.query = q
    existing = search.execute()

    if len(existing) > 0:
        raise AlreadySaved("A forensic sample to {0} from {1} "
                           "with a subject of {2} and arrival date of {3} "
                           "already exists in "
                           "Elasticsearch".format(to_,
                                                  from_,
                                                  subject,
                                                  arrival_date_human
                                                  ))

    parsed_sample = forensic_report["parsed_sample"]
    sample = _ForensicSampleDoc(
        raw=forensic_report["sample"],
        headers=headers,
        headers_only=forensic_report["sample_headers_only"],
        date=sample_date,
        subject=forensic_report["parsed_sample"]["subject"],
        filename_safe_subject=parsed_sample["filename_safe_subject"],
        body=forensic_report["parsed_sample"]["body"]
    )

    for address in forensic_report["parsed_sample"]["to"]:
        sample.add_to(display_name=address["display_name"],
                      address=address["address"])
    for address in forensic_report["parsed_sample"]["reply_to"]:
        sample.add_reply_to(display_name=address["display_name"],
                            address=address["address"])
    for address in forensic_report["parsed_sample"]["cc"]:
        sample.add_cc(display_name=address["display_name"],
                      address=address["address"])
    for address in forensic_report["parsed_sample"]["bcc"]:
        sample.add_bcc(display_name=address["display_name"],
                       address=address["address"])
    for attachment in forensic_report["parsed_sample"]["attachments"]:
        sample.add_attachment(filename=attachment["filename"],
                              content_type=attachment["mail_content_type"],
                              sha256=attachment["sha256"])
    try:
        forensic_doc = _ForensicReportDoc(
            feedback_type=forensic_report["feedback_type"],
            user_agent=forensic_report["user_agent"],
            version=forensic_report["version"],
            original_mail_from=forensic_report["original_mail_from"],
            arrival_date=arrival_date,
            domain=forensic_report["reported_domain"],
            original_envelope_id=forensic_report["original_envelope_id"],
            authentication_results=forensic_report["authentication_results"],
            delivery_results=forensic_report["delivery_result"],
            source_ip_address=forensic_report["source"]["ip_address"],
            source_country=forensic_report["source"]["country"],
            source_reverse_dns=forensic_report["source"]["reverse_dns"],
            source_base_domain=forensic_report["source"]["base_domain"],
            authentication_mechanisms=forensic_report[
                "authentication_mechanisms"],
            auth_failure=forensic_report["auth_failure"],
            dkim_domain=forensic_report["dkim_domain"],
            original_rcpt_to=forensic_report["original_rcpt_to"],
            sample=sample
        )

        index = "dmarc_forensic"
        if index_suffix:
            index = "{0}_{1}".format(index, index_suffix)
        if monthly_indexes:
            index_date = arrival_date.strftime("%Y-%m")
        else:
            index_date = arrival_date.strftime("%Y-%m-%d")
        index = "{0}-{1}".format(index, index_date)
        create_indexes([index])
        forensic_doc.meta.index = index
        try:
            forensic_doc.save()
        except Exception as e:
            raise ElasticsearchError(
                "Elasticsearch error: {0}".format(e.__str__()))
    except KeyError as e:
        raise InvalidForensicReport(
            "Forensic report missing required field: {0}".format(e.__str__()))
def save_forensic_report_to_elasticsearch(forensic_report,
                                          index_suffix=None,
                                          monthly_indexes=False):
    """
        Saves a parsed DMARC forensic report to ElasticSearch

        Args:
            forensic_report (OrderedDict): A parsed forensic report
            index_suffix (str): The suffix of the name of the index to save to
            monthly_indexes (bool): Use monthly indexes instead of daily
                                    indexes

        Raises:
            AlreadySaved

        """
    logger.debug("Saving forensic report to Elasticsearch")
    forensic_report = forensic_report.copy()
    sample_date = None
    if forensic_report["parsed_sample"]["date"] is not None:
        sample_date = forensic_report["parsed_sample"]["date"]
        sample_date = human_timestamp_to_datetime(sample_date)
    original_headers = forensic_report["parsed_sample"]["headers"]
    headers = OrderedDict()
    for original_header in original_headers:
        headers[original_header.lower()] = original_headers[original_header]

    arrival_date_human = forensic_report["arrival_date_utc"]
    arrival_date = human_timestamp_to_datetime(arrival_date_human)

    search = Search(index="dmarc_forensic*")
    arrival_query = {"match": {"arrival_date": arrival_date}}
    q = Q(arrival_query)

    from_ = None
    to_ = None
    subject = None
    if "from" in headers:
        from_ = headers["from"]
        from_query = {"match": {"sample.headers.from": from_}}
        q = q & Q(from_query)
    if "to" in headers:
        to_ = headers["to"]
        to_query = {"match": {"sample.headers.to": to_}}
        q = q & Q(to_query)
    if "subject" in headers:
        subject = headers["subject"]
        subject_query = {"match": {"sample.headers.subject": subject}}
        q = q & Q(subject_query)

    search.query = q
    existing = search.execute()

    if len(existing) > 0:
        raise AlreadySaved("A forensic sample to {0} from {1} "
                           "with a subject of {2} and arrival date of {3} "
                           "already exists in "
                           "Elasticsearch".format(to_,
                                                  from_,
                                                  subject,
                                                  arrival_date_human
                                                  ))

    parsed_sample = forensic_report["parsed_sample"]
    sample = _ForensicSampleDoc(
        raw=forensic_report["sample"],
        headers=headers,
        headers_only=forensic_report["sample_headers_only"],
        date=sample_date,
        subject=forensic_report["parsed_sample"]["subject"],
        filename_safe_subject=parsed_sample["filename_safe_subject"],
        body=forensic_report["parsed_sample"]["body"]
    )

    for address in forensic_report["parsed_sample"]["to"]:
        sample.add_to(display_name=address["display_name"],
                      address=address["address"])
    for address in forensic_report["parsed_sample"]["reply_to"]:
        sample.add_reply_to(display_name=address["display_name"],
                            address=address["address"])
    for address in forensic_report["parsed_sample"]["cc"]:
        sample.add_cc(display_name=address["display_name"],
                      address=address["address"])
    for address in forensic_report["parsed_sample"]["bcc"]:
        sample.add_bcc(display_name=address["display_name"],
                       address=address["address"])
    for attachment in forensic_report["parsed_sample"]["attachments"]:
        sample.add_attachment(filename=attachment["filename"],
                              content_type=attachment["mail_content_type"],
                              sha256=attachment["sha256"])
    try:
        forensic_doc = _ForensicReportDoc(
            feedback_type=forensic_report["feedback_type"],
            user_agent=forensic_report["user_agent"],
            version=forensic_report["version"],
            original_mail_from=forensic_report["original_mail_from"],
            arrival_date=arrival_date,
            domain=forensic_report["reported_domain"],
            original_envelope_id=forensic_report["original_envelope_id"],
            authentication_results=forensic_report["authentication_results"],
            delivery_results=forensic_report["delivery_result"],
            source_ip_address=forensic_report["source"]["ip_address"],
            source_country=forensic_report["source"]["country"],
            source_reverse_dns=forensic_report["source"]["reverse_dns"],
            source_base_domain=forensic_report["source"]["base_domain"],
            authentication_mechanisms=forensic_report[
                "authentication_mechanisms"],
            auth_failure=forensic_report["auth_failure"],
            dkim_domain=forensic_report["dkim_domain"],
            original_rcpt_to=forensic_report["original_rcpt_to"],
            sample=sample
        )

        index = "dmarc_forensic"
        if index_suffix:
            index = "{0}_{1}".format(index, index_suffix)
        if monthly_indexes:
            index_date = arrival_date.strftime("%Y-%m")
        else:
            index_date = arrival_date.strftime("%Y-%m-%d")
        index = "{0}-{1}".format(index, index_date)
        create_indexes([index])
        forensic_doc.meta.index = index
        try:
            forensic_doc.save()
        except Exception as e:
            raise ElasticsearchError(
                "Elasticsearch error: {0}".format(e.__str__()))
    except KeyError as e:
        raise InvalidForensicReport(
            "Forensic report missing required field: {0}".format(e.__str__()))
def save_forensic_report_to_elasticsearch ( forensic_report , index_suffix = None , monthly_indexes = False ) : logger . debug ( "Saving forensic report to Elasticsearch" ) forensic_report = forensic_report . copy ( ) sample_date = None if forensic_report [ "parsed_sample" ] [ "date" ] is not None : sample_date = forensic_report [ "parsed_sample" ] [ "date" ] sample_date = human_timestamp_to_datetime ( sample_date ) original_headers = forensic_report [ "parsed_sample" ] [ "headers" ] headers = OrderedDict ( ) for original_header in original_headers : headers [ original_header . lower ( ) ] = original_headers [ original_header ] arrival_date_human = forensic_report [ "arrival_date_utc" ] arrival_date = human_timestamp_to_datetime ( arrival_date_human ) search = Search ( index = "dmarc_forensic*" ) arrival_query = { "match" : { "arrival_date" : arrival_date } } q = Q ( arrival_query ) from_ = None to_ = None subject = None if "from" in headers : from_ = headers [ "from" ] from_query = { "match" : { "sample.headers.from" : from_ } } q = q & Q ( from_query ) if "to" in headers : to_ = headers [ "to" ] to_query = { "match" : { "sample.headers.to" : to_ } } q = q & Q ( to_query ) if "subject" in headers : subject = headers [ "subject" ] subject_query = { "match" : { "sample.headers.subject" : subject } } q = q & Q ( subject_query ) search . query = q existing = search . execute ( ) if len ( existing ) > 0 : raise AlreadySaved ( "A forensic sample to {0} from {1} " "with a subject of {2} and arrival date of {3} " "already exists in " "Elasticsearch" . format ( to_ , from_ , subject , arrival_date_human ) ) parsed_sample = forensic_report [ "parsed_sample" ] sample = _ForensicSampleDoc ( raw = forensic_report [ "sample" ] , headers = headers , headers_only = forensic_report [ "sample_headers_only" ] , date = sample_date , subject = forensic_report [ "parsed_sample" ] [ "subject" ] , filename_safe_subject = parsed_sample [ "filename_safe_subject" ] , body = forensic_report [ "parsed_sample" ] [ "body" ] ) for address in forensic_report [ "parsed_sample" ] [ "to" ] : sample . add_to ( display_name = address [ "display_name" ] , address = address [ "address" ] ) for address in forensic_report [ "parsed_sample" ] [ "reply_to" ] : sample . add_reply_to ( display_name = address [ "display_name" ] , address = address [ "address" ] ) for address in forensic_report [ "parsed_sample" ] [ "cc" ] : sample . add_cc ( display_name = address [ "display_name" ] , address = address [ "address" ] ) for address in forensic_report [ "parsed_sample" ] [ "bcc" ] : sample . add_bcc ( display_name = address [ "display_name" ] , address = address [ "address" ] ) for attachment in forensic_report [ "parsed_sample" ] [ "attachments" ] : sample . add_attachment ( filename = attachment [ "filename" ] , content_type = attachment [ "mail_content_type" ] , sha256 = attachment [ "sha256" ] ) try : forensic_doc = _ForensicReportDoc ( feedback_type = forensic_report [ "feedback_type" ] , user_agent = forensic_report [ "user_agent" ] , version = forensic_report [ "version" ] , original_mail_from = forensic_report [ "original_mail_from" ] , arrival_date = arrival_date , domain = forensic_report [ "reported_domain" ] , original_envelope_id = forensic_report [ "original_envelope_id" ] , authentication_results = forensic_report [ "authentication_results" ] , delivery_results = forensic_report [ "delivery_result" ] , source_ip_address = forensic_report [ "source" ] [ "ip_address" ] , source_country = forensic_report [ "source" ] [ "country" ] , source_reverse_dns = forensic_report [ "source" ] [ "reverse_dns" ] , source_base_domain = forensic_report [ "source" ] [ "base_domain" ] , authentication_mechanisms = forensic_report [ "authentication_mechanisms" ] , auth_failure = forensic_report [ "auth_failure" ] , dkim_domain = forensic_report [ "dkim_domain" ] , original_rcpt_to = forensic_report [ "original_rcpt_to" ] , sample = sample ) index = "dmarc_forensic" if index_suffix : index = "{0}_{1}" . format ( index , index_suffix ) if monthly_indexes : index_date = arrival_date . strftime ( "%Y-%m" ) else : index_date = arrival_date . strftime ( "%Y-%m-%d" ) index = "{0}-{1}" . format ( index , index_date ) create_indexes ( [ index ] ) forensic_doc . meta . index = index try : forensic_doc . save ( ) except Exception as e : raise ElasticsearchError ( "Elasticsearch error: {0}" . format ( e . __str__ ( ) ) ) except KeyError as e : raise InvalidForensicReport ( "Forensic report missing required field: {0}" . format ( e . __str__ ( ) ) )
Saves a parsed DMARC forensic report to ElasticSearch

        Args:
            forensic_report (OrderedDict): A parsed forensic report
            index_suffix (str): The suffix of the name of the index to save to
            monthly_indexes (bool): Use monthly indexes instead of daily
                                    indexes

        Raises:
            AlreadySaved
Saves a parsed DMARC forensic report to ElasticSearch