parsedmarc/__init__.py
def get_dmarc_reports_from_inbox(host=None,
                                 user=None,
                                 password=None,
                                 connection=None,
                                 port=None,
                                 ssl=True,
                                 ssl_context=None,
                                 move_supported=None,
                                 reports_folder="INBOX",
                                 archive_folder="Archive",
                                 delete=False,
                                 test=False,
                                 nameservers=None,
                                 dns_timeout=6.0,
                                 strip_attachment_payloads=False,
                                 results=None):
    """
    Fetches and parses DMARC reports from sn inbox

    Args:
        host: The mail server hostname or IP address
        user: The mail server user
        password: The mail server password
        connection: An IMAPCLient connection to reuse
        port: The mail server port
        ssl (bool): Use SSL/TLS
        ssl_context (SSLContext): A SSL context
        move_supported: Indicate if the IMAP server supports the MOVE command
        (autodetect if None)
        reports_folder: The IMAP folder where reports can be found
        archive_folder: The folder to move processed mail to
        delete (bool): Delete  messages after processing them
        test (bool): Do not move or delete messages after processing them
        nameservers (list): A list of DNS nameservers to query
        dns_timeout (float): Set the DNS query timeout
        strip_attachment_payloads (bool): Remove attachment payloads from
        forensic report results
        results (dict): Results from the previous run

    Returns:
        OrderedDict: Lists of ``aggregate_reports`` and ``forensic_reports``
    """

    def chunks(l, n):
        """Yield successive n-sized chunks from l."""
        for i in range(0, len(l), n):
            yield l[i:i + n]

    if delete and test:
        raise ValueError("delete and test options are mutually exclusive")

    if connection is None and (user is None or password is None):
        raise ValueError("Must supply a connection, or a username and "
                         "password")

    aggregate_reports = []
    forensic_reports = []
    aggregate_report_msg_uids = []
    forensic_report_msg_uids = []
    aggregate_reports_folder = "{0}/Aggregate".format(archive_folder)
    forensic_reports_folder = "{0}/Forensic".format(archive_folder)
    invalid_reports_folder = "{0}/Invalid".format(archive_folder)

    if results:
        aggregate_reports = results["aggregate_reports"].copy()
        forensic_reports = results["forensic_reports"].copy()

    try:
        if connection:
            server = connection
        else:
            if not ssl:
                logger.debug("Connecting to IMAP over plain text")
            if ssl_context is None:
                ssl_context = create_default_context()
            server = imapclient.IMAPClient(host,
                                           port=port,
                                           ssl=ssl,
                                           ssl_context=ssl_context,
                                           use_uid=True)
            server.login(user, password)

        if move_supported is None:
            server_capabilities = get_imap_capabilities(server)
            move_supported = "MOVE" in server_capabilities

        def delete_messages(msg_uids):
            logger.debug("Deleting message UID(s) {0}".format(",".join(
                str(uid) for uid in msg_uids)))
            if type(msg_uids) == str or type(msg_uids) == int:
                msg_uids = [int(msg_uids)]

            server.delete_messages(msg_uids, silent=True)
            server.expunge(msg_uids)

        def move_messages(msg_uids, folder):
            if type(msg_uids) == str or type(msg_uids) == int:
                msg_uids = [int(msg_uids)]
            for chunk in chunks(msg_uids, 100):
                if move_supported:
                    logger.debug("Moving message UID(s) {0} to {1}".format(
                        ",".join(str(uid) for uid in chunk), folder
                    ))
                    server.move(chunk, folder)
                else:
                    logger.debug("Copying message UID(s) {0} to {1}".format(
                        ",".join(str(uid) for uid in chunk), folder
                    ))
                    server.copy(msg_uids, folder)
                    delete_messages(msg_uids)

        if not server.folder_exists(archive_folder):
            logger.debug("Creating IMAP folder: {0}".format(archive_folder))
            server.create_folder(archive_folder)
        try:
            # Test subfolder creation
            if not server.folder_exists(aggregate_reports_folder):
                server.create_folder(aggregate_reports_folder)
                logger.debug(
                    "Creating IMAP folder: {0}".format(
                        aggregate_reports_folder))
        except imapclient.exceptions.IMAPClientError:
            #  Only replace / with . when . doesn't work
            # This usually indicates a dovecot IMAP server
            aggregate_reports_folder = aggregate_reports_folder.replace("/",
                                                                        ".")
            forensic_reports_folder = forensic_reports_folder.replace("/",
                                                                      ".")
            invalid_reports_folder = invalid_reports_folder.replace("/",
                                                                    ".")
        subfolders = [aggregate_reports_folder,
                      forensic_reports_folder,
                      invalid_reports_folder]

        for subfolder in subfolders:
            if not server.folder_exists(subfolder):
                logger.debug(
                    "Creating IMAP folder: {0}".format(subfolder))
                server.create_folder(subfolder)
        server.select_folder(reports_folder)
        messages = server.search()
        total_messages = len(messages)
        logger.debug("Found {0} messages in IMAP folder {1}".format(
            len(messages), reports_folder))
        for i in range(len(messages)):
            msg_uid = messages[i]
            logger.debug("Processing message {0} of {1}: UID {2}".format(
                i+1,
                total_messages,
                msg_uid
            ))
            try:
                try:
                    raw_msg = server.fetch(msg_uid,
                                           ["RFC822"])[msg_uid]
                    msg_keys = [b'RFC822', b'BODY[NULL]', b'BODY[]']
                    msg_key = ''
                    for key in msg_keys:
                        if key in raw_msg.keys():
                            msg_key = key
                            break
                    raw_msg = raw_msg[msg_key]

                except (ConnectionResetError, socket.error,
                        TimeoutError,
                        imapclient.exceptions.IMAPClientError) as error:
                    error = error.__str__().lstrip("b'").rstrip("'").rstrip(
                        ".")
                    logger.debug("IMAP error: {0}".format(error.__str__()))
                    logger.debug("Reconnecting to IMAP")
                    try:
                        server.shutdown()
                    except Exception as e:
                        logger.debug(
                            "Failed to log out: {0}".format(e.__str__()))
                    if not ssl:
                        logger.debug("Connecting to IMAP over plain text")
                    server = imapclient.IMAPClient(host,
                                                   port=port,
                                                   ssl=ssl,
                                                   ssl_context=ssl_context,
                                                   use_uid=True)
                    server.login(user, password)
                    server.select_folder(reports_folder)
                    raw_msg = server.fetch(msg_uid,
                                           ["RFC822"])[msg_uid][b"RFC822"]

                msg_content = raw_msg.decode("utf-8", errors="replace")
                sa = strip_attachment_payloads
                parsed_email = parse_report_email(msg_content,
                                                  nameservers=nameservers,
                                                  dns_timeout=dns_timeout,
                                                  strip_attachment_payloads=sa)
                if parsed_email["report_type"] == "aggregate":
                    aggregate_reports.append(parsed_email["report"])
                    aggregate_report_msg_uids.append(msg_uid)
                elif parsed_email["report_type"] == "forensic":
                    forensic_reports.append(parsed_email["report"])
                    forensic_report_msg_uids.append(msg_uid)
            except InvalidDMARCReport as error:
                logger.warning(error.__str__())
                if not test:
                    if delete:
                        logger.debug(
                            "Deleting message UID {0}".format(msg_uid))
                        delete_messages([msg_uid])
                    else:
                        logger.debug(
                            "Moving message UID {0} to {1}".format(
                                msg_uid, invalid_reports_folder))
                        move_messages([msg_uid], invalid_reports_folder)

        if not test:
            if delete:
                processed_messages = aggregate_report_msg_uids + \
                                     forensic_report_msg_uids

                number_of_processed_msgs = len(processed_messages)
                for i in range(number_of_processed_msgs):
                    msg_uid = processed_messages[i]
                    logger.debug(
                        "Deleting message {0} of {1}: UID {2}".format(
                            i + 1, number_of_processed_msgs, msg_uid))
                    try:
                        delete_messages([msg_uid])

                    except imapclient.exceptions.IMAPClientError as e:
                        e = e.__str__().lstrip("b'").rstrip(
                            "'").rstrip(".")
                        message = "Error deleting message UID"
                        e = "{0} {1}: " "{2}".format(message, msg_uid, e)
                        logger.error("IMAP error: {0}".format(e))
                    except (ConnectionResetError, socket.error,
                            TimeoutError) as e:
                        logger.debug("IMAP error: {0}".format(e.__str__()))
                        logger.debug("Reconnecting to IMAP")
                        try:
                            server.shutdown()
                        except Exception as e:
                            logger.debug(
                                "Failed to log out: {0}".format(e.__str__()))
                        if not ssl:
                            logger.debug("Connecting to IMAP over plain text")
                        server = imapclient.IMAPClient(host,
                                                       port=port,
                                                       ssl=ssl,
                                                       ssl_context=ssl_context,
                                                       use_uid=True)
                        server.login(user, password)
                        server.select_folder(reports_folder)
                        delete_messages([msg_uid])
            else:
                if len(aggregate_report_msg_uids) > 0:
                    log_message = "Moving aggregate report messages from"
                    logger.debug(
                        "{0} {1} to {1}".format(
                            log_message, reports_folder,
                            aggregate_reports_folder))
                    number_of_agg_report_msgs = len(aggregate_report_msg_uids)
                    for i in range(number_of_agg_report_msgs):
                        msg_uid = aggregate_report_msg_uids[i]
                        logger.debug(
                            "Moving message {0} of {1}: UID {2}".format(
                                i+1, number_of_agg_report_msgs, msg_uid))
                        try:
                            move_messages([msg_uid],
                                          aggregate_reports_folder)
                        except imapclient.exceptions.IMAPClientError as e:
                            e = e.__str__().lstrip("b'").rstrip(
                                "'").rstrip(".")
                            message = "Error moving message UID"
                            e = "{0} {1}: {2}".format(message, msg_uid, e)
                            logger.error("IMAP error: {0}".format(e))
                        except (ConnectionResetError, socket.error,
                                TimeoutError) as error:
                            logger.debug("IMAP error: {0}".format(
                                error.__str__()))
                            logger.debug("Reconnecting to IMAP")
                            try:
                                server.shutdown()
                            except Exception as e:
                                logger.debug("Failed to log out: {0}".format(
                                    e.__str__()))
                            if not ssl:
                                logger.debug(
                                    "Connecting to IMAP over plain text")
                            server = imapclient.IMAPClient(
                                host,
                                port=port,
                                ssl=ssl,
                                ssl_context=ssl_context,
                                use_uid=True
                            )
                            server.login(user, password)
                            server.select_folder(reports_folder)
                            move_messages([msg_uid],
                                          aggregate_reports_folder)

                if len(forensic_report_msg_uids) > 0:
                    message = "Moving forensic report messages from"
                    logger.debug(
                        "{0} {1} to {2}".format(message,
                                                reports_folder,
                                                forensic_reports_folder))
                    number_of_forensic_msgs = len(forensic_report_msg_uids)
                    for i in range(number_of_forensic_msgs):
                        msg_uid = forensic_report_msg_uids[i]
                        message = "Moving message"
                        logger.debug("{0} {1} of {2}: UID {2}".format(
                            message,
                            i + 1, number_of_forensic_msgs, msg_uid))
                        try:
                            move_messages([msg_uid],
                                          forensic_reports_folder)
                        except imapclient.exceptions.IMAPClientError as e:
                            e = e.__str__().lstrip("b'").rstrip(
                                "'").rstrip(".")
                            e = "Error moving message UID {0}: {1}".format(
                                msg_uid, e)
                            logger.error("IMAP error: {0}".format(e))
                        except (ConnectionResetError, TimeoutError) as error:
                            logger.debug("IMAP error: {0}".format(
                                error.__str__()))
                            logger.debug("Reconnecting to IMAP")
                            try:
                                server.shutdown()
                            except Exception as e:
                                logger.debug("Failed to "
                                             "disconnect: {0}".format(
                                                               e.__str__()))
                            if not ssl:
                                logger.debug(
                                    "Connecting to IMAP over plain text")
                            server = imapclient.IMAPClient(
                                host,
                                port=port,
                                ssl=ssl,
                                ssl_context=ssl_context,
                                use_uid=True)
                            server.login(user, password)
                            server.select_folder(reports_folder)
                            move_messages([msg_uid],
                                          forensic_reports_folder)

        results = OrderedDict([("aggregate_reports", aggregate_reports),
                               ("forensic_reports", forensic_reports)])

        if not test and total_messages > 0:
            # Process emails that came in during the last run
            results = get_dmarc_reports_from_inbox(
                host=host,
                user=user,
                password=password,
                connection=connection,
                port=port,
                ssl=ssl,
                ssl_context=ssl_context,
                move_supported=move_supported,
                reports_folder=reports_folder,
                archive_folder=archive_folder,
                delete=delete,
                test=test,
                nameservers=nameservers,
                dns_timeout=dns_timeout,
                strip_attachment_payloads=strip_attachment_payloads,
                results=results
            )

        return results
    except imapclient.exceptions.IMAPClientError as error:
        error = error.__str__().lstrip("b'").rstrip("'").rstrip(".")
        # Workaround for random Exchange/Office365 IMAP errors
        if "unexpected response" in error or "BAD" in error:
            sleep_minutes = 5
            logger.debug(
                "{0}. "
                "Waiting {1} minutes before trying again".format(
                    error,
                    sleep_minutes))
            time.sleep(sleep_minutes * 60)
            results = get_dmarc_reports_from_inbox(
                host=host,
                user=user,
                password=password,
                connection=connection,
                port=port,
                ssl=ssl,
                ssl_context=ssl_context,
                move_supported=move_supported,
                reports_folder=reports_folder,
                archive_folder=archive_folder,
                delete=delete,
                test=test,
                nameservers=nameservers,
                dns_timeout=dns_timeout,
                strip_attachment_payloads=strip_attachment_payloads,
                results=results
            )

            return results

        raise IMAPError(error)
    except socket.gaierror:
        raise IMAPError("DNS resolution failed")
    except ConnectionRefusedError:
        raise IMAPError("Connection refused")
    except ConnectionResetError:
        sleep_minutes = 5
        logger.debug(
            "Connection reset. "
            "Waiting {0} minutes before trying again".format(sleep_minutes))
        time.sleep(sleep_minutes * 60)
        results = get_dmarc_reports_from_inbox(
            host=host,
            user=user,
            password=password,
            connection=connection,
            port=port,
            ssl=ssl,
            ssl_context=ssl_context,
            move_supported=move_supported,
            reports_folder=reports_folder,
            archive_folder=archive_folder,
            delete=delete,
            test=test,
            nameservers=nameservers,
            dns_timeout=dns_timeout,
            strip_attachment_payloads=strip_attachment_payloads,
            results=results
        )

        return results
    except ConnectionAbortedError:
        raise IMAPError("Connection aborted")
    except TimeoutError:
        raise IMAPError("Connection timed out")
    except SSLError as error:
        raise IMAPError("SSL error: {0}".format(error.__str__()))
    except CertificateError as error:
        raise IMAPError("Certificate error: {0}".format(error.__str__()))
def get_dmarc_reports_from_inbox(host=None,
                                 user=None,
                                 password=None,
                                 connection=None,
                                 port=None,
                                 ssl=True,
                                 ssl_context=None,
                                 move_supported=None,
                                 reports_folder="INBOX",
                                 archive_folder="Archive",
                                 delete=False,
                                 test=False,
                                 nameservers=None,
                                 dns_timeout=6.0,
                                 strip_attachment_payloads=False,
                                 results=None):
    """
    Fetches and parses DMARC reports from sn inbox

    Args:
        host: The mail server hostname or IP address
        user: The mail server user
        password: The mail server password
        connection: An IMAPCLient connection to reuse
        port: The mail server port
        ssl (bool): Use SSL/TLS
        ssl_context (SSLContext): A SSL context
        move_supported: Indicate if the IMAP server supports the MOVE command
        (autodetect if None)
        reports_folder: The IMAP folder where reports can be found
        archive_folder: The folder to move processed mail to
        delete (bool): Delete  messages after processing them
        test (bool): Do not move or delete messages after processing them
        nameservers (list): A list of DNS nameservers to query
        dns_timeout (float): Set the DNS query timeout
        strip_attachment_payloads (bool): Remove attachment payloads from
        forensic report results
        results (dict): Results from the previous run

    Returns:
        OrderedDict: Lists of ``aggregate_reports`` and ``forensic_reports``
    """

    def chunks(l, n):
        """Yield successive n-sized chunks from l."""
        for i in range(0, len(l), n):
            yield l[i:i + n]

    if delete and test:
        raise ValueError("delete and test options are mutually exclusive")

    if connection is None and (user is None or password is None):
        raise ValueError("Must supply a connection, or a username and "
                         "password")

    aggregate_reports = []
    forensic_reports = []
    aggregate_report_msg_uids = []
    forensic_report_msg_uids = []
    aggregate_reports_folder = "{0}/Aggregate".format(archive_folder)
    forensic_reports_folder = "{0}/Forensic".format(archive_folder)
    invalid_reports_folder = "{0}/Invalid".format(archive_folder)

    if results:
        aggregate_reports = results["aggregate_reports"].copy()
        forensic_reports = results["forensic_reports"].copy()

    try:
        if connection:
            server = connection
        else:
            if not ssl:
                logger.debug("Connecting to IMAP over plain text")
            if ssl_context is None:
                ssl_context = create_default_context()
            server = imapclient.IMAPClient(host,
                                           port=port,
                                           ssl=ssl,
                                           ssl_context=ssl_context,
                                           use_uid=True)
            server.login(user, password)

        if move_supported is None:
            server_capabilities = get_imap_capabilities(server)
            move_supported = "MOVE" in server_capabilities

        def delete_messages(msg_uids):
            logger.debug("Deleting message UID(s) {0}".format(",".join(
                str(uid) for uid in msg_uids)))
            if type(msg_uids) == str or type(msg_uids) == int:
                msg_uids = [int(msg_uids)]

            server.delete_messages(msg_uids, silent=True)
            server.expunge(msg_uids)

        def move_messages(msg_uids, folder):
            if type(msg_uids) == str or type(msg_uids) == int:
                msg_uids = [int(msg_uids)]
            for chunk in chunks(msg_uids, 100):
                if move_supported:
                    logger.debug("Moving message UID(s) {0} to {1}".format(
                        ",".join(str(uid) for uid in chunk), folder
                    ))
                    server.move(chunk, folder)
                else:
                    logger.debug("Copying message UID(s) {0} to {1}".format(
                        ",".join(str(uid) for uid in chunk), folder
                    ))
                    server.copy(msg_uids, folder)
                    delete_messages(msg_uids)

        if not server.folder_exists(archive_folder):
            logger.debug("Creating IMAP folder: {0}".format(archive_folder))
            server.create_folder(archive_folder)
        try:
            # Test subfolder creation
            if not server.folder_exists(aggregate_reports_folder):
                server.create_folder(aggregate_reports_folder)
                logger.debug(
                    "Creating IMAP folder: {0}".format(
                        aggregate_reports_folder))
        except imapclient.exceptions.IMAPClientError:
            #  Only replace / with . when . doesn't work
            # This usually indicates a dovecot IMAP server
            aggregate_reports_folder = aggregate_reports_folder.replace("/",
                                                                        ".")
            forensic_reports_folder = forensic_reports_folder.replace("/",
                                                                      ".")
            invalid_reports_folder = invalid_reports_folder.replace("/",
                                                                    ".")
        subfolders = [aggregate_reports_folder,
                      forensic_reports_folder,
                      invalid_reports_folder]

        for subfolder in subfolders:
            if not server.folder_exists(subfolder):
                logger.debug(
                    "Creating IMAP folder: {0}".format(subfolder))
                server.create_folder(subfolder)
        server.select_folder(reports_folder)
        messages = server.search()
        total_messages = len(messages)
        logger.debug("Found {0} messages in IMAP folder {1}".format(
            len(messages), reports_folder))
        for i in range(len(messages)):
            msg_uid = messages[i]
            logger.debug("Processing message {0} of {1}: UID {2}".format(
                i+1,
                total_messages,
                msg_uid
            ))
            try:
                try:
                    raw_msg = server.fetch(msg_uid,
                                           ["RFC822"])[msg_uid]
                    msg_keys = [b'RFC822', b'BODY[NULL]', b'BODY[]']
                    msg_key = ''
                    for key in msg_keys:
                        if key in raw_msg.keys():
                            msg_key = key
                            break
                    raw_msg = raw_msg[msg_key]

                except (ConnectionResetError, socket.error,
                        TimeoutError,
                        imapclient.exceptions.IMAPClientError) as error:
                    error = error.__str__().lstrip("b'").rstrip("'").rstrip(
                        ".")
                    logger.debug("IMAP error: {0}".format(error.__str__()))
                    logger.debug("Reconnecting to IMAP")
                    try:
                        server.shutdown()
                    except Exception as e:
                        logger.debug(
                            "Failed to log out: {0}".format(e.__str__()))
                    if not ssl:
                        logger.debug("Connecting to IMAP over plain text")
                    server = imapclient.IMAPClient(host,
                                                   port=port,
                                                   ssl=ssl,
                                                   ssl_context=ssl_context,
                                                   use_uid=True)
                    server.login(user, password)
                    server.select_folder(reports_folder)
                    raw_msg = server.fetch(msg_uid,
                                           ["RFC822"])[msg_uid][b"RFC822"]

                msg_content = raw_msg.decode("utf-8", errors="replace")
                sa = strip_attachment_payloads
                parsed_email = parse_report_email(msg_content,
                                                  nameservers=nameservers,
                                                  dns_timeout=dns_timeout,
                                                  strip_attachment_payloads=sa)
                if parsed_email["report_type"] == "aggregate":
                    aggregate_reports.append(parsed_email["report"])
                    aggregate_report_msg_uids.append(msg_uid)
                elif parsed_email["report_type"] == "forensic":
                    forensic_reports.append(parsed_email["report"])
                    forensic_report_msg_uids.append(msg_uid)
            except InvalidDMARCReport as error:
                logger.warning(error.__str__())
                if not test:
                    if delete:
                        logger.debug(
                            "Deleting message UID {0}".format(msg_uid))
                        delete_messages([msg_uid])
                    else:
                        logger.debug(
                            "Moving message UID {0} to {1}".format(
                                msg_uid, invalid_reports_folder))
                        move_messages([msg_uid], invalid_reports_folder)

        if not test:
            if delete:
                processed_messages = aggregate_report_msg_uids + \
                                     forensic_report_msg_uids

                number_of_processed_msgs = len(processed_messages)
                for i in range(number_of_processed_msgs):
                    msg_uid = processed_messages[i]
                    logger.debug(
                        "Deleting message {0} of {1}: UID {2}".format(
                            i + 1, number_of_processed_msgs, msg_uid))
                    try:
                        delete_messages([msg_uid])

                    except imapclient.exceptions.IMAPClientError as e:
                        e = e.__str__().lstrip("b'").rstrip(
                            "'").rstrip(".")
                        message = "Error deleting message UID"
                        e = "{0} {1}: " "{2}".format(message, msg_uid, e)
                        logger.error("IMAP error: {0}".format(e))
                    except (ConnectionResetError, socket.error,
                            TimeoutError) as e:
                        logger.debug("IMAP error: {0}".format(e.__str__()))
                        logger.debug("Reconnecting to IMAP")
                        try:
                            server.shutdown()
                        except Exception as e:
                            logger.debug(
                                "Failed to log out: {0}".format(e.__str__()))
                        if not ssl:
                            logger.debug("Connecting to IMAP over plain text")
                        server = imapclient.IMAPClient(host,
                                                       port=port,
                                                       ssl=ssl,
                                                       ssl_context=ssl_context,
                                                       use_uid=True)
                        server.login(user, password)
                        server.select_folder(reports_folder)
                        delete_messages([msg_uid])
            else:
                if len(aggregate_report_msg_uids) > 0:
                    log_message = "Moving aggregate report messages from"
                    logger.debug(
                        "{0} {1} to {1}".format(
                            log_message, reports_folder,
                            aggregate_reports_folder))
                    number_of_agg_report_msgs = len(aggregate_report_msg_uids)
                    for i in range(number_of_agg_report_msgs):
                        msg_uid = aggregate_report_msg_uids[i]
                        logger.debug(
                            "Moving message {0} of {1}: UID {2}".format(
                                i+1, number_of_agg_report_msgs, msg_uid))
                        try:
                            move_messages([msg_uid],
                                          aggregate_reports_folder)
                        except imapclient.exceptions.IMAPClientError as e:
                            e = e.__str__().lstrip("b'").rstrip(
                                "'").rstrip(".")
                            message = "Error moving message UID"
                            e = "{0} {1}: {2}".format(message, msg_uid, e)
                            logger.error("IMAP error: {0}".format(e))
                        except (ConnectionResetError, socket.error,
                                TimeoutError) as error:
                            logger.debug("IMAP error: {0}".format(
                                error.__str__()))
                            logger.debug("Reconnecting to IMAP")
                            try:
                                server.shutdown()
                            except Exception as e:
                                logger.debug("Failed to log out: {0}".format(
                                    e.__str__()))
                            if not ssl:
                                logger.debug(
                                    "Connecting to IMAP over plain text")
                            server = imapclient.IMAPClient(
                                host,
                                port=port,
                                ssl=ssl,
                                ssl_context=ssl_context,
                                use_uid=True
                            )
                            server.login(user, password)
                            server.select_folder(reports_folder)
                            move_messages([msg_uid],
                                          aggregate_reports_folder)

                if len(forensic_report_msg_uids) > 0:
                    message = "Moving forensic report messages from"
                    logger.debug(
                        "{0} {1} to {2}".format(message,
                                                reports_folder,
                                                forensic_reports_folder))
                    number_of_forensic_msgs = len(forensic_report_msg_uids)
                    for i in range(number_of_forensic_msgs):
                        msg_uid = forensic_report_msg_uids[i]
                        message = "Moving message"
                        logger.debug("{0} {1} of {2}: UID {2}".format(
                            message,
                            i + 1, number_of_forensic_msgs, msg_uid))
                        try:
                            move_messages([msg_uid],
                                          forensic_reports_folder)
                        except imapclient.exceptions.IMAPClientError as e:
                            e = e.__str__().lstrip("b'").rstrip(
                                "'").rstrip(".")
                            e = "Error moving message UID {0}: {1}".format(
                                msg_uid, e)
                            logger.error("IMAP error: {0}".format(e))
                        except (ConnectionResetError, TimeoutError) as error:
                            logger.debug("IMAP error: {0}".format(
                                error.__str__()))
                            logger.debug("Reconnecting to IMAP")
                            try:
                                server.shutdown()
                            except Exception as e:
                                logger.debug("Failed to "
                                             "disconnect: {0}".format(
                                                               e.__str__()))
                            if not ssl:
                                logger.debug(
                                    "Connecting to IMAP over plain text")
                            server = imapclient.IMAPClient(
                                host,
                                port=port,
                                ssl=ssl,
                                ssl_context=ssl_context,
                                use_uid=True)
                            server.login(user, password)
                            server.select_folder(reports_folder)
                            move_messages([msg_uid],
                                          forensic_reports_folder)

        results = OrderedDict([("aggregate_reports", aggregate_reports),
                               ("forensic_reports", forensic_reports)])

        if not test and total_messages > 0:
            # Process emails that came in during the last run
            results = get_dmarc_reports_from_inbox(
                host=host,
                user=user,
                password=password,
                connection=connection,
                port=port,
                ssl=ssl,
                ssl_context=ssl_context,
                move_supported=move_supported,
                reports_folder=reports_folder,
                archive_folder=archive_folder,
                delete=delete,
                test=test,
                nameservers=nameservers,
                dns_timeout=dns_timeout,
                strip_attachment_payloads=strip_attachment_payloads,
                results=results
            )

        return results
    except imapclient.exceptions.IMAPClientError as error:
        error = error.__str__().lstrip("b'").rstrip("'").rstrip(".")
        # Workaround for random Exchange/Office365 IMAP errors
        if "unexpected response" in error or "BAD" in error:
            sleep_minutes = 5
            logger.debug(
                "{0}. "
                "Waiting {1} minutes before trying again".format(
                    error,
                    sleep_minutes))
            time.sleep(sleep_minutes * 60)
            results = get_dmarc_reports_from_inbox(
                host=host,
                user=user,
                password=password,
                connection=connection,
                port=port,
                ssl=ssl,
                ssl_context=ssl_context,
                move_supported=move_supported,
                reports_folder=reports_folder,
                archive_folder=archive_folder,
                delete=delete,
                test=test,
                nameservers=nameservers,
                dns_timeout=dns_timeout,
                strip_attachment_payloads=strip_attachment_payloads,
                results=results
            )

            return results

        raise IMAPError(error)
    except socket.gaierror:
        raise IMAPError("DNS resolution failed")
    except ConnectionRefusedError:
        raise IMAPError("Connection refused")
    except ConnectionResetError:
        sleep_minutes = 5
        logger.debug(
            "Connection reset. "
            "Waiting {0} minutes before trying again".format(sleep_minutes))
        time.sleep(sleep_minutes * 60)
        results = get_dmarc_reports_from_inbox(
            host=host,
            user=user,
            password=password,
            connection=connection,
            port=port,
            ssl=ssl,
            ssl_context=ssl_context,
            move_supported=move_supported,
            reports_folder=reports_folder,
            archive_folder=archive_folder,
            delete=delete,
            test=test,
            nameservers=nameservers,
            dns_timeout=dns_timeout,
            strip_attachment_payloads=strip_attachment_payloads,
            results=results
        )

        return results
    except ConnectionAbortedError:
        raise IMAPError("Connection aborted")
    except TimeoutError:
        raise IMAPError("Connection timed out")
    except SSLError as error:
        raise IMAPError("SSL error: {0}".format(error.__str__()))
    except CertificateError as error:
        raise IMAPError("Certificate error: {0}".format(error.__str__()))
def get_dmarc_reports_from_inbox ( host = None , user = None , password = None , connection = None , port = None , ssl = True , ssl_context = None , move_supported = None , reports_folder = "INBOX" , archive_folder = "Archive" , delete = False , test = False , nameservers = None , dns_timeout = 6.0 , strip_attachment_payloads = False , results = None ) : def chunks ( l , n ) : """Yield successive n-sized chunks from l.""" for i in range ( 0 , len ( l ) , n ) : yield l [ i : i + n ] if delete and test : raise ValueError ( "delete and test options are mutually exclusive" ) if connection is None and ( user is None or password is None ) : raise ValueError ( "Must supply a connection, or a username and " "password" ) aggregate_reports = [ ] forensic_reports = [ ] aggregate_report_msg_uids = [ ] forensic_report_msg_uids = [ ] aggregate_reports_folder = "{0}/Aggregate" . format ( archive_folder ) forensic_reports_folder = "{0}/Forensic" . format ( archive_folder ) invalid_reports_folder = "{0}/Invalid" . format ( archive_folder ) if results : aggregate_reports = results [ "aggregate_reports" ] . copy ( ) forensic_reports = results [ "forensic_reports" ] . copy ( ) try : if connection : server = connection else : if not ssl : logger . debug ( "Connecting to IMAP over plain text" ) if ssl_context is None : ssl_context = create_default_context ( ) server = imapclient . IMAPClient ( host , port = port , ssl = ssl , ssl_context = ssl_context , use_uid = True ) server . login ( user , password ) if move_supported is None : server_capabilities = get_imap_capabilities ( server ) move_supported = "MOVE" in server_capabilities def delete_messages ( msg_uids ) : logger . debug ( "Deleting message UID(s) {0}" . format ( "," . join ( str ( uid ) for uid in msg_uids ) ) ) if type ( msg_uids ) == str or type ( msg_uids ) == int : msg_uids = [ int ( msg_uids ) ] server . delete_messages ( msg_uids , silent = True ) server . expunge ( msg_uids ) def move_messages ( msg_uids , folder ) : if type ( msg_uids ) == str or type ( msg_uids ) == int : msg_uids = [ int ( msg_uids ) ] for chunk in chunks ( msg_uids , 100 ) : if move_supported : logger . debug ( "Moving message UID(s) {0} to {1}" . format ( "," . join ( str ( uid ) for uid in chunk ) , folder ) ) server . move ( chunk , folder ) else : logger . debug ( "Copying message UID(s) {0} to {1}" . format ( "," . join ( str ( uid ) for uid in chunk ) , folder ) ) server . copy ( msg_uids , folder ) delete_messages ( msg_uids ) if not server . folder_exists ( archive_folder ) : logger . debug ( "Creating IMAP folder: {0}" . format ( archive_folder ) ) server . create_folder ( archive_folder ) try : # Test subfolder creation if not server . folder_exists ( aggregate_reports_folder ) : server . create_folder ( aggregate_reports_folder ) logger . debug ( "Creating IMAP folder: {0}" . format ( aggregate_reports_folder ) ) except imapclient . exceptions . IMAPClientError : #  Only replace / with . when . doesn't work # This usually indicates a dovecot IMAP server aggregate_reports_folder = aggregate_reports_folder . replace ( "/" , "." ) forensic_reports_folder = forensic_reports_folder . replace ( "/" , "." ) invalid_reports_folder = invalid_reports_folder . replace ( "/" , "." ) subfolders = [ aggregate_reports_folder , forensic_reports_folder , invalid_reports_folder ] for subfolder in subfolders : if not server . folder_exists ( subfolder ) : logger . debug ( "Creating IMAP folder: {0}" . format ( subfolder ) ) server . create_folder ( subfolder ) server . select_folder ( reports_folder ) messages = server . search ( ) total_messages = len ( messages ) logger . debug ( "Found {0} messages in IMAP folder {1}" . format ( len ( messages ) , reports_folder ) ) for i in range ( len ( messages ) ) : msg_uid = messages [ i ] logger . debug ( "Processing message {0} of {1}: UID {2}" . format ( i + 1 , total_messages , msg_uid ) ) try : try : raw_msg = server . fetch ( msg_uid , [ "RFC822" ] ) [ msg_uid ] msg_keys = [ b'RFC822' , b'BODY[NULL]' , b'BODY[]' ] msg_key = '' for key in msg_keys : if key in raw_msg . keys ( ) : msg_key = key break raw_msg = raw_msg [ msg_key ] except ( ConnectionResetError , socket . error , TimeoutError , imapclient . exceptions . IMAPClientError ) as error : error = error . __str__ ( ) . lstrip ( "b'" ) . rstrip ( "'" ) . rstrip ( "." ) logger . debug ( "IMAP error: {0}" . format ( error . __str__ ( ) ) ) logger . debug ( "Reconnecting to IMAP" ) try : server . shutdown ( ) except Exception as e : logger . debug ( "Failed to log out: {0}" . format ( e . __str__ ( ) ) ) if not ssl : logger . debug ( "Connecting to IMAP over plain text" ) server = imapclient . IMAPClient ( host , port = port , ssl = ssl , ssl_context = ssl_context , use_uid = True ) server . login ( user , password ) server . select_folder ( reports_folder ) raw_msg = server . fetch ( msg_uid , [ "RFC822" ] ) [ msg_uid ] [ b"RFC822" ] msg_content = raw_msg . decode ( "utf-8" , errors = "replace" ) sa = strip_attachment_payloads parsed_email = parse_report_email ( msg_content , nameservers = nameservers , dns_timeout = dns_timeout , strip_attachment_payloads = sa ) if parsed_email [ "report_type" ] == "aggregate" : aggregate_reports . append ( parsed_email [ "report" ] ) aggregate_report_msg_uids . append ( msg_uid ) elif parsed_email [ "report_type" ] == "forensic" : forensic_reports . append ( parsed_email [ "report" ] ) forensic_report_msg_uids . append ( msg_uid ) except InvalidDMARCReport as error : logger . warning ( error . __str__ ( ) ) if not test : if delete : logger . debug ( "Deleting message UID {0}" . format ( msg_uid ) ) delete_messages ( [ msg_uid ] ) else : logger . debug ( "Moving message UID {0} to {1}" . format ( msg_uid , invalid_reports_folder ) ) move_messages ( [ msg_uid ] , invalid_reports_folder ) if not test : if delete : processed_messages = aggregate_report_msg_uids + forensic_report_msg_uids number_of_processed_msgs = len ( processed_messages ) for i in range ( number_of_processed_msgs ) : msg_uid = processed_messages [ i ] logger . debug ( "Deleting message {0} of {1}: UID {2}" . format ( i + 1 , number_of_processed_msgs , msg_uid ) ) try : delete_messages ( [ msg_uid ] ) except imapclient . exceptions . IMAPClientError as e : e = e . __str__ ( ) . lstrip ( "b'" ) . rstrip ( "'" ) . rstrip ( "." ) message = "Error deleting message UID" e = "{0} {1}: " "{2}" . format ( message , msg_uid , e ) logger . error ( "IMAP error: {0}" . format ( e ) ) except ( ConnectionResetError , socket . error , TimeoutError ) as e : logger . debug ( "IMAP error: {0}" . format ( e . __str__ ( ) ) ) logger . debug ( "Reconnecting to IMAP" ) try : server . shutdown ( ) except Exception as e : logger . debug ( "Failed to log out: {0}" . format ( e . __str__ ( ) ) ) if not ssl : logger . debug ( "Connecting to IMAP over plain text" ) server = imapclient . IMAPClient ( host , port = port , ssl = ssl , ssl_context = ssl_context , use_uid = True ) server . login ( user , password ) server . select_folder ( reports_folder ) delete_messages ( [ msg_uid ] ) else : if len ( aggregate_report_msg_uids ) > 0 : log_message = "Moving aggregate report messages from" logger . debug ( "{0} {1} to {1}" . format ( log_message , reports_folder , aggregate_reports_folder ) ) number_of_agg_report_msgs = len ( aggregate_report_msg_uids ) for i in range ( number_of_agg_report_msgs ) : msg_uid = aggregate_report_msg_uids [ i ] logger . debug ( "Moving message {0} of {1}: UID {2}" . format ( i + 1 , number_of_agg_report_msgs , msg_uid ) ) try : move_messages ( [ msg_uid ] , aggregate_reports_folder ) except imapclient . exceptions . IMAPClientError as e : e = e . __str__ ( ) . lstrip ( "b'" ) . rstrip ( "'" ) . rstrip ( "." ) message = "Error moving message UID" e = "{0} {1}: {2}" . format ( message , msg_uid , e ) logger . error ( "IMAP error: {0}" . format ( e ) ) except ( ConnectionResetError , socket . error , TimeoutError ) as error : logger . debug ( "IMAP error: {0}" . format ( error . __str__ ( ) ) ) logger . debug ( "Reconnecting to IMAP" ) try : server . shutdown ( ) except Exception as e : logger . debug ( "Failed to log out: {0}" . format ( e . __str__ ( ) ) ) if not ssl : logger . debug ( "Connecting to IMAP over plain text" ) server = imapclient . IMAPClient ( host , port = port , ssl = ssl , ssl_context = ssl_context , use_uid = True ) server . login ( user , password ) server . select_folder ( reports_folder ) move_messages ( [ msg_uid ] , aggregate_reports_folder ) if len ( forensic_report_msg_uids ) > 0 : message = "Moving forensic report messages from" logger . debug ( "{0} {1} to {2}" . format ( message , reports_folder , forensic_reports_folder ) ) number_of_forensic_msgs = len ( forensic_report_msg_uids ) for i in range ( number_of_forensic_msgs ) : msg_uid = forensic_report_msg_uids [ i ] message = "Moving message" logger . debug ( "{0} {1} of {2}: UID {2}" . format ( message , i + 1 , number_of_forensic_msgs , msg_uid ) ) try : move_messages ( [ msg_uid ] , forensic_reports_folder ) except imapclient . exceptions . IMAPClientError as e : e = e . __str__ ( ) . lstrip ( "b'" ) . rstrip ( "'" ) . rstrip ( "." ) e = "Error moving message UID {0}: {1}" . format ( msg_uid , e ) logger . error ( "IMAP error: {0}" . format ( e ) ) except ( ConnectionResetError , TimeoutError ) as error : logger . debug ( "IMAP error: {0}" . format ( error . __str__ ( ) ) ) logger . debug ( "Reconnecting to IMAP" ) try : server . shutdown ( ) except Exception as e : logger . debug ( "Failed to " "disconnect: {0}" . format ( e . __str__ ( ) ) ) if not ssl : logger . debug ( "Connecting to IMAP over plain text" ) server = imapclient . IMAPClient ( host , port = port , ssl = ssl , ssl_context = ssl_context , use_uid = True ) server . login ( user , password ) server . select_folder ( reports_folder ) move_messages ( [ msg_uid ] , forensic_reports_folder ) results = OrderedDict ( [ ( "aggregate_reports" , aggregate_reports ) , ( "forensic_reports" , forensic_reports ) ] ) if not test and total_messages > 0 : # Process emails that came in during the last run results = get_dmarc_reports_from_inbox ( host = host , user = user , password = password , connection = connection , port = port , ssl = ssl , ssl_context = ssl_context , move_supported = move_supported , reports_folder = reports_folder , archive_folder = archive_folder , delete = delete , test = test , nameservers = nameservers , dns_timeout = dns_timeout , strip_attachment_payloads = strip_attachment_payloads , results = results ) return results except imapclient . exceptions . IMAPClientError as error : error = error . __str__ ( ) . lstrip ( "b'" ) . rstrip ( "'" ) . rstrip ( "." ) # Workaround for random Exchange/Office365 IMAP errors if "unexpected response" in error or "BAD" in error : sleep_minutes = 5 logger . debug ( "{0}. " "Waiting {1} minutes before trying again" . format ( error , sleep_minutes ) ) time . sleep ( sleep_minutes * 60 ) results = get_dmarc_reports_from_inbox ( host = host , user = user , password = password , connection = connection , port = port , ssl = ssl , ssl_context = ssl_context , move_supported = move_supported , reports_folder = reports_folder , archive_folder = archive_folder , delete = delete , test = test , nameservers = nameservers , dns_timeout = dns_timeout , strip_attachment_payloads = strip_attachment_payloads , results = results ) return results raise IMAPError ( error ) except socket . gaierror : raise IMAPError ( "DNS resolution failed" ) except ConnectionRefusedError : raise IMAPError ( "Connection refused" ) except ConnectionResetError : sleep_minutes = 5 logger . debug ( "Connection reset. " "Waiting {0} minutes before trying again" . format ( sleep_minutes ) ) time . sleep ( sleep_minutes * 60 ) results = get_dmarc_reports_from_inbox ( host = host , user = user , password = password , connection = connection , port = port , ssl = ssl , ssl_context = ssl_context , move_supported = move_supported , reports_folder = reports_folder , archive_folder = archive_folder , delete = delete , test = test , nameservers = nameservers , dns_timeout = dns_timeout , strip_attachment_payloads = strip_attachment_payloads , results = results ) return results except ConnectionAbortedError : raise IMAPError ( "Connection aborted" ) except TimeoutError : raise IMAPError ( "Connection timed out" ) except SSLError as error : raise IMAPError ( "SSL error: {0}" . format ( error . __str__ ( ) ) ) except CertificateError as error : raise IMAPError ( "Certificate error: {0}" . format ( error . __str__ ( ) ) )
Fetches and parses DMARC reports from sn inbox

    Args:
        host: The mail server hostname or IP address
        user: The mail server user
        password: The mail server password
        connection: An IMAPCLient connection to reuse
        port: The mail server port
        ssl (bool): Use SSL/TLS
        ssl_context (SSLContext): A SSL context
        move_supported: Indicate if the IMAP server supports the MOVE command
        (autodetect if None)
        reports_folder: The IMAP folder where reports can be found
        archive_folder: The folder to move processed mail to
        delete (bool): Delete  messages after processing them
        test (bool): Do not move or delete messages after processing them
        nameservers (list): A list of DNS nameservers to query
        dns_timeout (float): Set the DNS query timeout
        strip_attachment_payloads (bool): Remove attachment payloads from
        forensic report results
        results (dict): Results from the previous run

    Returns:
        OrderedDict: Lists of ``aggregate_reports`` and ``forensic_reports``
Fetches and parses DMARC reports from sn inbox