We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[Import Document] Connector does not process MD files
## Description

The Import Document connector currently supports plain/text media type, however files with the `.md` file extension are not recognized as a valid document. 

## Environment

1. OS (where OpenCTI server runs): AWS ECS Fargate
2. OpenCTI version: 5.1.4
3. OpenCTI client: python
4. Other environment details:

## Reproducible Steps

Steps to create the smallest reproducible scenario:
1. Run the Import External Reference connector to get a .md file OR just upload a .md file to the platform
2. Try to run an enrichment on the .md file

## Expected Output

I would expect that the Import connector would or could import a file, regardless of the file name.  

## Actual Output

There is no Output as the connector/platform doesn't recognize the .md file.  Only work around is to download the file, rename to a .txt file extension, and upload to the platform.
 
## Screenshots (optional)
<img width="1483" alt="Screen Shot 2022-04-28 at 9 24 53 AM" src="https://user-images.githubusercontent.com/30411037/165775435-87f694cf-ada9-439f-9cf7-246228283d80.png">
<img width="753" alt="Screen Shot 2022-04-28 at 9 24 20 AM" src="https://user-images.githubusercontent.com/30411037/165775444-fa1ade88-51f8-45a1-9fd8-f1d14002d903.png">



--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `internal-import-file/import-document/src/reportimporter/report_parser.py`
Content:
```
1 import logging
2 import os
3 import io
4 from typing import Dict, List, Pattern, IO, Tuple
5 
6 import ioc_finder
7 from bs4 import BeautifulSoup
8 from pdfminer.high_level import extract_pages
9 from pdfminer.layout import LTTextContainer
10 from pycti import OpenCTIConnectorHelper
11 from reportimporter.constants import (
12     OBSERVABLE_CLASS,
13     ENTITY_CLASS,
14     RESULT_FORMAT_MATCH,
15     RESULT_FORMAT_TYPE,
16     RESULT_FORMAT_CATEGORY,
17     RESULT_FORMAT_RANGE,
18     MIME_PDF,
19     MIME_TXT,
20     MIME_HTML,
21     MIME_CSV,
22     OBSERVABLE_DETECTION_CUSTOM_REGEX,
23     OBSERVABLE_DETECTION_LIBRARY,
24 )
25 from reportimporter.models import Observable, Entity
26 from reportimporter.util import library_mapping
27 
28 
29 class ReportParser(object):
30     """
31     Report parser based on IOCParser
32     """
33 
34     def __init__(
35         self,
36         helper: OpenCTIConnectorHelper,
37         entity_list: List[Entity],
38         observable_list: List[Observable],
39     ):
40 
41         self.helper = helper
42         self.entity_list = entity_list
43         self.observable_list = observable_list
44 
45         # Disable INFO logging by pdfminer
46         logging.getLogger("pdfminer").setLevel(logging.WARNING)
47 
48         # Supported file types
49         self.supported_file_types = {
50             MIME_PDF: self._parse_pdf,
51             MIME_TXT: self._parse_text,
52             MIME_HTML: self._parse_html,
53             MIME_CSV: self._parse_text,
54         }
55 
56         self.library_lookup = library_mapping()
57 
58     def _is_whitelisted(self, regex_list: List[Pattern], ind_match: str):
59         for regex in regex_list:
60             self.helper.log_debug(f"Filter regex '{regex}' for value '{ind_match}'")
61             result = regex.search(ind_match)
62             if result:
63                 self.helper.log_debug(f"Value {ind_match} is whitelisted with {regex}")
64                 return True
65         return False
66 
67     def _post_parse_observables(
68         self, ind_match: str, observable: Observable, match_range: Tuple
69     ) -> Dict:
70         self.helper.log_debug(f"Observable match: {ind_match}")
71 
72         if self._is_whitelisted(observable.filter_regex, ind_match):
73             return {}
74 
75         return self._format_match(
76             OBSERVABLE_CLASS, observable.stix_target, ind_match, match_range
77         )
78 
79     def _parse(self, data: str) -> Dict[str, Dict]:
80         list_matches = {}
81 
82         # Defang text
83         data = ioc_finder.prepare_text(data)
84 
85         for observable in self.observable_list:
86             list_matches.update(self._extract_observable(observable, data))
87 
88         for entity in self.entity_list:
89             list_matches = self._extract_entity(entity, list_matches, data)
90 
91         self.helper.log_debug(f"Text: '{data}' -> extracts {list_matches}")
92         return list_matches
93 
94     def _parse_pdf(self, file_data: IO) -> Dict[str, Dict]:
95         parse_info = {}
96         try:
97             for page_layout in extract_pages(file_data):
98                 for element in page_layout:
99                     if isinstance(element, LTTextContainer):
100                         text = element.get_text()
101                         # Parsing with newlines has been deprecated
102                         no_newline_text = text.replace("\n", "")
103                         parse_info.update(self._parse(no_newline_text))
104 
105                 # TODO also extract information from images/figures using OCR
106                 # https://pdfminersix.readthedocs.io/en/latest/topic/converting_pdf_to_text.html#topic-pdf-to-text-layout
107 
108         except Exception as e:
109             logging.exception(f"Pdf Parsing Error: {e}")
110 
111         return parse_info
112 
113     def _parse_text(self, file_data: IO) -> Dict[str, Dict]:
114         parse_info = {}
115         for text in file_data.readlines():
116             parse_info.update(self._parse(text.decode("utf-8")))
117         return parse_info
118 
119     def _parse_html(self, file_data: IO) -> Dict[str, Dict]:
120         parse_info = {}
121         soup = BeautifulSoup(file_data, "html.parser")
122         buf = io.StringIO(soup.get_text())
123         for text in buf.readlines():
124             parse_info.update(self._parse(text))
125         return parse_info
126 
127     def run_parser(self, file_path: str, file_type: str) -> List[Dict]:
128         parsing_results = []
129 
130         file_parser = self.supported_file_types.get(file_type, None)
131         if not file_parser:
132             raise NotImplementedError(f"No parser available for file type {file_type}")
133 
134         if not os.path.isfile(file_path):
135             raise IOError(f"File path is not a file: {file_path}")
136 
137         self.helper.log_info(f"Parsing report {file_path} {file_type}")
138 
139         try:
140             with open(file_path, "rb") as file_data:
141                 parsing_results = file_parser(file_data)
142         except Exception as e:
143             logging.exception(f"Parsing Error: {e}")
144 
145         parsing_results = list(parsing_results.values())
146 
147         return parsing_results
148 
149     @staticmethod
150     def _format_match(
151         format_type: str, category: str, match: str, match_range: Tuple = (0, 0)
152     ) -> Dict:
153         return {
154             RESULT_FORMAT_TYPE: format_type,
155             RESULT_FORMAT_CATEGORY: category,
156             RESULT_FORMAT_MATCH: match,
157             RESULT_FORMAT_RANGE: match_range,
158         }
159 
160     @staticmethod
161     def _sco_present(
162         match_list: Dict, entity_range: Tuple, filter_sco_types: List
163     ) -> str:
164         for match_name, match_info in match_list.items():
165             if match_info[RESULT_FORMAT_CATEGORY] in filter_sco_types:
166                 if (
167                     match_info[RESULT_FORMAT_RANGE][0] <= entity_range[0]
168                     and entity_range[1] <= match_info[RESULT_FORMAT_RANGE][1]
169                 ):
170                     return match_name
171 
172         return ""
173 
174     def _extract_observable(self, observable: Observable, data: str) -> Dict:
175         list_matches = {}
176         if observable.detection_option == OBSERVABLE_DETECTION_CUSTOM_REGEX:
177             for regex in observable.regex:
178                 for match in regex.finditer(data):
179                     match_value = match.group()
180 
181                     ind_match = self._post_parse_observables(
182                         match_value, observable, match.span()
183                     )
184                     if ind_match:
185                         list_matches[match.group()] = ind_match
186 
187         elif observable.detection_option == OBSERVABLE_DETECTION_LIBRARY:
188             lookup_function = self.library_lookup.get(observable.stix_target, None)
189             if not lookup_function:
190                 self.helper.log_error(
191                     f"Selected library function is not implemented: {observable.iocfinder_function}"
192                 )
193                 return {}
194 
195             matches = lookup_function(data)
196 
197             for match in matches:
198                 match_str = str(match)
199                 if match_str in data:
200                     start = data.index(match_str)
201                 elif match_str in data.lower():
202                     self.helper.log_debug(
203                         f"External library manipulated the extracted value '{match_str}' from the "
204                         f"original text '{data}' to lower case"
205                     )
206                     start = data.lower().index(match_str)
207                 else:
208                     self.helper.log_error(
209                         f"The extracted text '{match_str}' is not part of the original text '{data}'. "
210                         f"Please open a GitHub issue to report this problem!"
211                     )
212                     continue
213 
214                 ind_match = self._post_parse_observables(
215                     match, observable, (start, len(match_str) + start)
216                 )
217                 if ind_match:
218                     list_matches[match] = ind_match
219 
220         return list_matches
221 
222     def _extract_entity(self, entity: Entity, list_matches: Dict, data: str) -> Dict:
223         regex_list = entity.regex
224 
225         observable_keys = []
226         end_index = set()
227         match_dict = {}
228         match_key = ""
229 
230         # Run all regexes for entity X
231         for regex in regex_list:
232             for match in regex.finditer(data):
233                 match_key = match.group()
234                 if match_key in match_dict:
235                     match_dict[match_key].append(match.span())
236                 else:
237                     match_dict[match_key] = [match.span()]
238 
239         # No maches for this entity
240         if len(match_dict) == 0:
241             return list_matches
242 
243         # Run through all matches for entity X and check if they are part of a domain
244         # yes -> skip
245         # no -> add index to end_index
246         for match, match_indices in match_dict.items():
247             for match_index in match_indices:
248                 skip_val = self._sco_present(
249                     list_matches, match_index, entity.omit_match_in
250                 )
251                 if skip_val:
252                     self.helper.log_debug(
253                         f"Skipping Entity '{match}', it is part of an omitted field '{entity.omit_match_in}' \"{skip_val}\""
254                     )
255                 else:
256                     self.helper.log_debug(
257                         f"Entity match: '{match}' of regex: '{regex_list}'"
258                     )
259                     end_index.add(match_index)
260                     if match in list_matches.keys():
261                         observable_keys.append(match)
262 
263         # Remove all observables which found the same information/Entity match
264         for observable_key in observable_keys:
265             if observable_key in list_matches:
266                 del list_matches[observable_key]
267                 self.helper.log_debug(
268                     f"Value {observable_key} is also matched by entity {entity.name}"
269                 )
270 
271         # Check if entity was matched at least once in the text
272         # If yes, then add identity to match list
273         if end_index:
274             list_matches[match_key] = self._format_match(
275                 ENTITY_CLASS, entity.name, entity.stix_id
276             )
277 
278         return list_matches
279
```

Path: `internal-import-file/import-document/src/reportimporter/constants.py`
Content:
```
1 MIME_PDF = "application/pdf"
2 MIME_TXT = "text/plain"
3 MIME_HTML = "text/html"
4 MIME_CSV = "text/csv"
5 
6 RESULT_FORMAT_TYPE = "type"
7 RESULT_FORMAT_CATEGORY = "category"
8 RESULT_FORMAT_MATCH = "match"
9 RESULT_FORMAT_RANGE = "range"
10 
11 ENTITY_CLASS = "entity"
12 OBSERVABLE_CLASS = "observable"
13 
14 CONFIG_PATH = "filter_list"
15 COMMENT_INDICATOR = "#"
16 
17 OBSERVABLE_DETECTION_CUSTOM_REGEX = "custom_regex"
18 OBSERVABLE_DETECTION_LIBRARY = "library"
19 OBSERVABLE_DETECTION_OPTIONS = [
20     OBSERVABLE_DETECTION_LIBRARY,
21     OBSERVABLE_DETECTION_CUSTOM_REGEX,
22 ]
23
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/internal-import-file/import-document/src/reportimporter/constants.py b/internal-import-file/import-document/src/reportimporter/constants.py
--- a/internal-import-file/import-document/src/reportimporter/constants.py
+++ b/internal-import-file/import-document/src/reportimporter/constants.py
@@ -2,6 +2,7 @@
 MIME_TXT = "text/plain"
 MIME_HTML = "text/html"
 MIME_CSV = "text/csv"
+MIME_MD = "text/markdown"
 
 RESULT_FORMAT_TYPE = "type"
 RESULT_FORMAT_CATEGORY = "category"
diff --git a/internal-import-file/import-document/src/reportimporter/report_parser.py b/internal-import-file/import-document/src/reportimporter/report_parser.py
--- a/internal-import-file/import-document/src/reportimporter/report_parser.py
+++ b/internal-import-file/import-document/src/reportimporter/report_parser.py
@@ -19,6 +19,7 @@
     MIME_TXT,
     MIME_HTML,
     MIME_CSV,
+    MIME_MD,
     OBSERVABLE_DETECTION_CUSTOM_REGEX,
     OBSERVABLE_DETECTION_LIBRARY,
 )
@@ -51,6 +52,7 @@
             MIME_TXT: self._parse_text,
             MIME_HTML: self._parse_html,
             MIME_CSV: self._parse_text,
+            MIME_MD: self._parse_text,
         }
 
         self.library_lookup = library_mapping()

{"golden_diff": "diff --git a/internal-import-file/import-document/src/reportimporter/constants.py b/internal-import-file/import-document/src/reportimporter/constants.py\n--- a/internal-import-file/import-document/src/reportimporter/constants.py\n+++ b/internal-import-file/import-document/src/reportimporter/constants.py\n@@ -2,6 +2,7 @@\n MIME_TXT = \"text/plain\"\n MIME_HTML = \"text/html\"\n MIME_CSV = \"text/csv\"\n+MIME_MD = \"text/markdown\"\n \n RESULT_FORMAT_TYPE = \"type\"\n RESULT_FORMAT_CATEGORY = \"category\"\ndiff --git a/internal-import-file/import-document/src/reportimporter/report_parser.py b/internal-import-file/import-document/src/reportimporter/report_parser.py\n--- a/internal-import-file/import-document/src/reportimporter/report_parser.py\n+++ b/internal-import-file/import-document/src/reportimporter/report_parser.py\n@@ -19,6 +19,7 @@\n     MIME_TXT,\n     MIME_HTML,\n     MIME_CSV,\n+    MIME_MD,\n     OBSERVABLE_DETECTION_CUSTOM_REGEX,\n     OBSERVABLE_DETECTION_LIBRARY,\n )\n@@ -51,6 +52,7 @@\n             MIME_TXT: self._parse_text,\n             MIME_HTML: self._parse_html,\n             MIME_CSV: self._parse_text,\n+            MIME_MD: self._parse_text,\n         }\n \n         self.library_lookup = library_mapping()\n", "issue": "[Import Document] Connector does not process MD files\n## Description\r\n\r\nThe Import Document connector currently supports plain/text media type, however files with the `.md` file extension are not recognized as a valid document. \r\n\r\n## Environment\r\n\r\n1. OS (where OpenCTI server runs): AWS ECS Fargate\r\n2. OpenCTI version: 5.1.4\r\n3. OpenCTI client: python\r\n4. Other environment details:\r\n\r\n## Reproducible Steps\r\n\r\nSteps to create the smallest reproducible scenario:\r\n1. Run the Import External Reference connector to get a .md file OR just upload a .md file to the platform\r\n2. Try to run an enrichment on the .md file\r\n\r\n## Expected Output\r\n\r\nI would expect that the Import connector would or could import a file, regardless of the file name.  \r\n\r\n## Actual Output\r\n\r\nThere is no Output as the connector/platform doesn't recognize the .md file.  Only work around is to download the file, rename to a .txt file extension, and upload to the platform.\r\n \r\n## Screenshots (optional)\r\n<img width=\"1483\" alt=\"Screen Shot 2022-04-28 at 9 24 53 AM\" src=\"https://user-images.githubusercontent.com/30411037/165775435-87f694cf-ada9-439f-9cf7-246228283d80.png\">\r\n<img width=\"753\" alt=\"Screen Shot 2022-04-28 at 9 24 20 AM\" src=\"https://user-images.githubusercontent.com/30411037/165775444-fa1ade88-51f8-45a1-9fd8-f1d14002d903.png\">\r\n\r\n\n", "before_files": [{"content": "import logging\nimport os\nimport io\nfrom typing import Dict, List, Pattern, IO, Tuple\n\nimport ioc_finder\nfrom bs4 import BeautifulSoup\nfrom pdfminer.high_level import extract_pages\nfrom pdfminer.layout import LTTextContainer\nfrom pycti import OpenCTIConnectorHelper\nfrom reportimporter.constants import (\n    OBSERVABLE_CLASS,\n    ENTITY_CLASS,\n    RESULT_FORMAT_MATCH,\n    RESULT_FORMAT_TYPE,\n    RESULT_FORMAT_CATEGORY,\n    RESULT_FORMAT_RANGE,\n    MIME_PDF,\n    MIME_TXT,\n    MIME_HTML,\n    MIME_CSV,\n    OBSERVABLE_DETECTION_CUSTOM_REGEX,\n    OBSERVABLE_DETECTION_LIBRARY,\n)\nfrom reportimporter.models import Observable, Entity\nfrom reportimporter.util import library_mapping\n\n\nclass ReportParser(object):\n    \"\"\"\n    Report parser based on IOCParser\n    \"\"\"\n\n    def __init__(\n        self,\n        helper: OpenCTIConnectorHelper,\n        entity_list: List[Entity],\n        observable_list: List[Observable],\n    ):\n\n        self.helper = helper\n        self.entity_list = entity_list\n        self.observable_list = observable_list\n\n        # Disable INFO logging by pdfminer\n        logging.getLogger(\"pdfminer\").setLevel(logging.WARNING)\n\n        # Supported file types\n        self.supported_file_types = {\n            MIME_PDF: self._parse_pdf,\n            MIME_TXT: self._parse_text,\n            MIME_HTML: self._parse_html,\n            MIME_CSV: self._parse_text,\n        }\n\n        self.library_lookup = library_mapping()\n\n    def _is_whitelisted(self, regex_list: List[Pattern], ind_match: str):\n        for regex in regex_list:\n            self.helper.log_debug(f\"Filter regex '{regex}' for value '{ind_match}'\")\n            result = regex.search(ind_match)\n            if result:\n                self.helper.log_debug(f\"Value {ind_match} is whitelisted with {regex}\")\n                return True\n        return False\n\n    def _post_parse_observables(\n        self, ind_match: str, observable: Observable, match_range: Tuple\n    ) -> Dict:\n        self.helper.log_debug(f\"Observable match: {ind_match}\")\n\n        if self._is_whitelisted(observable.filter_regex, ind_match):\n            return {}\n\n        return self._format_match(\n            OBSERVABLE_CLASS, observable.stix_target, ind_match, match_range\n        )\n\n    def _parse(self, data: str) -> Dict[str, Dict]:\n        list_matches = {}\n\n        # Defang text\n        data = ioc_finder.prepare_text(data)\n\n        for observable in self.observable_list:\n            list_matches.update(self._extract_observable(observable, data))\n\n        for entity in self.entity_list:\n            list_matches = self._extract_entity(entity, list_matches, data)\n\n        self.helper.log_debug(f\"Text: '{data}' -> extracts {list_matches}\")\n        return list_matches\n\n    def _parse_pdf(self, file_data: IO) -> Dict[str, Dict]:\n        parse_info = {}\n        try:\n            for page_layout in extract_pages(file_data):\n                for element in page_layout:\n                    if isinstance(element, LTTextContainer):\n                        text = element.get_text()\n                        # Parsing with newlines has been deprecated\n                        no_newline_text = text.replace(\"\\n\", \"\")\n                        parse_info.update(self._parse(no_newline_text))\n\n                # TODO also extract information from images/figures using OCR\n                # https://pdfminersix.readthedocs.io/en/latest/topic/converting_pdf_to_text.html#topic-pdf-to-text-layout\n\n        except Exception as e:\n            logging.exception(f\"Pdf Parsing Error: {e}\")\n\n        return parse_info\n\n    def _parse_text(self, file_data: IO) -> Dict[str, Dict]:\n        parse_info = {}\n        for text in file_data.readlines():\n            parse_info.update(self._parse(text.decode(\"utf-8\")))\n        return parse_info\n\n    def _parse_html(self, file_data: IO) -> Dict[str, Dict]:\n        parse_info = {}\n        soup = BeautifulSoup(file_data, \"html.parser\")\n        buf = io.StringIO(soup.get_text())\n        for text in buf.readlines():\n            parse_info.update(self._parse(text))\n        return parse_info\n\n    def run_parser(self, file_path: str, file_type: str) -> List[Dict]:\n        parsing_results = []\n\n        file_parser = self.supported_file_types.get(file_type, None)\n        if not file_parser:\n            raise NotImplementedError(f\"No parser available for file type {file_type}\")\n\n        if not os.path.isfile(file_path):\n            raise IOError(f\"File path is not a file: {file_path}\")\n\n        self.helper.log_info(f\"Parsing report {file_path} {file_type}\")\n\n        try:\n            with open(file_path, \"rb\") as file_data:\n                parsing_results = file_parser(file_data)\n        except Exception as e:\n            logging.exception(f\"Parsing Error: {e}\")\n\n        parsing_results = list(parsing_results.values())\n\n        return parsing_results\n\n    @staticmethod\n    def _format_match(\n        format_type: str, category: str, match: str, match_range: Tuple = (0, 0)\n    ) -> Dict:\n        return {\n            RESULT_FORMAT_TYPE: format_type,\n            RESULT_FORMAT_CATEGORY: category,\n            RESULT_FORMAT_MATCH: match,\n            RESULT_FORMAT_RANGE: match_range,\n        }\n\n    @staticmethod\n    def _sco_present(\n        match_list: Dict, entity_range: Tuple, filter_sco_types: List\n    ) -> str:\n        for match_name, match_info in match_list.items():\n            if match_info[RESULT_FORMAT_CATEGORY] in filter_sco_types:\n                if (\n                    match_info[RESULT_FORMAT_RANGE][0] <= entity_range[0]\n                    and entity_range[1] <= match_info[RESULT_FORMAT_RANGE][1]\n                ):\n                    return match_name\n\n        return \"\"\n\n    def _extract_observable(self, observable: Observable, data: str) -> Dict:\n        list_matches = {}\n        if observable.detection_option == OBSERVABLE_DETECTION_CUSTOM_REGEX:\n            for regex in observable.regex:\n                for match in regex.finditer(data):\n                    match_value = match.group()\n\n                    ind_match = self._post_parse_observables(\n                        match_value, observable, match.span()\n                    )\n                    if ind_match:\n                        list_matches[match.group()] = ind_match\n\n        elif observable.detection_option == OBSERVABLE_DETECTION_LIBRARY:\n            lookup_function = self.library_lookup.get(observable.stix_target, None)\n            if not lookup_function:\n                self.helper.log_error(\n                    f\"Selected library function is not implemented: {observable.iocfinder_function}\"\n                )\n                return {}\n\n            matches = lookup_function(data)\n\n            for match in matches:\n                match_str = str(match)\n                if match_str in data:\n                    start = data.index(match_str)\n                elif match_str in data.lower():\n                    self.helper.log_debug(\n                        f\"External library manipulated the extracted value '{match_str}' from the \"\n                        f\"original text '{data}' to lower case\"\n                    )\n                    start = data.lower().index(match_str)\n                else:\n                    self.helper.log_error(\n                        f\"The extracted text '{match_str}' is not part of the original text '{data}'. \"\n                        f\"Please open a GitHub issue to report this problem!\"\n                    )\n                    continue\n\n                ind_match = self._post_parse_observables(\n                    match, observable, (start, len(match_str) + start)\n                )\n                if ind_match:\n                    list_matches[match] = ind_match\n\n        return list_matches\n\n    def _extract_entity(self, entity: Entity, list_matches: Dict, data: str) -> Dict:\n        regex_list = entity.regex\n\n        observable_keys = []\n        end_index = set()\n        match_dict = {}\n        match_key = \"\"\n\n        # Run all regexes for entity X\n        for regex in regex_list:\n            for match in regex.finditer(data):\n                match_key = match.group()\n                if match_key in match_dict:\n                    match_dict[match_key].append(match.span())\n                else:\n                    match_dict[match_key] = [match.span()]\n\n        # No maches for this entity\n        if len(match_dict) == 0:\n            return list_matches\n\n        # Run through all matches for entity X and check if they are part of a domain\n        # yes -> skip\n        # no -> add index to end_index\n        for match, match_indices in match_dict.items():\n            for match_index in match_indices:\n                skip_val = self._sco_present(\n                    list_matches, match_index, entity.omit_match_in\n                )\n                if skip_val:\n                    self.helper.log_debug(\n                        f\"Skipping Entity '{match}', it is part of an omitted field '{entity.omit_match_in}' \\\"{skip_val}\\\"\"\n                    )\n                else:\n                    self.helper.log_debug(\n                        f\"Entity match: '{match}' of regex: '{regex_list}'\"\n                    )\n                    end_index.add(match_index)\n                    if match in list_matches.keys():\n                        observable_keys.append(match)\n\n        # Remove all observables which found the same information/Entity match\n        for observable_key in observable_keys:\n            if observable_key in list_matches:\n                del list_matches[observable_key]\n                self.helper.log_debug(\n                    f\"Value {observable_key} is also matched by entity {entity.name}\"\n                )\n\n        # Check if entity was matched at least once in the text\n        # If yes, then add identity to match list\n        if end_index:\n            list_matches[match_key] = self._format_match(\n                ENTITY_CLASS, entity.name, entity.stix_id\n            )\n\n        return list_matches\n", "path": "internal-import-file/import-document/src/reportimporter/report_parser.py"}, {"content": "MIME_PDF = \"application/pdf\"\nMIME_TXT = \"text/plain\"\nMIME_HTML = \"text/html\"\nMIME_CSV = \"text/csv\"\n\nRESULT_FORMAT_TYPE = \"type\"\nRESULT_FORMAT_CATEGORY = \"category\"\nRESULT_FORMAT_MATCH = \"match\"\nRESULT_FORMAT_RANGE = \"range\"\n\nENTITY_CLASS = \"entity\"\nOBSERVABLE_CLASS = \"observable\"\n\nCONFIG_PATH = \"filter_list\"\nCOMMENT_INDICATOR = \"#\"\n\nOBSERVABLE_DETECTION_CUSTOM_REGEX = \"custom_regex\"\nOBSERVABLE_DETECTION_LIBRARY = \"library\"\nOBSERVABLE_DETECTION_OPTIONS = [\n    OBSERVABLE_DETECTION_LIBRARY,\n    OBSERVABLE_DETECTION_CUSTOM_REGEX,\n]\n", "path": "internal-import-file/import-document/src/reportimporter/constants.py"}], "after_files": [{"content": "import logging\nimport os\nimport io\nfrom typing import Dict, List, Pattern, IO, Tuple\n\nimport ioc_finder\nfrom bs4 import BeautifulSoup\nfrom pdfminer.high_level import extract_pages\nfrom pdfminer.layout import LTTextContainer\nfrom pycti import OpenCTIConnectorHelper\nfrom reportimporter.constants import (\n    OBSERVABLE_CLASS,\n    ENTITY_CLASS,\n    RESULT_FORMAT_MATCH,\n    RESULT_FORMAT_TYPE,\n    RESULT_FORMAT_CATEGORY,\n    RESULT_FORMAT_RANGE,\n    MIME_PDF,\n    MIME_TXT,\n    MIME_HTML,\n    MIME_CSV,\n    MIME_MD,\n    OBSERVABLE_DETECTION_CUSTOM_REGEX,\n    OBSERVABLE_DETECTION_LIBRARY,\n)\nfrom reportimporter.models import Observable, Entity\nfrom reportimporter.util import library_mapping\n\n\nclass ReportParser(object):\n    \"\"\"\n    Report parser based on IOCParser\n    \"\"\"\n\n    def __init__(\n        self,\n        helper: OpenCTIConnectorHelper,\n        entity_list: List[Entity],\n        observable_list: List[Observable],\n    ):\n\n        self.helper = helper\n        self.entity_list = entity_list\n        self.observable_list = observable_list\n\n        # Disable INFO logging by pdfminer\n        logging.getLogger(\"pdfminer\").setLevel(logging.WARNING)\n\n        # Supported file types\n        self.supported_file_types = {\n            MIME_PDF: self._parse_pdf,\n            MIME_TXT: self._parse_text,\n            MIME_HTML: self._parse_html,\n            MIME_CSV: self._parse_text,\n            MIME_MD: self._parse_text,\n        }\n\n        self.library_lookup = library_mapping()\n\n    def _is_whitelisted(self, regex_list: List[Pattern], ind_match: str):\n        for regex in regex_list:\n            self.helper.log_debug(f\"Filter regex '{regex}' for value '{ind_match}'\")\n            result = regex.search(ind_match)\n            if result:\n                self.helper.log_debug(f\"Value {ind_match} is whitelisted with {regex}\")\n                return True\n        return False\n\n    def _post_parse_observables(\n        self, ind_match: str, observable: Observable, match_range: Tuple\n    ) -> Dict:\n        self.helper.log_debug(f\"Observable match: {ind_match}\")\n\n        if self._is_whitelisted(observable.filter_regex, ind_match):\n            return {}\n\n        return self._format_match(\n            OBSERVABLE_CLASS, observable.stix_target, ind_match, match_range\n        )\n\n    def _parse(self, data: str) -> Dict[str, Dict]:\n        list_matches = {}\n\n        # Defang text\n        data = ioc_finder.prepare_text(data)\n\n        for observable in self.observable_list:\n            list_matches.update(self._extract_observable(observable, data))\n\n        for entity in self.entity_list:\n            list_matches = self._extract_entity(entity, list_matches, data)\n\n        self.helper.log_debug(f\"Text: '{data}' -> extracts {list_matches}\")\n        return list_matches\n\n    def _parse_pdf(self, file_data: IO) -> Dict[str, Dict]:\n        parse_info = {}\n        try:\n            for page_layout in extract_pages(file_data):\n                for element in page_layout:\n                    if isinstance(element, LTTextContainer):\n                        text = element.get_text()\n                        # Parsing with newlines has been deprecated\n                        no_newline_text = text.replace(\"\\n\", \"\")\n                        parse_info.update(self._parse(no_newline_text))\n\n                # TODO also extract information from images/figures using OCR\n                # https://pdfminersix.readthedocs.io/en/latest/topic/converting_pdf_to_text.html#topic-pdf-to-text-layout\n\n        except Exception as e:\n            logging.exception(f\"Pdf Parsing Error: {e}\")\n\n        return parse_info\n\n    def _parse_text(self, file_data: IO) -> Dict[str, Dict]:\n        parse_info = {}\n        for text in file_data.readlines():\n            parse_info.update(self._parse(text.decode(\"utf-8\")))\n        return parse_info\n\n    def _parse_html(self, file_data: IO) -> Dict[str, Dict]:\n        parse_info = {}\n        soup = BeautifulSoup(file_data, \"html.parser\")\n        buf = io.StringIO(soup.get_text())\n        for text in buf.readlines():\n            parse_info.update(self._parse(text))\n        return parse_info\n\n    def run_parser(self, file_path: str, file_type: str) -> List[Dict]:\n        parsing_results = []\n\n        file_parser = self.supported_file_types.get(file_type, None)\n        if not file_parser:\n            raise NotImplementedError(f\"No parser available for file type {file_type}\")\n\n        if not os.path.isfile(file_path):\n            raise IOError(f\"File path is not a file: {file_path}\")\n\n        self.helper.log_info(f\"Parsing report {file_path} {file_type}\")\n\n        try:\n            with open(file_path, \"rb\") as file_data:\n                parsing_results = file_parser(file_data)\n        except Exception as e:\n            logging.exception(f\"Parsing Error: {e}\")\n\n        parsing_results = list(parsing_results.values())\n\n        return parsing_results\n\n    @staticmethod\n    def _format_match(\n        format_type: str, category: str, match: str, match_range: Tuple = (0, 0)\n    ) -> Dict:\n        return {\n            RESULT_FORMAT_TYPE: format_type,\n            RESULT_FORMAT_CATEGORY: category,\n            RESULT_FORMAT_MATCH: match,\n            RESULT_FORMAT_RANGE: match_range,\n        }\n\n    @staticmethod\n    def _sco_present(\n        match_list: Dict, entity_range: Tuple, filter_sco_types: List\n    ) -> str:\n        for match_name, match_info in match_list.items():\n            if match_info[RESULT_FORMAT_CATEGORY] in filter_sco_types:\n                if (\n                    match_info[RESULT_FORMAT_RANGE][0] <= entity_range[0]\n                    and entity_range[1] <= match_info[RESULT_FORMAT_RANGE][1]\n                ):\n                    return match_name\n\n        return \"\"\n\n    def _extract_observable(self, observable: Observable, data: str) -> Dict:\n        list_matches = {}\n        if observable.detection_option == OBSERVABLE_DETECTION_CUSTOM_REGEX:\n            for regex in observable.regex:\n                for match in regex.finditer(data):\n                    match_value = match.group()\n\n                    ind_match = self._post_parse_observables(\n                        match_value, observable, match.span()\n                    )\n                    if ind_match:\n                        list_matches[match.group()] = ind_match\n\n        elif observable.detection_option == OBSERVABLE_DETECTION_LIBRARY:\n            lookup_function = self.library_lookup.get(observable.stix_target, None)\n            if not lookup_function:\n                self.helper.log_error(\n                    f\"Selected library function is not implemented: {observable.iocfinder_function}\"\n                )\n                return {}\n\n            matches = lookup_function(data)\n\n            for match in matches:\n                match_str = str(match)\n                if match_str in data:\n                    start = data.index(match_str)\n                elif match_str in data.lower():\n                    self.helper.log_debug(\n                        f\"External library manipulated the extracted value '{match_str}' from the \"\n                        f\"original text '{data}' to lower case\"\n                    )\n                    start = data.lower().index(match_str)\n                else:\n                    self.helper.log_error(\n                        f\"The extracted text '{match_str}' is not part of the original text '{data}'. \"\n                        f\"Please open a GitHub issue to report this problem!\"\n                    )\n                    continue\n\n                ind_match = self._post_parse_observables(\n                    match, observable, (start, len(match_str) + start)\n                )\n                if ind_match:\n                    list_matches[match] = ind_match\n\n        return list_matches\n\n    def _extract_entity(self, entity: Entity, list_matches: Dict, data: str) -> Dict:\n        regex_list = entity.regex\n\n        observable_keys = []\n        end_index = set()\n        match_dict = {}\n        match_key = \"\"\n\n        # Run all regexes for entity X\n        for regex in regex_list:\n            for match in regex.finditer(data):\n                match_key = match.group()\n                if match_key in match_dict:\n                    match_dict[match_key].append(match.span())\n                else:\n                    match_dict[match_key] = [match.span()]\n\n        # No maches for this entity\n        if len(match_dict) == 0:\n            return list_matches\n\n        # Run through all matches for entity X and check if they are part of a domain\n        # yes -> skip\n        # no -> add index to end_index\n        for match, match_indices in match_dict.items():\n            for match_index in match_indices:\n                skip_val = self._sco_present(\n                    list_matches, match_index, entity.omit_match_in\n                )\n                if skip_val:\n                    self.helper.log_debug(\n                        f\"Skipping Entity '{match}', it is part of an omitted field '{entity.omit_match_in}' \\\"{skip_val}\\\"\"\n                    )\n                else:\n                    self.helper.log_debug(\n                        f\"Entity match: '{match}' of regex: '{regex_list}'\"\n                    )\n                    end_index.add(match_index)\n                    if match in list_matches.keys():\n                        observable_keys.append(match)\n\n        # Remove all observables which found the same information/Entity match\n        for observable_key in observable_keys:\n            if observable_key in list_matches:\n                del list_matches[observable_key]\n                self.helper.log_debug(\n                    f\"Value {observable_key} is also matched by entity {entity.name}\"\n                )\n\n        # Check if entity was matched at least once in the text\n        # If yes, then add identity to match list\n        if end_index:\n            list_matches[match_key] = self._format_match(\n                ENTITY_CLASS, entity.name, entity.stix_id\n            )\n\n        return list_matches\n", "path": "internal-import-file/import-document/src/reportimporter/report_parser.py"}, {"content": "MIME_PDF = \"application/pdf\"\nMIME_TXT = \"text/plain\"\nMIME_HTML = \"text/html\"\nMIME_CSV = \"text/csv\"\nMIME_MD = \"text/markdown\"\n\nRESULT_FORMAT_TYPE = \"type\"\nRESULT_FORMAT_CATEGORY = \"category\"\nRESULT_FORMAT_MATCH = \"match\"\nRESULT_FORMAT_RANGE = \"range\"\n\nENTITY_CLASS = \"entity\"\nOBSERVABLE_CLASS = \"observable\"\n\nCONFIG_PATH = \"filter_list\"\nCOMMENT_INDICATOR = \"#\"\n\nOBSERVABLE_DETECTION_CUSTOM_REGEX = \"custom_regex\"\nOBSERVABLE_DETECTION_LIBRARY = \"library\"\nOBSERVABLE_DETECTION_OPTIONS = [\n    OBSERVABLE_DETECTION_LIBRARY,\n    OBSERVABLE_DETECTION_CUSTOM_REGEX,\n]\n", "path": "internal-import-file/import-document/src/reportimporter/constants.py"}]}