We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Whitespace removal during preprocessing
When splitting documents into passages based on wordcount and respecting sentence boundary we are missing a whitespace between sentences.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `haystack/preprocessor/preprocessor.py`
Content:
```
1 import logging
2 import re
3 from copy import deepcopy
4 from functools import partial, reduce
5 from itertools import chain
6 from typing import List, Optional, Generator, Set
7 
8 import nltk
9 from more_itertools import windowed
10 
11 from haystack.preprocessor.base import BasePreProcessor
12 
13 logger = logging.getLogger(__name__)
14 
15 
16 class PreProcessor(BasePreProcessor):
17     def __init__(
18         self,
19         clean_whitespace: Optional[bool] = True,
20         clean_header_footer: Optional[bool] = False,
21         clean_empty_lines: Optional[bool] = True,
22         split_by: Optional[str] = "word",
23         split_length: Optional[int] = 1000,
24         split_stride: Optional[int] = None,
25         split_respect_sentence_boundary: Optional[bool] = True,
26     ):
27         """
28         :param clean_header_footer: Use heuristic to remove footers and headers across different pages by searching
29                                      for the longest common string. This heuristic uses exact matches and therefore
30                                      works well for footers like "Copyright 2019 by XXX", but won't detect "Page 3 of 4"
31                                      or similar.
32         :param clean_whitespace: Strip whitespaces before or after each line in the text.
33         :param clean_empty_lines: Remove more than two empty lines in the text.
34         :param split_by: Unit for splitting the document. Can be "word", "sentence", or "passage". Set to None to disable splitting.
35         :param split_length: Max. number of the above split unit (e.g. words) that are allowed in one document. For instance, if n -> 10 & split_by ->
36                            "sentence", then each output document will have 10 sentences.
37         :param split_stride: Length of striding window over the splits. For example, if split_by -> `word`,
38                              split_length -> 5 & split_stride -> 2, then the splits would be like:
39                              [w1 w2 w3 w4 w5, w4 w5 w6 w7 w8, w7 w8 w10 w11 w12].
40                              Set the value to None to disable striding behaviour.
41         :param split_respect_sentence_boundary: Whether to split in partial sentences if split_by -> `word`. If set
42                                                 to True, the individual split will always have complete sentences &
43                                                 the number of words will be <= split_length.
44         """
45         nltk.download("punkt")
46         self.clean_whitespace = clean_whitespace
47         self.clean_header_footer = clean_header_footer
48         self.clean_empty_lines = clean_empty_lines
49         self.split_by = split_by
50         self.split_length = split_length
51         self.split_stride = split_stride
52         self.split_respect_sentence_boundary = split_respect_sentence_boundary
53 
54     def clean(self, document: dict) -> dict:
55         text = document["text"]
56         if self.clean_header_footer:
57             text = self._find_and_remove_header_footer(
58                 text, n_chars=300, n_first_pages_to_ignore=1, n_last_pages_to_ignore=1
59             )
60 
61         if self.clean_whitespace:
62             lines = text.splitlines()
63 
64             cleaned_lines = []
65             for line in lines:
66                 line = line.strip()
67                 cleaned_lines.append(line)
68             text = "\n".join(cleaned_lines)
69 
70         if self.clean_empty_lines:
71             text = re.sub(r"\n\n+", "\n\n", text)
72 
73         document["text"] = text
74         return document
75 
76     def split(self, document: dict) -> List[dict]:
77         if not self.split_by:
78             return [document]
79 
80         if not self.split_length:
81             raise Exception("split_length needs be set when using split_by.")
82 
83         if self.split_respect_sentence_boundary and self.split_by not in("word","sentence"):
84             raise NotImplementedError("'split_respect_sentence_boundary=True' is only compatible with"
85                                       " split_by='word' or split_by='sentence'.")
86 
87         text = document["text"]
88 
89         if self.split_respect_sentence_boundary and self.split_by == "word":
90             # split by words ensuring no sub sentence splits
91             sentences = nltk.tokenize.sent_tokenize(text)
92             word_count = 0
93             text_splits = []
94             current_slice = ""
95             for sen in sentences:
96                 current_word_count = len(sen.split(" "))
97                 if current_word_count > self.split_length:
98                     logger.warning(f"A sentence found with word count higher than the split length.")
99                 if word_count + current_word_count > self.split_length:
100                     text_splits.append(current_slice)
101                     current_slice = ""
102                     word_count = 0
103                 current_slice += sen
104                 word_count += len(sen.split(" "))
105             if current_slice:
106                 text_splits.append(current_slice)
107         else:
108             # create individual "elements" of passage, sentence, or word
109             if self.split_by == "passage":
110                 elements = text.split("\n\n")
111             elif self.split_by == "sentence":
112                 elements = nltk.tokenize.sent_tokenize(text)
113             elif self.split_by == "word":
114                 elements = text.split(" ")
115             else:
116                 raise NotImplementedError("PreProcessor only supports 'passage' or 'sentence' split_by options.")
117 
118             # concatenate individual elements based on split_length & split_stride
119             if self.split_stride:
120                 segments = windowed(elements, n=self.split_length, step=self.split_length - self.split_stride)
121             else:
122                 segments = windowed(elements, n=self.split_length, step=self.split_length)
123             text_splits = []
124             for seg in segments:
125                 txt = " ".join([t for t in seg if t])
126                 text_splits.append(txt)
127 
128         # create new document dicts for each text split
129         documents = []
130         for i, txt in enumerate(text_splits):
131             doc = deepcopy(document)
132             doc["text"] = txt
133             if "meta" not in doc.keys() or doc["meta"] is None:
134                 doc["meta"] = {}
135             doc["meta"]["_split_id"] = i
136             documents.append(doc)
137 
138         return documents
139 
140     def _find_and_remove_header_footer(
141         self, text: str, n_chars: int, n_first_pages_to_ignore: int, n_last_pages_to_ignore: int
142     ) -> str:
143         """
144         Heuristic to find footers and headers across different pages by searching for the longest common string.
145         For headers we only search in the first n_chars characters (for footer: last n_chars).
146         Note: This heuristic uses exact matches and therefore works well for footers like "Copyright 2019 by XXX",
147          but won't detect "Page 3 of 4" or similar.
148 
149         :param n_chars: number of first/last characters where the header/footer shall be searched in
150         :param n_first_pages_to_ignore: number of first pages to ignore (e.g. TOCs often don't contain footer/header)
151         :param n_last_pages_to_ignore: number of last pages to ignore
152         :return: (cleaned pages, found_header_str, found_footer_str)
153         """
154 
155         pages = text.split("\f")
156 
157         # header
158         start_of_pages = [p[:n_chars] for p in pages[n_first_pages_to_ignore:-n_last_pages_to_ignore]]
159         found_header = self._find_longest_common_ngram(start_of_pages)
160         if found_header:
161             pages = [page.replace(found_header, "") for page in pages]
162 
163         # footer
164         end_of_pages = [p[-n_chars:] for p in pages[n_first_pages_to_ignore:-n_last_pages_to_ignore]]
165         found_footer = self._find_longest_common_ngram(end_of_pages)
166         if found_footer:
167             pages = [page.replace(found_footer, "") for page in pages]
168         logger.debug(f"Removed header '{found_header}' and footer '{found_footer}' in document")
169         text = "\f".join(pages)
170         return text
171 
172     def _ngram(self, seq: str, n: int) -> Generator[str, None, None]:
173         """
174         Return ngram (of tokens - currently split by whitespace)
175         :param seq: str, string from which the ngram shall be created
176         :param n: int, n of ngram
177         :return: str, ngram as string
178         """
179 
180         # In order to maintain the original whitespace, but still consider \n and \t for n-gram tokenization,
181         # we add a space here and remove it after creation of the ngrams again (see below)
182         seq = seq.replace("\n", " \n")
183         seq = seq.replace("\t", " \t")
184 
185         words = seq.split(" ")
186         ngrams = (
187             " ".join(words[i : i + n]).replace(" \n", "\n").replace(" \t", "\t") for i in range(0, len(words) - n + 1)
188         )
189 
190         return ngrams
191 
192     def _allngram(self, seq: str, min_ngram: int, max_ngram: int) -> Set[str]:
193         lengths = range(min_ngram, max_ngram) if max_ngram else range(min_ngram, len(seq))
194         ngrams = map(partial(self._ngram, seq), lengths)
195         res = set(chain.from_iterable(ngrams))
196         return res
197 
198     def _find_longest_common_ngram(
199         self, sequences: List[str], max_ngram: int = 30, min_ngram: int = 3
200     ) -> Optional[str]:
201         """
202         Find the longest common ngram across different text sequences (e.g. start of pages).
203         Considering all ngrams between the specified range. Helpful for finding footers, headers etc.
204 
205         :param sequences: list[str], list of strings that shall be searched for common n_grams
206         :param max_ngram: int, maximum length of ngram to consider
207         :param min_ngram: minimum length of ngram to consider
208         :return: str, common string of all sections
209         """
210         sequences = [s for s in sequences if s]  # filter empty sequences
211         if not sequences:
212             return None
213         seqs_ngrams = map(partial(self._allngram, min_ngram=min_ngram, max_ngram=max_ngram), sequences)
214         intersection = reduce(set.intersection, seqs_ngrams)
215 
216         try:
217             longest = max(intersection, key=len)
218         except ValueError:
219             # no common sequence found
220             longest = ""
221         return longest if longest.strip() else None
222
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/haystack/preprocessor/preprocessor.py b/haystack/preprocessor/preprocessor.py
--- a/haystack/preprocessor/preprocessor.py
+++ b/haystack/preprocessor/preprocessor.py
@@ -100,8 +100,10 @@
                     text_splits.append(current_slice)
                     current_slice = ""
                     word_count = 0
+                if len(current_slice) != 0:
+                    sen = " " + sen
                 current_slice += sen
-                word_count += len(sen.split(" "))
+                word_count += current_word_count
             if current_slice:
                 text_splits.append(current_slice)
         else:

{"golden_diff": "diff --git a/haystack/preprocessor/preprocessor.py b/haystack/preprocessor/preprocessor.py\n--- a/haystack/preprocessor/preprocessor.py\n+++ b/haystack/preprocessor/preprocessor.py\n@@ -100,8 +100,10 @@\n                     text_splits.append(current_slice)\n                     current_slice = \"\"\n                     word_count = 0\n+                if len(current_slice) != 0:\n+                    sen = \" \" + sen\n                 current_slice += sen\n-                word_count += len(sen.split(\" \"))\n+                word_count += current_word_count\n             if current_slice:\n                 text_splits.append(current_slice)\n         else:\n", "issue": "Whitespace removal during preprocessing\nWhen splitting documents into passages based on wordcount and respecting sentence boundary we are missing a whitespace between sentences.\r\n\n", "before_files": [{"content": "import logging\nimport re\nfrom copy import deepcopy\nfrom functools import partial, reduce\nfrom itertools import chain\nfrom typing import List, Optional, Generator, Set\n\nimport nltk\nfrom more_itertools import windowed\n\nfrom haystack.preprocessor.base import BasePreProcessor\n\nlogger = logging.getLogger(__name__)\n\n\nclass PreProcessor(BasePreProcessor):\n    def __init__(\n        self,\n        clean_whitespace: Optional[bool] = True,\n        clean_header_footer: Optional[bool] = False,\n        clean_empty_lines: Optional[bool] = True,\n        split_by: Optional[str] = \"word\",\n        split_length: Optional[int] = 1000,\n        split_stride: Optional[int] = None,\n        split_respect_sentence_boundary: Optional[bool] = True,\n    ):\n        \"\"\"\n        :param clean_header_footer: Use heuristic to remove footers and headers across different pages by searching\n                                     for the longest common string. This heuristic uses exact matches and therefore\n                                     works well for footers like \"Copyright 2019 by XXX\", but won't detect \"Page 3 of 4\"\n                                     or similar.\n        :param clean_whitespace: Strip whitespaces before or after each line in the text.\n        :param clean_empty_lines: Remove more than two empty lines in the text.\n        :param split_by: Unit for splitting the document. Can be \"word\", \"sentence\", or \"passage\". Set to None to disable splitting.\n        :param split_length: Max. number of the above split unit (e.g. words) that are allowed in one document. For instance, if n -> 10 & split_by ->\n                           \"sentence\", then each output document will have 10 sentences.\n        :param split_stride: Length of striding window over the splits. For example, if split_by -> `word`,\n                             split_length -> 5 & split_stride -> 2, then the splits would be like:\n                             [w1 w2 w3 w4 w5, w4 w5 w6 w7 w8, w7 w8 w10 w11 w12].\n                             Set the value to None to disable striding behaviour.\n        :param split_respect_sentence_boundary: Whether to split in partial sentences if split_by -> `word`. If set\n                                                to True, the individual split will always have complete sentences &\n                                                the number of words will be <= split_length.\n        \"\"\"\n        nltk.download(\"punkt\")\n        self.clean_whitespace = clean_whitespace\n        self.clean_header_footer = clean_header_footer\n        self.clean_empty_lines = clean_empty_lines\n        self.split_by = split_by\n        self.split_length = split_length\n        self.split_stride = split_stride\n        self.split_respect_sentence_boundary = split_respect_sentence_boundary\n\n    def clean(self, document: dict) -> dict:\n        text = document[\"text\"]\n        if self.clean_header_footer:\n            text = self._find_and_remove_header_footer(\n                text, n_chars=300, n_first_pages_to_ignore=1, n_last_pages_to_ignore=1\n            )\n\n        if self.clean_whitespace:\n            lines = text.splitlines()\n\n            cleaned_lines = []\n            for line in lines:\n                line = line.strip()\n                cleaned_lines.append(line)\n            text = \"\\n\".join(cleaned_lines)\n\n        if self.clean_empty_lines:\n            text = re.sub(r\"\\n\\n+\", \"\\n\\n\", text)\n\n        document[\"text\"] = text\n        return document\n\n    def split(self, document: dict) -> List[dict]:\n        if not self.split_by:\n            return [document]\n\n        if not self.split_length:\n            raise Exception(\"split_length needs be set when using split_by.\")\n\n        if self.split_respect_sentence_boundary and self.split_by not in(\"word\",\"sentence\"):\n            raise NotImplementedError(\"'split_respect_sentence_boundary=True' is only compatible with\"\n                                      \" split_by='word' or split_by='sentence'.\")\n\n        text = document[\"text\"]\n\n        if self.split_respect_sentence_boundary and self.split_by == \"word\":\n            # split by words ensuring no sub sentence splits\n            sentences = nltk.tokenize.sent_tokenize(text)\n            word_count = 0\n            text_splits = []\n            current_slice = \"\"\n            for sen in sentences:\n                current_word_count = len(sen.split(\" \"))\n                if current_word_count > self.split_length:\n                    logger.warning(f\"A sentence found with word count higher than the split length.\")\n                if word_count + current_word_count > self.split_length:\n                    text_splits.append(current_slice)\n                    current_slice = \"\"\n                    word_count = 0\n                current_slice += sen\n                word_count += len(sen.split(\" \"))\n            if current_slice:\n                text_splits.append(current_slice)\n        else:\n            # create individual \"elements\" of passage, sentence, or word\n            if self.split_by == \"passage\":\n                elements = text.split(\"\\n\\n\")\n            elif self.split_by == \"sentence\":\n                elements = nltk.tokenize.sent_tokenize(text)\n            elif self.split_by == \"word\":\n                elements = text.split(\" \")\n            else:\n                raise NotImplementedError(\"PreProcessor only supports 'passage' or 'sentence' split_by options.\")\n\n            # concatenate individual elements based on split_length & split_stride\n            if self.split_stride:\n                segments = windowed(elements, n=self.split_length, step=self.split_length - self.split_stride)\n            else:\n                segments = windowed(elements, n=self.split_length, step=self.split_length)\n            text_splits = []\n            for seg in segments:\n                txt = \" \".join([t for t in seg if t])\n                text_splits.append(txt)\n\n        # create new document dicts for each text split\n        documents = []\n        for i, txt in enumerate(text_splits):\n            doc = deepcopy(document)\n            doc[\"text\"] = txt\n            if \"meta\" not in doc.keys() or doc[\"meta\"] is None:\n                doc[\"meta\"] = {}\n            doc[\"meta\"][\"_split_id\"] = i\n            documents.append(doc)\n\n        return documents\n\n    def _find_and_remove_header_footer(\n        self, text: str, n_chars: int, n_first_pages_to_ignore: int, n_last_pages_to_ignore: int\n    ) -> str:\n        \"\"\"\n        Heuristic to find footers and headers across different pages by searching for the longest common string.\n        For headers we only search in the first n_chars characters (for footer: last n_chars).\n        Note: This heuristic uses exact matches and therefore works well for footers like \"Copyright 2019 by XXX\",\n         but won't detect \"Page 3 of 4\" or similar.\n\n        :param n_chars: number of first/last characters where the header/footer shall be searched in\n        :param n_first_pages_to_ignore: number of first pages to ignore (e.g. TOCs often don't contain footer/header)\n        :param n_last_pages_to_ignore: number of last pages to ignore\n        :return: (cleaned pages, found_header_str, found_footer_str)\n        \"\"\"\n\n        pages = text.split(\"\\f\")\n\n        # header\n        start_of_pages = [p[:n_chars] for p in pages[n_first_pages_to_ignore:-n_last_pages_to_ignore]]\n        found_header = self._find_longest_common_ngram(start_of_pages)\n        if found_header:\n            pages = [page.replace(found_header, \"\") for page in pages]\n\n        # footer\n        end_of_pages = [p[-n_chars:] for p in pages[n_first_pages_to_ignore:-n_last_pages_to_ignore]]\n        found_footer = self._find_longest_common_ngram(end_of_pages)\n        if found_footer:\n            pages = [page.replace(found_footer, \"\") for page in pages]\n        logger.debug(f\"Removed header '{found_header}' and footer '{found_footer}' in document\")\n        text = \"\\f\".join(pages)\n        return text\n\n    def _ngram(self, seq: str, n: int) -> Generator[str, None, None]:\n        \"\"\"\n        Return ngram (of tokens - currently split by whitespace)\n        :param seq: str, string from which the ngram shall be created\n        :param n: int, n of ngram\n        :return: str, ngram as string\n        \"\"\"\n\n        # In order to maintain the original whitespace, but still consider \\n and \\t for n-gram tokenization,\n        # we add a space here and remove it after creation of the ngrams again (see below)\n        seq = seq.replace(\"\\n\", \" \\n\")\n        seq = seq.replace(\"\\t\", \" \\t\")\n\n        words = seq.split(\" \")\n        ngrams = (\n            \" \".join(words[i : i + n]).replace(\" \\n\", \"\\n\").replace(\" \\t\", \"\\t\") for i in range(0, len(words) - n + 1)\n        )\n\n        return ngrams\n\n    def _allngram(self, seq: str, min_ngram: int, max_ngram: int) -> Set[str]:\n        lengths = range(min_ngram, max_ngram) if max_ngram else range(min_ngram, len(seq))\n        ngrams = map(partial(self._ngram, seq), lengths)\n        res = set(chain.from_iterable(ngrams))\n        return res\n\n    def _find_longest_common_ngram(\n        self, sequences: List[str], max_ngram: int = 30, min_ngram: int = 3\n    ) -> Optional[str]:\n        \"\"\"\n        Find the longest common ngram across different text sequences (e.g. start of pages).\n        Considering all ngrams between the specified range. Helpful for finding footers, headers etc.\n\n        :param sequences: list[str], list of strings that shall be searched for common n_grams\n        :param max_ngram: int, maximum length of ngram to consider\n        :param min_ngram: minimum length of ngram to consider\n        :return: str, common string of all sections\n        \"\"\"\n        sequences = [s for s in sequences if s]  # filter empty sequences\n        if not sequences:\n            return None\n        seqs_ngrams = map(partial(self._allngram, min_ngram=min_ngram, max_ngram=max_ngram), sequences)\n        intersection = reduce(set.intersection, seqs_ngrams)\n\n        try:\n            longest = max(intersection, key=len)\n        except ValueError:\n            # no common sequence found\n            longest = \"\"\n        return longest if longest.strip() else None\n", "path": "haystack/preprocessor/preprocessor.py"}], "after_files": [{"content": "import logging\nimport re\nfrom copy import deepcopy\nfrom functools import partial, reduce\nfrom itertools import chain\nfrom typing import List, Optional, Generator, Set\n\nimport nltk\nfrom more_itertools import windowed\n\nfrom haystack.preprocessor.base import BasePreProcessor\n\nlogger = logging.getLogger(__name__)\n\n\nclass PreProcessor(BasePreProcessor):\n    def __init__(\n        self,\n        clean_whitespace: Optional[bool] = True,\n        clean_header_footer: Optional[bool] = False,\n        clean_empty_lines: Optional[bool] = True,\n        split_by: Optional[str] = \"word\",\n        split_length: Optional[int] = 1000,\n        split_stride: Optional[int] = None,\n        split_respect_sentence_boundary: Optional[bool] = True,\n    ):\n        \"\"\"\n        :param clean_header_footer: Use heuristic to remove footers and headers across different pages by searching\n                                     for the longest common string. This heuristic uses exact matches and therefore\n                                     works well for footers like \"Copyright 2019 by XXX\", but won't detect \"Page 3 of 4\"\n                                     or similar.\n        :param clean_whitespace: Strip whitespaces before or after each line in the text.\n        :param clean_empty_lines: Remove more than two empty lines in the text.\n        :param split_by: Unit for splitting the document. Can be \"word\", \"sentence\", or \"passage\". Set to None to disable splitting.\n        :param split_length: Max. number of the above split unit (e.g. words) that are allowed in one document. For instance, if n -> 10 & split_by ->\n                           \"sentence\", then each output document will have 10 sentences.\n        :param split_stride: Length of striding window over the splits. For example, if split_by -> `word`,\n                             split_length -> 5 & split_stride -> 2, then the splits would be like:\n                             [w1 w2 w3 w4 w5, w4 w5 w6 w7 w8, w7 w8 w10 w11 w12].\n                             Set the value to None to disable striding behaviour.\n        :param split_respect_sentence_boundary: Whether to split in partial sentences if split_by -> `word`. If set\n                                                to True, the individual split will always have complete sentences &\n                                                the number of words will be <= split_length.\n        \"\"\"\n        nltk.download(\"punkt\")\n        self.clean_whitespace = clean_whitespace\n        self.clean_header_footer = clean_header_footer\n        self.clean_empty_lines = clean_empty_lines\n        self.split_by = split_by\n        self.split_length = split_length\n        self.split_stride = split_stride\n        self.split_respect_sentence_boundary = split_respect_sentence_boundary\n\n    def clean(self, document: dict) -> dict:\n        text = document[\"text\"]\n        if self.clean_header_footer:\n            text = self._find_and_remove_header_footer(\n                text, n_chars=300, n_first_pages_to_ignore=1, n_last_pages_to_ignore=1\n            )\n\n        if self.clean_whitespace:\n            lines = text.splitlines()\n\n            cleaned_lines = []\n            for line in lines:\n                line = line.strip()\n                cleaned_lines.append(line)\n            text = \"\\n\".join(cleaned_lines)\n\n        if self.clean_empty_lines:\n            text = re.sub(r\"\\n\\n+\", \"\\n\\n\", text)\n\n        document[\"text\"] = text\n        return document\n\n    def split(self, document: dict) -> List[dict]:\n        if not self.split_by:\n            return [document]\n\n        if not self.split_length:\n            raise Exception(\"split_length needs be set when using split_by.\")\n\n        if self.split_respect_sentence_boundary and self.split_by not in(\"word\",\"sentence\"):\n            raise NotImplementedError(\"'split_respect_sentence_boundary=True' is only compatible with\"\n                                      \" split_by='word' or split_by='sentence'.\")\n\n        text = document[\"text\"]\n\n        if self.split_respect_sentence_boundary and self.split_by == \"word\":\n            # split by words ensuring no sub sentence splits\n            sentences = nltk.tokenize.sent_tokenize(text)\n            word_count = 0\n            text_splits = []\n            current_slice = \"\"\n            for sen in sentences:\n                current_word_count = len(sen.split(\" \"))\n                if current_word_count > self.split_length:\n                    logger.warning(f\"A sentence found with word count higher than the split length.\")\n                if word_count + current_word_count > self.split_length:\n                    text_splits.append(current_slice)\n                    current_slice = \"\"\n                    word_count = 0\n                if len(current_slice) != 0:\n                    sen = \" \" + sen\n                current_slice += sen\n                word_count += current_word_count\n            if current_slice:\n                text_splits.append(current_slice)\n        else:\n            # create individual \"elements\" of passage, sentence, or word\n            if self.split_by == \"passage\":\n                elements = text.split(\"\\n\\n\")\n            elif self.split_by == \"sentence\":\n                elements = nltk.tokenize.sent_tokenize(text)\n            elif self.split_by == \"word\":\n                elements = text.split(\" \")\n            else:\n                raise NotImplementedError(\"PreProcessor only supports 'passage' or 'sentence' split_by options.\")\n\n            # concatenate individual elements based on split_length & split_stride\n            if self.split_stride:\n                segments = windowed(elements, n=self.split_length, step=self.split_length - self.split_stride)\n            else:\n                segments = windowed(elements, n=self.split_length, step=self.split_length)\n            text_splits = []\n            for seg in segments:\n                txt = \" \".join([t for t in seg if t])\n                text_splits.append(txt)\n\n        # create new document dicts for each text split\n        documents = []\n        for i, txt in enumerate(text_splits):\n            doc = deepcopy(document)\n            doc[\"text\"] = txt\n            if \"meta\" not in doc.keys() or doc[\"meta\"] is None:\n                doc[\"meta\"] = {}\n            doc[\"meta\"][\"_split_id\"] = i\n            documents.append(doc)\n\n        return documents\n\n    def _find_and_remove_header_footer(\n        self, text: str, n_chars: int, n_first_pages_to_ignore: int, n_last_pages_to_ignore: int\n    ) -> str:\n        \"\"\"\n        Heuristic to find footers and headers across different pages by searching for the longest common string.\n        For headers we only search in the first n_chars characters (for footer: last n_chars).\n        Note: This heuristic uses exact matches and therefore works well for footers like \"Copyright 2019 by XXX\",\n         but won't detect \"Page 3 of 4\" or similar.\n\n        :param n_chars: number of first/last characters where the header/footer shall be searched in\n        :param n_first_pages_to_ignore: number of first pages to ignore (e.g. TOCs often don't contain footer/header)\n        :param n_last_pages_to_ignore: number of last pages to ignore\n        :return: (cleaned pages, found_header_str, found_footer_str)\n        \"\"\"\n\n        pages = text.split(\"\\f\")\n\n        # header\n        start_of_pages = [p[:n_chars] for p in pages[n_first_pages_to_ignore:-n_last_pages_to_ignore]]\n        found_header = self._find_longest_common_ngram(start_of_pages)\n        if found_header:\n            pages = [page.replace(found_header, \"\") for page in pages]\n\n        # footer\n        end_of_pages = [p[-n_chars:] for p in pages[n_first_pages_to_ignore:-n_last_pages_to_ignore]]\n        found_footer = self._find_longest_common_ngram(end_of_pages)\n        if found_footer:\n            pages = [page.replace(found_footer, \"\") for page in pages]\n        logger.debug(f\"Removed header '{found_header}' and footer '{found_footer}' in document\")\n        text = \"\\f\".join(pages)\n        return text\n\n    def _ngram(self, seq: str, n: int) -> Generator[str, None, None]:\n        \"\"\"\n        Return ngram (of tokens - currently split by whitespace)\n        :param seq: str, string from which the ngram shall be created\n        :param n: int, n of ngram\n        :return: str, ngram as string\n        \"\"\"\n\n        # In order to maintain the original whitespace, but still consider \\n and \\t for n-gram tokenization,\n        # we add a space here and remove it after creation of the ngrams again (see below)\n        seq = seq.replace(\"\\n\", \" \\n\")\n        seq = seq.replace(\"\\t\", \" \\t\")\n\n        words = seq.split(\" \")\n        ngrams = (\n            \" \".join(words[i : i + n]).replace(\" \\n\", \"\\n\").replace(\" \\t\", \"\\t\") for i in range(0, len(words) - n + 1)\n        )\n\n        return ngrams\n\n    def _allngram(self, seq: str, min_ngram: int, max_ngram: int) -> Set[str]:\n        lengths = range(min_ngram, max_ngram) if max_ngram else range(min_ngram, len(seq))\n        ngrams = map(partial(self._ngram, seq), lengths)\n        res = set(chain.from_iterable(ngrams))\n        return res\n\n    def _find_longest_common_ngram(\n        self, sequences: List[str], max_ngram: int = 30, min_ngram: int = 3\n    ) -> Optional[str]:\n        \"\"\"\n        Find the longest common ngram across different text sequences (e.g. start of pages).\n        Considering all ngrams between the specified range. Helpful for finding footers, headers etc.\n\n        :param sequences: list[str], list of strings that shall be searched for common n_grams\n        :param max_ngram: int, maximum length of ngram to consider\n        :param min_ngram: minimum length of ngram to consider\n        :return: str, common string of all sections\n        \"\"\"\n        sequences = [s for s in sequences if s]  # filter empty sequences\n        if not sequences:\n            return None\n        seqs_ngrams = map(partial(self._allngram, min_ngram=min_ngram, max_ngram=max_ngram), sequences)\n        intersection = reduce(set.intersection, seqs_ngrams)\n\n        try:\n            longest = max(intersection, key=len)\n        except ValueError:\n            # no common sequence found\n            longest = \"\"\n        return longest if longest.strip() else None\n", "path": "haystack/preprocessor/preprocessor.py"}]}