prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Make NLP dependencies optional
  Some dependencies are only needed in some images, we should make some dependencies optional so that they are only installed when necessary.
  
  For example, Spacy is the biggest dependency, but it is currently not really used by any image.
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `setup.py`
  Content:
  ```
  1 # -*- coding: utf-8 -*-
  2 # This Source Code Form is subject to the terms of the Mozilla Public
  3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
  4 # You can obtain one at http://mozilla.org/MPL/2.0/.
  5 
  6 import os
  7 
  8 from setuptools import find_packages, setup
  9 
  10 here = os.path.dirname(__file__)
  11 
  12 
  13 def read_requirements(file_):
  14     requires = []
  15     links = []
  16     with open(os.path.join(here, file_)) as f:
  17         for line in f.readlines():
  18             line = line.strip()
  19 
  20             if line.startswith("https://"):
  21                 links.append(line + "-1.0.0")
  22                 extras = ""
  23                 if "[" in line:
  24                     extras = "[" + line.split("[")[1].split("]")[0] + "]"
  25                 line = line.split("#")[1].split("egg=")[1] + extras
  26             elif line == "" or line.startswith("#") or line.startswith("-"):
  27                 continue
  28             line = line.split("#")[0].strip()
  29             requires.append(line)
  30 
  31     return sorted(list(set(requires))), links
  32 
  33 
  34 install_requires, dependency_links = read_requirements("requirements.txt")
  35 
  36 
  37 with open(os.path.join(here, "VERSION")) as f:
  38     version = f.read().strip()
  39 
  40 setup(
  41     name="bugbug",
  42     version=version,
  43     description="ML tools for Mozilla projects",
  44     author="Marco Castelluccio",
  45     author_email="mcastelluccio@mozilla.com",
  46     install_requires=install_requires,
  47     dependency_links=dependency_links,
  48     packages=find_packages(exclude=["contrib", "docs", "tests"]),
  49     include_package_data=True,
  50     license="MPL2",
  51     entry_points={
  52         "console_scripts": [
  53             "bugbug-data-commits = scripts.commit_retriever:main",
  54             "bugbug-data-bugzilla = scripts.bug_retriever:main",
  55             "bugbug-train = scripts.trainer:main",
  56         ]
  57     },
  58 )
  59
  ```
  
  Path: `bugbug/nlp.py`
  Content:
  ```
  1 # -*- coding: utf-8 -*-
  2 # This Source Code Form is subject to the terms of the Mozilla Public
  3 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
  4 # You can obtain one at http://mozilla.org/MPL/2.0/.
  5 
  6 from collections import defaultdict
  7 from functools import lru_cache
  8 
  9 import numpy as np
  10 import spacy
  11 from gensim.models import KeyedVectors
  12 from sklearn.base import BaseEstimator, TransformerMixin
  13 from sklearn.feature_extraction.text import TfidfVectorizer
  14 from spacy.tokenizer import Tokenizer
  15 
  16 nlp = spacy.load("en_core_web_sm")
  17 
  18 
  19 def spacy_token_lemmatizer(text):
  20     if len(text) > nlp.max_length:
  21         text = text[: nlp.max_length - 1]
  22     doc = nlp(text)
  23     return [token.lemma_ for token in doc]
  24 
  25 
  26 class SpacyVectorizer(TfidfVectorizer):
  27     def __init__(self, *args, **kwargs):
  28         super().__init__(tokenizer=spacy_token_lemmatizer, *args, **kwargs)
  29 
  30 
  31 @lru_cache()
  32 def get_word_embeddings():
  33     word_embeddings = KeyedVectors.load_word2vec_format("wiki-news-300d-1M-subword.vec")
  34     word_embeddings.init_sims(replace=True)
  35     return word_embeddings
  36 
  37 
  38 class MeanEmbeddingTransformer(BaseEstimator, TransformerMixin):
  39     def __init__(self):
  40         self.model = get_word_embeddings()
  41         self.dim = len(self.model["if"])
  42 
  43     def fit(self, x, y=None):
  44         return self
  45 
  46     def transform(self, data):
  47         tokenizer = Tokenizer(nlp.vocab)
  48         return np.array(
  49             [
  50                 np.mean(
  51                     [
  52                         self.model[w.text.lower()]
  53                         for w in words
  54                         if w.text.lower() in self.model
  55                     ]
  56                     or [np.zeros(self.dim)],
  57                     axis=0,
  58                 )
  59                 for words in tokenizer.pipe(data)
  60             ]
  61         )
  62 
  63     def get_feature_names(self):
  64         return np.array([f"_{i}" for i in range(self.dim)], dtype=object)
  65 
  66 
  67 class TfidfMeanEmbeddingTransformer(MeanEmbeddingTransformer):
  68     def __init__(self):
  69         super().__init__()
  70         self.word2weight = None
  71 
  72     def fit(self, X, y=None):
  73         tfidf = TfidfVectorizer(analyzer=lambda x: x)
  74         tfidf.fit(X)
  75 
  76         # If a word was never seen, it must be at least as infrequent as any of the known words.
  77         # So, the default idf is the max of known idfs.
  78         max_idf = max(tfidf.idf_)
  79         self.word2weight = defaultdict(
  80             lambda: max_idf, [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()]
  81         )
  82 
  83         return self
  84 
  85     def transform(self, data):
  86         tokenizer = Tokenizer(nlp.vocab)
  87         return np.array(
  88             [
  89                 np.mean(
  90                     [
  91                         self.model[w.text.lower()] * self.word2weight[w.text.lower()]
  92                         for w in words
  93                         if w.text.lower() in self.model
  94                     ]
  95                     or [np.zeros(self.dim)],
  96                     axis=0,
  97                 )
  98                 for words in tokenizer.pipe(data)
  99             ]
  100         )
  101
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/bugbug/nlp.py b/bugbug/nlp.py
  --- a/bugbug/nlp.py
  +++ b/bugbug/nlp.py
  @@ -3,17 +3,41 @@
   # License, v. 2.0. If a copy of the MPL was not distributed with this file,
   # You can obtain one at http://mozilla.org/MPL/2.0/.
   
  +import sys
   from collections import defaultdict
   from functools import lru_cache
   
   import numpy as np
  -import spacy
  -from gensim.models import KeyedVectors
   from sklearn.base import BaseEstimator, TransformerMixin
   from sklearn.feature_extraction.text import TfidfVectorizer
  -from spacy.tokenizer import Tokenizer
   
  -nlp = spacy.load("en_core_web_sm")
  +HAS_OPTIONAL_DEPENDENCIES = False
  +
  +try:
  +    import spacy
  +
  +    from spacy.tokenizer import Tokenizer
  +    from gensim.models import KeyedVectors
  +
  +    HAS_OPTIONAL_DEPENDENCIES = True
  +except ImportError:
  +    pass
  +
  +try:
  +    if HAS_OPTIONAL_DEPENDENCIES:
  +        nlp = spacy.load("en_core_web_sm")
  +except OSError:
  +    msg = (
  +        "Spacy model is missing, install it with: "
  +        f"{sys.executable} -m spacy download en_core_web_sm"
  +    )
  +    print(msg, file=sys.stderr)
  +
  +OPT_MSG_MISSING = (
  +    "Optional dependencies are missing, install them with: pip install bugbug[nlp]\n"
  +    "You might need also to download the models with: "
  +    f"{sys.executable} -m spacy download en_core_web_sm"
  +)
   
   
   def spacy_token_lemmatizer(text):
  @@ -25,6 +49,11 @@
   
   class SpacyVectorizer(TfidfVectorizer):
       def __init__(self, *args, **kwargs):
  +
  +        # Detect when the Spacy optional dependency is missing
  +        if not HAS_OPTIONAL_DEPENDENCIES:
  +            raise NotImplementedError(OPT_MSG_MISSING)
  +
           super().__init__(tokenizer=spacy_token_lemmatizer, *args, **kwargs)
   
   
  @@ -37,6 +66,10 @@
   
   class MeanEmbeddingTransformer(BaseEstimator, TransformerMixin):
       def __init__(self):
  +        # Detect when the Gensim optional dependency are missing
  +        if not HAS_OPTIONAL_DEPENDENCIES:
  +            raise NotImplementedError(OPT_MSG_MISSING)
  +
           self.model = get_word_embeddings()
           self.dim = len(self.model["if"])
   
  diff --git a/setup.py b/setup.py
  --- a/setup.py
  +++ b/setup.py
  @@ -37,6 +37,20 @@
   with open(os.path.join(here, "VERSION")) as f:
       version = f.read().strip()
   
  +# Read the extra requirements
  +extras = ["nlp"]
  +
  +extras_require = {}
  +
  +for extra in extras:
  +    extras_install, extra_links = read_requirements("extra-%s-requirements.txt" % extra)
  +
  +    # Merge the dependency links
  +    dependency_links.extend(extra_links)
  +
  +    extras_require[extra] = extras_install
  +
  +
   setup(
       name="bugbug",
       version=version,
  @@ -44,6 +58,7 @@
       author="Marco Castelluccio",
       author_email="mcastelluccio@mozilla.com",
       install_requires=install_requires,
  +    extras_require=extras_require,
       dependency_links=dependency_links,
       packages=find_packages(exclude=["contrib", "docs", "tests"]),
       include_package_data=True,
  @@ -55,4 +70,9 @@
               "bugbug-train = scripts.trainer:main",
           ]
       },
  +    classifiers=[
  +        "Programming Language :: Python :: 3.6",
  +        "Programming Language :: Python :: 3 :: Only",
  +        "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
  +    ],
   )
  
verification_info: {"golden_diff": "diff --git a/bugbug/nlp.py b/bugbug/nlp.py\n--- a/bugbug/nlp.py\n+++ b/bugbug/nlp.py\n@@ -3,17 +3,41 @@\n # License, v. 2.0. If a copy of the MPL was not distributed with this file,\n # You can obtain one at http://mozilla.org/MPL/2.0/.\n \n+import sys\n from collections import defaultdict\n from functools import lru_cache\n \n import numpy as np\n-import spacy\n-from gensim.models import KeyedVectors\n from sklearn.base import BaseEstimator, TransformerMixin\n from sklearn.feature_extraction.text import TfidfVectorizer\n-from spacy.tokenizer import Tokenizer\n \n-nlp = spacy.load(\"en_core_web_sm\")\n+HAS_OPTIONAL_DEPENDENCIES = False\n+\n+try:\n+    import spacy\n+\n+    from spacy.tokenizer import Tokenizer\n+    from gensim.models import KeyedVectors\n+\n+    HAS_OPTIONAL_DEPENDENCIES = True\n+except ImportError:\n+    pass\n+\n+try:\n+    if HAS_OPTIONAL_DEPENDENCIES:\n+        nlp = spacy.load(\"en_core_web_sm\")\n+except OSError:\n+    msg = (\n+        \"Spacy model is missing, install it with: \"\n+        f\"{sys.executable} -m spacy download en_core_web_sm\"\n+    )\n+    print(msg, file=sys.stderr)\n+\n+OPT_MSG_MISSING = (\n+    \"Optional dependencies are missing, install them with: pip install bugbug[nlp]\\n\"\n+    \"You might need also to download the models with: \"\n+    f\"{sys.executable} -m spacy download en_core_web_sm\"\n+)\n \n \n def spacy_token_lemmatizer(text):\n@@ -25,6 +49,11 @@\n \n class SpacyVectorizer(TfidfVectorizer):\n     def __init__(self, *args, **kwargs):\n+\n+        # Detect when the Spacy optional dependency is missing\n+        if not HAS_OPTIONAL_DEPENDENCIES:\n+            raise NotImplementedError(OPT_MSG_MISSING)\n+\n         super().__init__(tokenizer=spacy_token_lemmatizer, *args, **kwargs)\n \n \n@@ -37,6 +66,10 @@\n \n class MeanEmbeddingTransformer(BaseEstimator, TransformerMixin):\n     def __init__(self):\n+        # Detect when the Gensim optional dependency are missing\n+        if not HAS_OPTIONAL_DEPENDENCIES:\n+            raise NotImplementedError(OPT_MSG_MISSING)\n+\n         self.model = get_word_embeddings()\n         self.dim = len(self.model[\"if\"])\n \ndiff --git a/setup.py b/setup.py\n--- a/setup.py\n+++ b/setup.py\n@@ -37,6 +37,20 @@\n with open(os.path.join(here, \"VERSION\")) as f:\n     version = f.read().strip()\n \n+# Read the extra requirements\n+extras = [\"nlp\"]\n+\n+extras_require = {}\n+\n+for extra in extras:\n+    extras_install, extra_links = read_requirements(\"extra-%s-requirements.txt\" % extra)\n+\n+    # Merge the dependency links\n+    dependency_links.extend(extra_links)\n+\n+    extras_require[extra] = extras_install\n+\n+\n setup(\n     name=\"bugbug\",\n     version=version,\n@@ -44,6 +58,7 @@\n     author=\"Marco Castelluccio\",\n     author_email=\"mcastelluccio@mozilla.com\",\n     install_requires=install_requires,\n+    extras_require=extras_require,\n     dependency_links=dependency_links,\n     packages=find_packages(exclude=[\"contrib\", \"docs\", \"tests\"]),\n     include_package_data=True,\n@@ -55,4 +70,9 @@\n             \"bugbug-train = scripts.trainer:main\",\n         ]\n     },\n+    classifiers=[\n+        \"Programming Language :: Python :: 3.6\",\n+        \"Programming Language :: Python :: 3 :: Only\",\n+        \"License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)\",\n+    ],\n )\n", "issue": "Make NLP dependencies optional\nSome dependencies are only needed in some images, we should make some dependencies optional so that they are only installed when necessary.\r\n\r\nFor example, Spacy is the biggest dependency, but it is currently not really used by any image.\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport os\n\nfrom setuptools import find_packages, setup\n\nhere = os.path.dirname(__file__)\n\n\ndef read_requirements(file_):\n    requires = []\n    links = []\n    with open(os.path.join(here, file_)) as f:\n        for line in f.readlines():\n            line = line.strip()\n\n            if line.startswith(\"https://\"):\n                links.append(line + \"-1.0.0\")\n                extras = \"\"\n                if \"[\" in line:\n                    extras = \"[\" + line.split(\"[\")[1].split(\"]\")[0] + \"]\"\n                line = line.split(\"#\")[1].split(\"egg=\")[1] + extras\n            elif line == \"\" or line.startswith(\"#\") or line.startswith(\"-\"):\n                continue\n            line = line.split(\"#\")[0].strip()\n            requires.append(line)\n\n    return sorted(list(set(requires))), links\n\n\ninstall_requires, dependency_links = read_requirements(\"requirements.txt\")\n\n\nwith open(os.path.join(here, \"VERSION\")) as f:\n    version = f.read().strip()\n\nsetup(\n    name=\"bugbug\",\n    version=version,\n    description=\"ML tools for Mozilla projects\",\n    author=\"Marco Castelluccio\",\n    author_email=\"mcastelluccio@mozilla.com\",\n    install_requires=install_requires,\n    dependency_links=dependency_links,\n    packages=find_packages(exclude=[\"contrib\", \"docs\", \"tests\"]),\n    include_package_data=True,\n    license=\"MPL2\",\n    entry_points={\n        \"console_scripts\": [\n            \"bugbug-data-commits = scripts.commit_retriever:main\",\n            \"bugbug-data-bugzilla = scripts.bug_retriever:main\",\n            \"bugbug-train = scripts.trainer:main\",\n        ]\n    },\n)\n", "path": "setup.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nfrom collections import defaultdict\nfrom functools import lru_cache\n\nimport numpy as np\nimport spacy\nfrom gensim.models import KeyedVectors\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom spacy.tokenizer import Tokenizer\n\nnlp = spacy.load(\"en_core_web_sm\")\n\n\ndef spacy_token_lemmatizer(text):\n    if len(text) > nlp.max_length:\n        text = text[: nlp.max_length - 1]\n    doc = nlp(text)\n    return [token.lemma_ for token in doc]\n\n\nclass SpacyVectorizer(TfidfVectorizer):\n    def __init__(self, *args, **kwargs):\n        super().__init__(tokenizer=spacy_token_lemmatizer, *args, **kwargs)\n\n\n@lru_cache()\ndef get_word_embeddings():\n    word_embeddings = KeyedVectors.load_word2vec_format(\"wiki-news-300d-1M-subword.vec\")\n    word_embeddings.init_sims(replace=True)\n    return word_embeddings\n\n\nclass MeanEmbeddingTransformer(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        self.model = get_word_embeddings()\n        self.dim = len(self.model[\"if\"])\n\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, data):\n        tokenizer = Tokenizer(nlp.vocab)\n        return np.array(\n            [\n                np.mean(\n                    [\n                        self.model[w.text.lower()]\n                        for w in words\n                        if w.text.lower() in self.model\n                    ]\n                    or [np.zeros(self.dim)],\n                    axis=0,\n                )\n                for words in tokenizer.pipe(data)\n            ]\n        )\n\n    def get_feature_names(self):\n        return np.array([f\"_{i}\" for i in range(self.dim)], dtype=object)\n\n\nclass TfidfMeanEmbeddingTransformer(MeanEmbeddingTransformer):\n    def __init__(self):\n        super().__init__()\n        self.word2weight = None\n\n    def fit(self, X, y=None):\n        tfidf = TfidfVectorizer(analyzer=lambda x: x)\n        tfidf.fit(X)\n\n        # If a word was never seen, it must be at least as infrequent as any of the known words.\n        # So, the default idf is the max of known idfs.\n        max_idf = max(tfidf.idf_)\n        self.word2weight = defaultdict(\n            lambda: max_idf, [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()]\n        )\n\n        return self\n\n    def transform(self, data):\n        tokenizer = Tokenizer(nlp.vocab)\n        return np.array(\n            [\n                np.mean(\n                    [\n                        self.model[w.text.lower()] * self.word2weight[w.text.lower()]\n                        for w in words\n                        if w.text.lower() in self.model\n                    ]\n                    or [np.zeros(self.dim)],\n                    axis=0,\n                )\n                for words in tokenizer.pipe(data)\n            ]\n        )\n", "path": "bugbug/nlp.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport os\n\nfrom setuptools import find_packages, setup\n\nhere = os.path.dirname(__file__)\n\n\ndef read_requirements(file_):\n    requires = []\n    links = []\n    with open(os.path.join(here, file_)) as f:\n        for line in f.readlines():\n            line = line.strip()\n\n            if line.startswith(\"https://\"):\n                links.append(line + \"-1.0.0\")\n                extras = \"\"\n                if \"[\" in line:\n                    extras = \"[\" + line.split(\"[\")[1].split(\"]\")[0] + \"]\"\n                line = line.split(\"#\")[1].split(\"egg=\")[1] + extras\n            elif line == \"\" or line.startswith(\"#\") or line.startswith(\"-\"):\n                continue\n            line = line.split(\"#\")[0].strip()\n            requires.append(line)\n\n    return sorted(list(set(requires))), links\n\n\ninstall_requires, dependency_links = read_requirements(\"requirements.txt\")\n\n\nwith open(os.path.join(here, \"VERSION\")) as f:\n    version = f.read().strip()\n\n# Read the extra requirements\nextras = [\"nlp\"]\n\nextras_require = {}\n\nfor extra in extras:\n    extras_install, extra_links = read_requirements(\"extra-%s-requirements.txt\" % extra)\n\n    # Merge the dependency links\n    dependency_links.extend(extra_links)\n\n    extras_require[extra] = extras_install\n\n\nsetup(\n    name=\"bugbug\",\n    version=version,\n    description=\"ML tools for Mozilla projects\",\n    author=\"Marco Castelluccio\",\n    author_email=\"mcastelluccio@mozilla.com\",\n    install_requires=install_requires,\n    extras_require=extras_require,\n    dependency_links=dependency_links,\n    packages=find_packages(exclude=[\"contrib\", \"docs\", \"tests\"]),\n    include_package_data=True,\n    license=\"MPL2\",\n    entry_points={\n        \"console_scripts\": [\n            \"bugbug-data-commits = scripts.commit_retriever:main\",\n            \"bugbug-data-bugzilla = scripts.bug_retriever:main\",\n            \"bugbug-train = scripts.trainer:main\",\n        ]\n    },\n    classifiers=[\n        \"Programming Language :: Python :: 3.6\",\n        \"Programming Language :: Python :: 3 :: Only\",\n        \"License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)\",\n    ],\n)\n", "path": "setup.py"}, {"content": "# -*- coding: utf-8 -*-\n# This Source Code Form is subject to the terms of the Mozilla Public\n# License, v. 2.0. If a copy of the MPL was not distributed with this file,\n# You can obtain one at http://mozilla.org/MPL/2.0/.\n\nimport sys\nfrom collections import defaultdict\nfrom functools import lru_cache\n\nimport numpy as np\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nHAS_OPTIONAL_DEPENDENCIES = False\n\ntry:\n    import spacy\n\n    from spacy.tokenizer import Tokenizer\n    from gensim.models import KeyedVectors\n\n    HAS_OPTIONAL_DEPENDENCIES = True\nexcept ImportError:\n    pass\n\ntry:\n    if HAS_OPTIONAL_DEPENDENCIES:\n        nlp = spacy.load(\"en_core_web_sm\")\nexcept OSError:\n    msg = (\n        \"Spacy model is missing, install it with: \"\n        f\"{sys.executable} -m spacy download en_core_web_sm\"\n    )\n    print(msg, file=sys.stderr)\n\nOPT_MSG_MISSING = (\n    \"Optional dependencies are missing, install them with: pip install bugbug[nlp]\\n\"\n    \"You might need also to download the models with: \"\n    f\"{sys.executable} -m spacy download en_core_web_sm\"\n)\n\n\ndef spacy_token_lemmatizer(text):\n    if len(text) > nlp.max_length:\n        text = text[: nlp.max_length - 1]\n    doc = nlp(text)\n    return [token.lemma_ for token in doc]\n\n\nclass SpacyVectorizer(TfidfVectorizer):\n    def __init__(self, *args, **kwargs):\n\n        # Detect when the Spacy optional dependency is missing\n        if not HAS_OPTIONAL_DEPENDENCIES:\n            raise NotImplementedError(OPT_MSG_MISSING)\n\n        super().__init__(tokenizer=spacy_token_lemmatizer, *args, **kwargs)\n\n\n@lru_cache()\ndef get_word_embeddings():\n    word_embeddings = KeyedVectors.load_word2vec_format(\"wiki-news-300d-1M-subword.vec\")\n    word_embeddings.init_sims(replace=True)\n    return word_embeddings\n\n\nclass MeanEmbeddingTransformer(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        # Detect when the Gensim optional dependency are missing\n        if not HAS_OPTIONAL_DEPENDENCIES:\n            raise NotImplementedError(OPT_MSG_MISSING)\n\n        self.model = get_word_embeddings()\n        self.dim = len(self.model[\"if\"])\n\n    def fit(self, x, y=None):\n        return self\n\n    def transform(self, data):\n        tokenizer = Tokenizer(nlp.vocab)\n        return np.array(\n            [\n                np.mean(\n                    [\n                        self.model[w.text.lower()]\n                        for w in words\n                        if w.text.lower() in self.model\n                    ]\n                    or [np.zeros(self.dim)],\n                    axis=0,\n                )\n                for words in tokenizer.pipe(data)\n            ]\n        )\n\n    def get_feature_names(self):\n        return np.array([f\"_{i}\" for i in range(self.dim)], dtype=object)\n\n\nclass TfidfMeanEmbeddingTransformer(MeanEmbeddingTransformer):\n    def __init__(self):\n        super().__init__()\n        self.word2weight = None\n\n    def fit(self, X, y=None):\n        tfidf = TfidfVectorizer(analyzer=lambda x: x)\n        tfidf.fit(X)\n\n        # If a word was never seen, it must be at least as infrequent as any of the known words.\n        # So, the default idf is the max of known idfs.\n        max_idf = max(tfidf.idf_)\n        self.word2weight = defaultdict(\n            lambda: max_idf, [(w, tfidf.idf_[i]) for w, i in tfidf.vocabulary_.items()]\n        )\n\n        return self\n\n    def transform(self, data):\n        tokenizer = Tokenizer(nlp.vocab)\n        return np.array(\n            [\n                np.mean(\n                    [\n                        self.model[w.text.lower()] * self.word2weight[w.text.lower()]\n                        for w in words\n                        if w.text.lower() in self.model\n                    ]\n                    or [np.zeros(self.dim)],\n                    axis=0,\n                )\n                for words in tokenizer.pipe(data)\n            ]\n        )\n", "path": "bugbug/nlp.py"}]}