Upload model

modeling_gzipembed.py

@@ -30,7 +30,7 @@ class GZIPEmbeddingModel(PreTrainedModel):
             ncd = [0] * len(self.config.corpus)
             with multiprocessing.Pool(num_procs) as pool:
                 data = enumerate(self.config.corpus)
-                results = pool.map(calculate_ncd_row,(data,p))
+                results = pool.map(calculate_ncd_row,data)
             for i,row in results:
                 ncd[i]=row
             x.append(ncd)
@@ -40,10 +40,6 @@ class GZIPEmbeddingModel(PreTrainedModel):
             return self.reduction_head(x)
         return x if not return_tensor else torch.tensor(x)
 
-    def ncd_r(self,r):
-        i=r[0][0]
-        return i,self.ncd(r[0][1],r[1])
-
     def normalize(self, x):
         x = ''.join([char for char in x.lower() if char in "abcdefghijklmnopqrstuvwxyz "])
         x = word_tokenize(x)