              queue = next_queue
          return heads[tokenid]
  
  
      def _make_span_index(self, heads):
          # stores the direct dependents for each token
          # for searching top-down through a tree
          spans = {}
          for tokenid, head in enumerate(heads):
              if tokenid == head: # root
                  continue
              if head not in spans:
                  spans[head] = []
              spans[head].append(tokenid)
          return spans
  
  
      def _filter_labels(self, labeled_trees, cutoff):
          # throw away infrequent decorated labels
          # can't learn them reliably anyway and keeps label set smaller
          freqs = Counter([ label for _,labels in labeled_trees for label in labels if label.find('||') != -1 ])
          filtered = []
          for proj_heads,deco_labels in labeled_trees:
 
</s> remove         freqs = Counter([ label for _,labels in labeled_trees for label in labels if label.find('||') != -1 ])
 </s> add  </s> remove         for proj_heads,deco_labels in labeled_trees:
            filtered_labels = [ label.partition('||')[0] if freqs.get(label,cutoff) < cutoff else label for label in deco_labels ]
            filtered.append((proj_heads,filtered_labels))
 </s> add         for raw_text, sents in gold_tuples:
            filtered_sents = []
            for (ids, words, tags, heads, labels, iob), ctnts in sents:
                filtered_labels = [ cls.decompose(label)[0] if freqs.get(label,cutoff) < cutoff else label for label in labels ]
                filtered_sents.append(((ids,words,tags,heads,filtered_labels,iob), ctnts))
            filtered.append((raw_text, filtered_sents)) </s> remove         return heads[tokenid]
 </s> add         return token.head </s> remove         if not spans:
            spans = self._make_span_index(heads)
        queue = spans.get(rootid,[])
        queue.remove(tokenid) # don't search in the subtree of the nonproj arc
 </s> add         queue = [token.head] </s> remove         for heads,labels in labeled_trees:
            proj_heads,deco_labels = self.projectivize(heads,labels)
            # set the label to ROOT for each root dependent
            deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
            preprocessed.append((proj_heads,deco_labels))
 </s> add         freqs = Counter()
        for raw_text, sents in gold_tuples:
            prepro_sents = []
            for (ids, words, tags, heads, labels, iob), ctnts in sents:
                proj_heads,deco_labels = cls.projectivize(heads,labels)
                # set the label to ROOT for each root dependent
                deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
                # count label frequencies
                if label_freq_cutoff > 0:
                    freqs.update( label for label in deco_labels if cls.is_decorated(label) )
                prepro_sents.append(((ids,words,tags,proj_heads,deco_labels,iob), ctnts))
            preprocessed.append((raw_text, prepro_sents))
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
              queue = next_queue
          return heads[tokenid]
  
  
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
    @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
          # throw away infrequent decorated labels
          # can't learn them reliably anyway and keeps label set smaller
          freqs = Counter([ label for _,labels in labeled_trees for label in labels if label.find('||') != -1 ])
          filtered = []
          for proj_heads,deco_labels in labeled_trees:
keep keep keep keep replace replace replace replace replace replace replace replace replace replace replace replace replace replace keep keep keep keep keep
 integrated pseudo-projective parsing into parser

- nonproj.pyx holds a class PseudoProjectivity which currently holds
  all functionality to implement Nivre & Nilsson 2005's pseudo-projective
  parsing using the HEAD decoration scheme
- changed lefts/rights in Token to account for possible non-projective
  structures 
spacy/syntax/nonproj.pyx