  
      def preprocess_training_data(self, labeled_trees, label_freq_cutoff=30):
          # expects a sequence of pairs of head arrays and labels
          preprocessed = []
          for heads,labels in labeled_trees:
              proj_heads,deco_labels = self.projectivize(heads,labels)
              # set the label to ROOT for each root dependent
              deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
              preprocessed.append((proj_heads,deco_labels))
  
          if label_freq_cutoff > 0:
              return self._filter_labels(preprocessed,label_freq_cutoff)
          return preprocessed
  
 
</s> remove     def preprocess_training_data(self, labeled_trees, label_freq_cutoff=30):
        # expects a sequence of pairs of head arrays and labels
 </s> add     delimiter = '||'

    @classmethod
    def decompose(cls, label):
        return label.partition(cls.delimiter)[::2]

    @classmethod
    def is_decorated(cls, label):
        return label.find(cls.delimiter) != -1

    @classmethod
    def preprocess_training_data(cls, gold_tuples, label_freq_cutoff=30): </s> remove             return self._filter_labels(preprocessed,label_freq_cutoff)
 </s> add             return cls._filter_labels(preprocessed,label_freq_cutoff,freqs) </s> remove         freqs = Counter([ label for _,labels in labeled_trees for label in labels if label.find('||') != -1 ])
 </s> add  </s> remove         for proj_heads,deco_labels in labeled_trees:
            filtered_labels = [ label.partition('||')[0] if freqs.get(label,cutoff) < cutoff else label for label in deco_labels ]
            filtered.append((proj_heads,filtered_labels))
 </s> add         for raw_text, sents in gold_tuples:
            filtered_sents = []
            for (ids, words, tags, heads, labels, iob), ctnts in sents:
                filtered_labels = [ cls.decompose(label)[0] if freqs.get(label,cutoff) < cutoff else label for label in labels ]
                filtered_sents.append(((ids,words,tags,heads,filtered_labels,iob), ctnts))
            filtered.append((raw_text, filtered_sents)) </s> remove     def _make_span_index(self, heads):
        # stores the direct dependents for each token
        # for searching top-down through a tree
        spans = {}
        for tokenid, head in enumerate(heads):
            if tokenid == head: # root
                continue
            if head not in spans:
                spans[head] = []
            spans[head].append(tokenid)
        return spans


    def _filter_labels(self, labeled_trees, cutoff):
 </s> add     @classmethod
    def _filter_labels(cls, gold_tuples, cutoff, freqs):
        freqs = Counter()
        for raw_text, sents in gold_tuples:
            prepro_sents = []
            for (ids, words, tags, heads, labels, iob), ctnts in sents:
                proj_heads,deco_labels = cls.projectivize(heads,labels)
                # set the label to ROOT for each root dependent
                deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
                # count label frequencies
                if label_freq_cutoff > 0:
                    freqs.update( label for label in deco_labels if cls.is_decorated(label) )
                prepro_sents.append(((ids,words,tags,proj_heads,deco_labels,iob), ctnts))
            preprocessed.append((raw_text, prepro_sents))
  
      def preprocess_training_data(self, labeled_trees, label_freq_cutoff=30):
          # expects a sequence of pairs of head arrays and labels
          preprocessed = []
        freqs = Counter()
        for raw_text, sents in gold_tuples:
            prepro_sents = []
            for (ids, words, tags, heads, labels, iob), ctnts in sents:
                proj_heads,deco_labels = cls.projectivize(heads,labels)
                # set the label to ROOT for each root dependent
                deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
                # count label frequencies
                if label_freq_cutoff > 0:
                    freqs.update( label for label in deco_labels if cls.is_decorated(label) )
                prepro_sents.append(((ids,words,tags,proj_heads,deco_labels,iob), ctnts))
            preprocessed.append((raw_text, prepro_sents))
        freqs = Counter()
        for raw_text, sents in gold_tuples:
            prepro_sents = []
            for (ids, words, tags, heads, labels, iob), ctnts in sents:
                proj_heads,deco_labels = cls.projectivize(heads,labels)
                # set the label to ROOT for each root dependent
                deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
                # count label frequencies
                if label_freq_cutoff > 0:
                    freqs.update( label for label in deco_labels if cls.is_decorated(label) )
                prepro_sents.append(((ids,words,tags,proj_heads,deco_labels,iob), ctnts))
            preprocessed.append((raw_text, prepro_sents))
        freqs = Counter()
        for raw_text, sents in gold_tuples:
            prepro_sents = []
            for (ids, words, tags, heads, labels, iob), ctnts in sents:
                proj_heads,deco_labels = cls.projectivize(heads,labels)
                # set the label to ROOT for each root dependent
                deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
                # count label frequencies
                if label_freq_cutoff > 0:
                    freqs.update( label for label in deco_labels if cls.is_decorated(label) )
                prepro_sents.append(((ids,words,tags,proj_heads,deco_labels,iob), ctnts))
            preprocessed.append((raw_text, prepro_sents))
        freqs = Counter()
        for raw_text, sents in gold_tuples:
            prepro_sents = []
            for (ids, words, tags, heads, labels, iob), ctnts in sents:
                proj_heads,deco_labels = cls.projectivize(heads,labels)
                # set the label to ROOT for each root dependent
                deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
                # count label frequencies
                if label_freq_cutoff > 0:
                    freqs.update( label for label in deco_labels if cls.is_decorated(label) )
                prepro_sents.append(((ids,words,tags,proj_heads,deco_labels,iob), ctnts))
            preprocessed.append((raw_text, prepro_sents))
        freqs = Counter()
        for raw_text, sents in gold_tuples:
            prepro_sents = []
            for (ids, words, tags, heads, labels, iob), ctnts in sents:
                proj_heads,deco_labels = cls.projectivize(heads,labels)
                # set the label to ROOT for each root dependent
                deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
                # count label frequencies
                if label_freq_cutoff > 0:
                    freqs.update( label for label in deco_labels if cls.is_decorated(label) )
                prepro_sents.append(((ids,words,tags,proj_heads,deco_labels,iob), ctnts))
            preprocessed.append((raw_text, prepro_sents))
  
          if label_freq_cutoff > 0:
              return self._filter_labels(preprocessed,label_freq_cutoff)
          return preprocessed
  
keep keep keep keep replace replace replace replace replace keep keep keep keep keep
 integrated pseudo-projective parsing into parser

- nonproj.pyx holds a class PseudoProjectivity which currently holds
  all functionality to implement Nivre & Nilsson 2005's pseudo-projective
  parsing using the HEAD decoration scheme
- changed lefts/rights in Token to account for possible non-projective
  structures 
spacy/syntax/nonproj.pyx