      # implements the projectivize/deprojectivize mechanism in Nivre & Nilsson 2005
      # for doing pseudo-projective parsing
      # implementation uses the HEAD decoration scheme
  
      def preprocess_training_data(self, labeled_trees, label_freq_cutoff=30):
          # expects a sequence of pairs of head arrays and labels
          preprocessed = []
          for heads,labels in labeled_trees:
              proj_heads,deco_labels = self.projectivize(heads,labels)
              # set the label to ROOT for each root dependent
              deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
 
</s> remove         for heads,labels in labeled_trees:
            proj_heads,deco_labels = self.projectivize(heads,labels)
            # set the label to ROOT for each root dependent
            deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
            preprocessed.append((proj_heads,deco_labels))
 </s> add         freqs = Counter()
        for raw_text, sents in gold_tuples:
            prepro_sents = []
            for (ids, words, tags, heads, labels, iob), ctnts in sents:
                proj_heads,deco_labels = cls.projectivize(heads,labels)
                # set the label to ROOT for each root dependent
                deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
                # count label frequencies
                if label_freq_cutoff > 0:
                    freqs.update( label for label in deco_labels if cls.is_decorated(label) )
                prepro_sents.append(((ids,words,tags,proj_heads,deco_labels,iob), ctnts))
            preprocessed.append((raw_text, prepro_sents)) </s> remove class PseudoProjective:
 </s> add cdef class PseudoProjectivity: </s> remove             return self._filter_labels(preprocessed,label_freq_cutoff)
 </s> add             return cls._filter_labels(preprocessed,label_freq_cutoff,freqs) </s> remove         freqs = Counter([ label for _,labels in labeled_trees for label in labels if label.find('||') != -1 ])
 </s> add  </s> remove         newheads, newlabels = copy(heads), copy(labels)
        spans = None
        for tokenid, head in enumerate(heads):
            if labels[tokenid].find('||') != -1:
                newlabel,_,headlabel = labels[tokenid].partition('||')
                newhead = self._find_new_head(head,tokenid,headlabel,heads,labels,spans=spans)
                newheads[tokenid] = newhead
                newlabels[tokenid] = newlabel
        return newheads, newlabels


    def _decorate(self, heads, proj_heads, labels):
 </s> add         parse = tokens.to_array([HEAD, DEP])
        labels = [ tokens.vocab.strings[int(p[1])] for p in parse ]
        for token in tokens:
            if cls.is_decorated(token.dep_):
                newlabel,headlabel = cls.decompose(token.dep_)
                newhead = cls._find_new_head(token,headlabel)
                parse[token.i,1] = tokens.vocab.strings[newlabel]
                parse[token.i,0] = newhead.i - token.i
        tokens.from_array([HEAD, DEP],parse)


    @classmethod
    def _decorate(cls, heads, proj_heads, labels):
    delimiter = '||'

    @classmethod
    def decompose(cls, label):
        return label.partition(cls.delimiter)[::2]

    @classmethod
    def is_decorated(cls, label):
        return label.find(cls.delimiter) != -1

    @classmethod
    def preprocess_training_data(cls, gold_tuples, label_freq_cutoff=30):
      # implements the projectivize/deprojectivize mechanism in Nivre & Nilsson 2005
      # for doing pseudo-projective parsing
      # implementation uses the HEAD decoration scheme
  
    delimiter = '||'

    @classmethod
    def decompose(cls, label):
        return label.partition(cls.delimiter)[::2]

    @classmethod
    def is_decorated(cls, label):
        return label.find(cls.delimiter) != -1

    @classmethod
    def preprocess_training_data(cls, gold_tuples, label_freq_cutoff=30):
    delimiter = '||'

    @classmethod
    def decompose(cls, label):
        return label.partition(cls.delimiter)[::2]

    @classmethod
    def is_decorated(cls, label):
        return label.find(cls.delimiter) != -1

    @classmethod
    def preprocess_training_data(cls, gold_tuples, label_freq_cutoff=30):
          preprocessed = []
          for heads,labels in labeled_trees:
              proj_heads,deco_labels = self.projectivize(heads,labels)
              # set the label to ROOT for each root dependent
              deco_labels = [ 'ROOT' if head == i else deco_labels[i] for i,head in enumerate(proj_heads) ]
keep keep keep keep replace replace keep keep keep keep keep
 integrated pseudo-projective parsing into parser

- nonproj.pyx holds a class PseudoProjectivity which currently holds
  all functionality to implement Nivre & Nilsson 2005's pseudo-projective
  parsing using the HEAD decoration scheme
- changed lefts/rights in Token to account for possible non-projective
  structures 
spacy/syntax/nonproj.pyx