'choose the best number of ngrams to include in bow.
Given an intent classification problem and a set of ordered ngrams (ordered in terms
of importance by pick_applicable_ngrams) we choose the best number of ngrams to include
in our bow vecs by cross validation.'

def _cross_validation(self, examples, labels, max_ngrams):

    from sklearn import preprocessing
    from sklearn.linear_model import LogisticRegression
    from sklearn.model_selection import cross_val_score
    import numpy as np
    if examples:
        collected_features = [e.get(u'text_features') for e in examples if (e.get(u'text_features') is not None)]
    else:
        collected_features = []
    existing_text_features = (np.stack(collected_features) if collected_features else None)
    def features_with_ngrams(max_ngrams):
        ngrams_to_use = self._ngrams_to_use(max_ngrams)
        extras = np.array(self._ngrams_in_sentences(examples, ngrams_to_use))
        if (existing_text_features is not None):
            return np.hstack((existing_text_features, extras))
        else:
            return extras
    clf2 = LogisticRegression(class_weight=u'balanced')
    intent_encoder = preprocessing.LabelEncoder()
    intent_encoder.fit(labels)
    y = intent_encoder.transform(labels)
    cv_splits = (min(10, np.min(np.bincount(y))) if (y.size > 0) else 0)
    if (cv_splits >= 3):
        logger.debug(u'Started    ngram    cross-validation    to    find    best    number    of    ngrams    to    use...')
        num_ngrams = np.unique(list(map(int, np.floor(np.linspace(1, max_ngrams, 8)))))
        if (existing_text_features is not None):
            no_ngrams_X = features_with_ngrams(max_ngrams=0)
            no_ngrams_score = np.mean(cross_val_score(clf2, no_ngrams_X, y, cv=cv_splits))
        else:
            no_ngrams_score = 0.0
        scores = []
        for n in num_ngrams:
            X = features_with_ngrams(max_ngrams=n)
            score = np.mean(cross_val_score(clf2, X, y, cv=cv_splits))
            scores.append(score)
            logger.debug(u'Evaluating    usage    of    {}    ngrams.    Score:    {}'.format(n, score))
        n_top = num_ngrams[np.argmax(scores)]
        logger.debug(u'Score    without    ngrams:    {}'.format(no_ngrams_score))
        logger.info(u'Best    score    with    {}    ngrams:    {}'.format(n_top, np.max(scores)))
        return n_top
    else:
        warnings.warn(u"Can't    cross-validate    ngram    featurizer.    There    aren't    enough    examples    per    intent    (at    least    3)")
        return max_ngrams
