BERT_rationale_benchmark/models/pipeline/pipeline_utils.py
def decode_batch(data: List[Tuple[SentenceEvidence, SentenceEvidence]], name: str, score: bool = False,
                     annotations: List[Annotation] = None) -> dict:
        """Identifies evidence statements and then makes classifications based on it.

        Args:
            data: a paired list of SentenceEvidences, differing only in the kls field.
                  The first corresponds to whether or not something is evidence, and the second corresponds to an evidence class
            name: a name for a results dict
        """

        num_uniques = len(set((x.ann_id, x.docid) for x, _ in data))
        logging.info(f'Decoding dataset {name} with {len(data)} sentences, {num_uniques} annotations')
        identifier_data, classifier_data = zip(*data)
        results = dict()
        IdentificationClassificationResult = namedtuple('IdentificationClassificationResult',
                                                        'identification_data classification_data soft_identification hard_identification soft_classification hard_classification')
        with torch.no_grad():
            # make predictions for the evidence_identifier
            evidence_identifier.eval()
            evidence_classifier.eval()

            _, soft_identification_preds, hard_identification_preds, _ = make_preds_epoch(evidence_identifier,
                                                                                          identifier_data, batch_size,
                                                                                          device,
                                                                                          tensorize_model_inputs=tensorize_model_inputs)
            assert len(soft_identification_preds) == len(data)
            identification_results = defaultdict(list)
            for id_data, cls_data, soft_id_pred, hard_id_pred in zip(identifier_data, classifier_data,
                                                                     soft_identification_preds,
                                                                     hard_identification_preds):
                res = IdentificationClassificationResult(identification_data=id_data,
                                                         classification_data=cls_data,
                                                         # 1 is p(evidence|sent,query)
                                                         soft_identification=soft_id_pred[1].float().item(),
                                                         hard_identification=hard_id_pred,
                                                         soft_classification=None,
                                                         hard_classification=False)
                identification_results[(id_data.ann_id, id_data.docid)].append(res)

            best_identification_results = {key: max(value, key=lambda x: x.soft_identification) for key, value in
                                           identification_results.items()}
            logging.info(
                f'Selected the best sentence for {len(identification_results)} examples from a total of {len(soft_identification_preds)} sentences')
            ids, classification_data = zip(
                *[(k, v.classification_data) for k, v in best_identification_results.items()])
            _, soft_classification_preds, hard_classification_preds, classification_truth = make_preds_epoch(
                evidence_classifier, classification_data, batch_size, device,
                tensorize_model_inputs=tensorize_model_inputs)
            classification_results = dict()
            for eyeD, soft_class, hard_class in zip(ids, soft_classification_preds, hard_classification_preds):
                input_id_result = best_identification_results[eyeD]
                res = IdentificationClassificationResult(identification_data=input_id_result.identification_data,
                                                         classification_data=input_id_result.classification_data,
                                                         soft_identification=input_id_result.soft_identification,
                                                         hard_identification=input_id_result.hard_identification,
                                                         soft_classification=soft_class,
                                                         hard_classification=hard_class)
                classification_results[eyeD] = res

            if score:
                truth = []
                pred = []
                for res in classification_results.values():
                    truth.append(res.classification_data.kls)
                    pred.append(res.hard_classification)
                # results[f'{name}_f1'] = classification_report(classification_truth, pred, target_names=class_labels, output_dict=True)
                results[f'{name}_f1'] = classification_report(classification_truth, hard_classification_preds,
                                                              target_names=class_labels, output_dict=True)
                results[f'{name}_acc'] = accuracy_score(classification_truth, hard_classification_preds)
                results[f'{name}_rationale'] = score_rationales(annotations, decoding_docs, identifier_data,
                                                                soft_identification_preds)

            # turn the above results into a format suitable for scoring via the rationale scorer
            # n.b. the sentence-level evidence predictions (hard and soft) are
            # broadcast to the token level for scoring. The comprehensiveness class
            # score is also a lie since the pipeline model above is faithful by
            # design.
            decoded = dict()
            decoded_scores = defaultdict(list)
            for (ann_id, docid), pred in classification_results.items():
                sentence_prediction_scores = [x.soft_identification for x in identification_results[(ann_id, docid)]]
                sentence_start_token = sum(len(s) for s in decoding_docs[docid][:pred.identification_data.index])
                sentence_end_token = sentence_start_token + len(decoding_docs[docid][pred.classification_data.index])
                hard_rationale_predictions = [{'start_token': sentence_start_token, 'end_token': sentence_end_token}]
                soft_rationale_predictions = []
                for sent_result in sorted(identification_results[(ann_id, docid)],
                                          key=lambda x: x.identification_data.index):
                    soft_rationale_predictions.extend(sent_result.soft_identification for _ in range(len(
                        decoding_docs[sent_result.identification_data.docid][sent_result.identification_data.index])))
                if ann_id not in decoded:
                    decoded[ann_id] = {
                        "annotation_id": ann_id,
                        "rationales": [],
                        "classification": class_labels[pred.hard_classification],
                        "classification_scores": {class_labels[i]: s.item() for i, s in
                                                  enumerate(pred.soft_classification)},
                        # TODO this should turn into the data distribution for the predicted class
                        # "comprehensiveness_classification_scores": 0.0,
                        "truth": pred.classification_data.kls,
                    }
                decoded[ann_id]['rationales'].append({
                    "docid": docid,
                    "hard_rationale_predictions": hard_rationale_predictions,
                    "soft_rationale_predictions": soft_rationale_predictions,
                    "soft_sentence_predictions": sentence_prediction_scores,
                })
                decoded_scores[ann_id].append(pred.soft_classification)

            # in practice, this is always a single element operation:
            # in evidence inference (prompt is really a prompt + document), fever (we split documents into two classifications), movies (you only have one opinion about a movie), or boolQ (single document prompts)
            # this exists to support weird models we *might* implement for cose/esnli
            for ann_id, scores_list in decoded_scores.items():
                scores = torch.stack(scores_list)
                score_avg = torch.mean(scores, dim=0)
                # .float() because pytorch 1.3 introduces a bug where argmax is unsupported for float16
                hard_pred = torch.argmax(score_avg.float()).item()
                decoded[ann_id]['classification'] = class_labels[hard_pred]
                decoded[ann_id]['classification_scores'] = {class_labels[i]: s.item() for i, s in enumerate(score_avg)}
            return results, list(decoded.values())
Identifies evidence statements and then makes classifications based on it.

        Args:
            data: a paired list of SentenceEvidences, differing only in the kls field.
                  The first corresponds to whether or not something is evidence, and the second corresponds to an evidence class
            name: a name for a results dict
        
Identifies evidence statements and then makes classifications based on it.
Identifies evidence statements and then makes classifications based on it .
def decode_batch(data: List[Tuple[SentenceEvidence, SentenceEvidence]], name: str, score: bool = False,
                     annotations: List[Annotation] = None) -> dict:
        num_uniques = len(set((x.ann_id, x.docid) for x, _ in data))
        logging.info(f'Decoding dataset {name} with {len(data)} sentences, {num_uniques} annotations')
        identifier_data, classifier_data = zip(*data)
        results = dict()
        IdentificationClassificationResult = namedtuple('IdentificationClassificationResult',
                                                        'identification_data classification_data soft_identification hard_identification soft_classification hard_classification')
        with torch.no_grad():
            evidence_identifier.eval()
            evidence_classifier.eval()
            _, soft_identification_preds, hard_identification_preds, _ = make_preds_epoch(evidence_identifier,
                                                                                          identifier_data, batch_size,
                                                                                          device,
                                                                                          tensorize_model_inputs=tensorize_model_inputs)
            assert len(soft_identification_preds) == len(data)
            identification_results = defaultdict(list)
            for id_data, cls_data, soft_id_pred, hard_id_pred in zip(identifier_data, classifier_data,
                                                                     soft_identification_preds,
                                                                     hard_identification_preds):
                res = IdentificationClassificationResult(identification_data=id_data,
                                                         classification_data=cls_data,
                                                         soft_identification=soft_id_pred[1].float().item(),
                                                         hard_identification=hard_id_pred,
                                                         soft_classification=None,
                                                         hard_classification=False)
                identification_results[(id_data.ann_id, id_data.docid)].append(res)
            best_identification_results = {key: max(value, key=lambda x: x.soft_identification) for key, value in
                                           identification_results.items()}
            logging.info(
                f'Selected the best sentence for {len(identification_results)} examples from a total of {len(soft_identification_preds)} sentences')
            ids, classification_data = zip(
                *[(k, v.classification_data) for k, v in best_identification_results.items()])
            _, soft_classification_preds, hard_classification_preds, classification_truth = make_preds_epoch(
                evidence_classifier, classification_data, batch_size, device,
                tensorize_model_inputs=tensorize_model_inputs)
            classification_results = dict()
            for eyeD, soft_class, hard_class in zip(ids, soft_classification_preds, hard_classification_preds):
                input_id_result = best_identification_results[eyeD]
                res = IdentificationClassificationResult(identification_data=input_id_result.identification_data,
                                                         classification_data=input_id_result.classification_data,
                                                         soft_identification=input_id_result.soft_identification,
                                                         hard_identification=input_id_result.hard_identification,
                                                         soft_classification=soft_class,
                                                         hard_classification=hard_class)
                classification_results[eyeD] = res
            if score:
                truth = []
                pred = []
                for res in classification_results.values():
                    truth.append(res.classification_data.kls)
                    pred.append(res.hard_classification)
                results[f'{name}_f1'] = classification_report(classification_truth, hard_classification_preds,
                                                              target_names=class_labels, output_dict=True)
                results[f'{name}_acc'] = accuracy_score(classification_truth, hard_classification_preds)
                results[f'{name}_rationale'] = score_rationales(annotations, decoding_docs, identifier_data,
                                                                soft_identification_preds)
            decoded = dict()
            decoded_scores = defaultdict(list)
            for (ann_id, docid), pred in classification_results.items():
                sentence_prediction_scores = [x.soft_identification for x in identification_results[(ann_id, docid)]]
                sentence_start_token = sum(len(s) for s in decoding_docs[docid][:pred.identification_data.index])
                sentence_end_token = sentence_start_token + len(decoding_docs[docid][pred.classification_data.index])
                hard_rationale_predictions = [{'start_token': sentence_start_token, 'end_token': sentence_end_token}]
                soft_rationale_predictions = []
                for sent_result in sorted(identification_results[(ann_id, docid)],
                                          key=lambda x: x.identification_data.index):
                    soft_rationale_predictions.extend(sent_result.soft_identification for _ in range(len(
                        decoding_docs[sent_result.identification_data.docid][sent_result.identification_data.index])))
                if ann_id not in decoded:
                    decoded[ann_id] = {
                        "annotation_id": ann_id,
                        "rationales": [],
                        "classification": class_labels[pred.hard_classification],
                        "classification_scores": {class_labels[i]: s.item() for i, s in
                                                  enumerate(pred.soft_classification)},
                        "truth": pred.classification_data.kls,
                    }
                decoded[ann_id]['rationales'].append({
                    "docid": docid,
                    "hard_rationale_predictions": hard_rationale_predictions,
                    "soft_rationale_predictions": soft_rationale_predictions,
                    "soft_sentence_predictions": sentence_prediction_scores,
                })
                decoded_scores[ann_id].append(pred.soft_classification)
            for ann_id, scores_list in decoded_scores.items():
                scores = torch.stack(scores_list)
                score_avg = torch.mean(scores, dim=0)
                hard_pred = torch.argmax(score_avg.float()).item()
                decoded[ann_id]['classification'] = class_labels[hard_pred]
                decoded[ann_id]['classification_scores'] = {class_labels[i]: s.item() for i, s in enumerate(score_avg)}
            return results, list(decoded.values())
def decode_batch ( data : List [ Tuple [ SentenceEvidence , SentenceEvidence ] ] , name : str , score : bool = False , annotations : List [ Annotation ] = None ) -> dict : num_uniques = len ( set ( ( x . ann_id , x . docid ) for x , _ in data ) ) logging . info ( f'Decoding dataset {name} with {len(data)} sentences, {num_uniques} annotations' ) identifier_data , classifier_data = zip ( * data ) results = dict ( ) IdentificationClassificationResult = namedtuple ( 'IdentificationClassificationResult' , 'identification_data classification_data soft_identification hard_identification soft_classification hard_classification' ) with torch . no_grad ( ) : evidence_identifier . eval ( ) evidence_classifier . eval ( ) _ , soft_identification_preds , hard_identification_preds , _ = make_preds_epoch ( evidence_identifier , identifier_data , batch_size , device , tensorize_model_inputs = tensorize_model_inputs ) assert len ( soft_identification_preds ) == len ( data ) identification_results = defaultdict ( list ) for id_data , cls_data , soft_id_pred , hard_id_pred in zip ( identifier_data , classifier_data , soft_identification_preds , hard_identification_preds ) : res = IdentificationClassificationResult ( identification_data = id_data , classification_data = cls_data , soft_identification = soft_id_pred [ 1 ] . float ( ) . item ( ) , hard_identification = hard_id_pred , soft_classification = None , hard_classification = False ) identification_results [ ( id_data . ann_id , id_data . docid ) ] . append ( res ) best_identification_results = { key : max ( value , key = lambda x : x . soft_identification ) for key , value in identification_results . items ( ) } logging . info ( f'Selected the best sentence for {len(identification_results)} examples from a total of {len(soft_identification_preds)} sentences' ) ids , classification_data = zip ( * [ ( k , v . classification_data ) for k , v in best_identification_results . items ( ) ] ) _ , soft_classification_preds , hard_classification_preds , classification_truth = make_preds_epoch ( evidence_classifier , classification_data , batch_size , device , tensorize_model_inputs = tensorize_model_inputs ) classification_results = dict ( ) for eyeD , soft_class , hard_class in zip ( ids , soft_classification_preds , hard_classification_preds ) : input_id_result = best_identification_results [ eyeD ] res = IdentificationClassificationResult ( identification_data = input_id_result . identification_data , classification_data = input_id_result . classification_data , soft_identification = input_id_result . soft_identification , hard_identification = input_id_result . hard_identification , soft_classification = soft_class , hard_classification = hard_class ) classification_results [ eyeD ] = res if score : truth = [ ] pred = [ ] for res in classification_results . values ( ) : truth . append ( res . classification_data . kls ) pred . append ( res . hard_classification ) results [ f'{name}_f1' ] = classification_report ( classification_truth , hard_classification_preds , target_names = class_labels , output_dict = True ) results [ f'{name}_acc' ] = accuracy_score ( classification_truth , hard_classification_preds ) results [ f'{name}_rationale' ] = score_rationales ( annotations , decoding_docs , identifier_data , soft_identification_preds ) decoded = dict ( ) decoded_scores = defaultdict ( list ) for ( ann_id , docid ) , pred in classification_results . items ( ) : sentence_prediction_scores = [ x . soft_identification for x in identification_results [ ( ann_id , docid ) ] ] sentence_start_token = sum ( len ( s ) for s in decoding_docs [ docid ] [ : pred . identification_data . index ] ) sentence_end_token = sentence_start_token + len ( decoding_docs [ docid ] [ pred . classification_data . index ] ) hard_rationale_predictions = [ { 'start_token' : sentence_start_token , 'end_token' : sentence_end_token } ] soft_rationale_predictions = [ ] for sent_result in sorted ( identification_results [ ( ann_id , docid ) ] , key = lambda x : x . identification_data . index ) : soft_rationale_predictions . extend ( sent_result . soft_identification for _ in range ( len ( decoding_docs [ sent_result . identification_data . docid ] [ sent_result . identification_data . index ] ) ) ) if ann_id not in decoded : decoded [ ann_id ] = { "annotation_id" : ann_id , "rationales" : [ ] , "classification" : class_labels [ pred . hard_classification ] , "classification_scores" : { class_labels [ i ] : s . item ( ) for i , s in enumerate ( pred . soft_classification ) } , "truth" : pred . classification_data . kls , } decoded [ ann_id ] [ 'rationales' ] . append ( { "docid" : docid , "hard_rationale_predictions" : hard_rationale_predictions , "soft_rationale_predictions" : soft_rationale_predictions , "soft_sentence_predictions" : sentence_prediction_scores , } ) decoded_scores [ ann_id ] . append ( pred . soft_classification ) for ann_id , scores_list in decoded_scores . items ( ) : scores = torch . stack ( scores_list ) score_avg = torch . mean ( scores , dim = 0 ) hard_pred = torch . argmax ( score_avg . float ( ) ) . item ( ) decoded [ ann_id ] [ 'classification' ] = class_labels [ hard_pred ] decoded [ ann_id ] [ 'classification_scores' ] = { class_labels [ i ] : s . item ( ) for i , s in enumerate ( score_avg ) } return results , list ( decoded . values ( ) )
Identifies evidence statements and then makes classifications based on it.
Identifies evidence statements and then makes classifications based on it .
"""Identifies evidence statements and then makes classifications based on it.

        Args:
            data: a paired list of SentenceEvidences, differing only in the kls field.
                  The first corresponds to whether or not something is evidence, and the second corresponds to an evidence class
            name: a name for a results dict
        """ # make predictions for the evidence_identifier # 1 is p(evidence|sent,query) # results[f'{name}_f1'] = classification_report(classification_truth, pred, target_names=class_labels, output_dict=True) # turn the above results into a format suitable for scoring via the rationale scorer # n.b. the sentence-level evidence predictions (hard and soft) are # broadcast to the token level for scoring. The comprehensiveness class # score is also a lie since the pipeline model above is faithful by # design. # TODO this should turn into the data distribution for the predicted class # "comprehensiveness_classification_scores": 0.0, # in practice, this is always a single element operation: # in evidence inference (prompt is really a prompt + document), fever (we split documents into two classifications), movies (you only have one opinion about a movie), or boolQ (single document prompts) # this exists to support weird models we *might* implement for cose/esnli # .float() because pytorch 1.3 introduces a bug where argmax is unsupported for float16
data
List[Tuple[SentenceEvidence, SentenceEvidence]]
a paired list of SentenceEvidences, differing only in the kls field.
The first corresponds to whether or not something is evidence, and the second corresponds to an evidence class
a paired list of SentenceEvidences differing only in the kls field . The first corresponds to whether or not something is evidence and the second corresponds to an evidence class
name
str
a name for a results dict
a name for a results dict
score
bool
annotations
List[Annotation]