text: /************************************************************************
   *                                                                      *
   *  Program package 'lvq_pak':                                          *
   *                                                                      *
   *  mcnemar.c                                                           *
   *	Runs the McNemar's test to compare the statistical              *
   *	significance of the difference between the classification       *
   *	accuracies of two classifiers, that have been tested            *
   *	using the SAME data (the samples used to test the               *
   *	classifiers are thus, of course, *NOT* independent).            *
   *                                                                      *
   *	See, for example, the following reference:                      *
   *                                                                      *
   *	J. S. Milton, Jesse C. Arnold, "Probability and statistics in   *
   *	the engineering and computing sciences", McGraw-Hill, New York, *
   *	St. Louis, San Francisco,..., 2nd printing, 1987,               *
   *	pp. 543-544.                                                    *
   *                                                                      *
   *  Version 3.0                                                         *
   *  Date: 1 Mar 1995                                                    *
   *                                                                      *
   *  NOTE: This program package is copyrighted in the sense that it      *
   *  may be used for scientific purposes. The package as a whole, or     *
   *  parts thereof, cannot be included or used in any commercial         *
   *  application without written permission granted by its producents.   *
   *  No programs contained in this package may be copied for commercial  *
   *  distribution.                                                       *
   *                                                                      *
   *  All comments  concerning this program package may be sent to the    *
   *  e-mail address 'lvq@cochlea.hut.fi'.                                *
   *                                                                      *
   ************************************************************************/
  
  
  #include <stdio.h>
  #include <float.h>
  #include "lvq_pak.h"
  #include "datafile.h"
  
  int tbl[2][2] = { {0, 0}, {0, 0} };
  
  
  static double alpha[4]  = {0.05, 0.025, 0.01, 0.005},
                chi_sq[4] = {3.84, 5.02,  6.63, 7.88};
  
  char *usage = "\n\
  Usage: mcnemar classification_file1 classification_file2\n\
   You must first run \"accuracy\" with option \"-cfout classification_file\"\n\
   to create the files containing classification information.\n";
  
  
  int main(int argc, char **argv)
  {
    FILE *cf1, *cf2;
    struct file_info *cfi1 = NULL, *cfi2 = NULL;
    int i1,i2,c1,c2,cnt,i;
    double testv, tmp;
    int error = 1;
  
    if (argc != 3) {
      fprintf(stderr, usage);
      exit(1);
    }
    if ((cfi1 = open_file(argv[1],"r")) == NULL) {
      fprintf(stderr, "\nCannot open %s\n",argv[1]);
      goto cleanup;
    }
    cf1 = fi2fp(cfi1);
  
    if ( (cfi2 = open_file(argv[2],"r")) == NULL) {
      fprintf(stderr, "\nCannot open %s\n",argv[2]);
      goto cleanup;
    }
    cf2 = fi2fp(cfi2);
    
    for (;;) {
      i1 = fscanf(cf1,"%d", &c1);
      i2 = fscanf(cf2,"%d", &c2);
  
      if (i1 != i2) {
        fprintf(stderr, "\nERROR: Unequal numbers of classifications in files.\n");
        goto cleanup;
      }
      if (i1 != 1) break;
  
      if ( (c1!=0 && c1!=1) || (c2!=0 && c2!=1)) {
        fprintf(stderr, "\nFiles contain other than 0's and 1's.\n");
        goto cleanup;
      }
  
      c1 = 1 - c1;
      c2 = 1 - c2;
      tbl[c1][c2]++;
    }
    
    cnt = tbl[0][1] + tbl[1][0];
    if (cnt) {
      fprintf(stderr, "\nStatistics of the results of the two classifiers:");
      fprintf(stderr, "\n             1st correct,  1st errors");
      fprintf(stderr, "\n2nd correct:      %6d       %6d",   tbl[0][0],tbl[1][0]);
      fprintf(stderr, "\n2nd errors:       %6d       %6d\n", tbl[0][1],tbl[1][1]);
      tmp = tbl[0][1] - tbl[1][0];
      testv = tmp*tmp;
      testv /= cnt;
      fprintf(stderr, "\nTest statistics (%.3f)", testv);
      
      for (i=3; i>=0; i--) if (testv > chi_sq[i]) break;
      
      if (i>=0) {
        fprintf(stderr, " is significant at risk level %.3f\n", alpha[i]);
        fprintf(stderr, "The classifiers are significantly different!\n");
      } else {
        fprintf(stderr, " is not significant!\n");
        fprintf(stderr, "The classifiers are not significantly different!\n");
      }
      
    }
    else
      fprintf(stderr, "\nRecognition result files are equal!\n");
  
    error = 0;
   cleanup:
  
    if (cfi1)
      close_file(cfi1);
    if (cfi2)
      close_file(cfi2);
  
    return(error);
  }
  
  
  