text: /*M///////////////////////////////////////////////////////////////////////////////////////
  //
  //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
  //
  //  By downloading, copying, installing or using the software you agree to this license.
  //  If you do not agree to this license, do not download, install,
  //  copy or use the software.
  //
  //
  //                        Intel License Agreement
  //
  // Copyright (C) 2000, Intel Corporation, all rights reserved.
  // Third party copyrights are property of their respective owners.
  //
  // Redistribution and use in source and binary forms, with or without modification,
  // are permitted provided that the following conditions are met:
  //
  //   * Redistribution's of source code must retain the above copyright notice,
  //     this list of conditions and the following disclaimer.
  //
  //   * Redistribution's in binary form must reproduce the above copyright notice,
  //     this list of conditions and the following disclaimer in the documentation
  //     and/or other materials provided with the distribution.
  //
  //   * The name of Intel Corporation may not be used to endorse or promote products
  //     derived from this software without specific prior written permission.
  //
  // This software is provided by the copyright holders and contributors "as is" and
  // any express or implied warranties, including, but not limited to, the implied
  // warranties of merchantability and fitness for a particular purpose are disclaimed.
  // In no event shall the Intel Corporation or contributors be liable for any direct,
  // indirect, incidental, special, exemplary, or consequential damages
  // (including, but not limited to, procurement of substitute goods or services;
  // loss of use, data, or profits; or business interruption) however caused
  // and on any theory of liability, whether in contract, strict liability,
  // or tort (including negligence or otherwise) arising in any way out of
  // the use of this software, even if advised of the possibility of such damage.
  //
  //M*/
  
  #include "precomp.hpp"
  #include <ctype.h>
  #include <algorithm>
  #include <iterator>
  
  namespace cv { namespace ml {
  
  static const float MISSED_VAL = TrainData::missingValue();
  static const int VAR_MISSED = VAR_ORDERED;
  
  TrainData::~TrainData() {}
  
  Mat TrainData::getSubVector(const Mat& vec, const Mat& idx)
  {
      if( idx.empty() )
          return vec;
      int i, j, n = idx.checkVector(1, CV_32S);
      int type = vec.type();
      CV_Assert( type == CV_32S || type == CV_32F || type == CV_64F );
      int dims = 1, m;
  
      if( vec.cols == 1 || vec.rows == 1 )
      {
          dims = 1;
          m = vec.cols + vec.rows - 1;
      }
      else
      {
          dims = vec.cols;
          m = vec.rows;
      }
  
      Mat subvec;
  
      if( vec.cols == m )
          subvec.create(dims, n, type);
      else
          subvec.create(n, dims, type);
      if( type == CV_32S )
          for( i = 0; i < n; i++ )
          {
              int k = idx.at<int>(i);
              CV_Assert( 0 <= k && k < m );
              if( dims == 1 )
                  subvec.at<int>(i) = vec.at<int>(k);
              else
                  for( j = 0; j < dims; j++ )
                      subvec.at<int>(i, j) = vec.at<int>(k, j);
          }
      else if( type == CV_32F )
          for( i = 0; i < n; i++ )
          {
              int k = idx.at<int>(i);
              CV_Assert( 0 <= k && k < m );
              if( dims == 1 )
                  subvec.at<float>(i) = vec.at<float>(k);
              else
                  for( j = 0; j < dims; j++ )
                      subvec.at<float>(i, j) = vec.at<float>(k, j);
          }
      else
          for( i = 0; i < n; i++ )
          {
              int k = idx.at<int>(i);
              CV_Assert( 0 <= k && k < m );
              if( dims == 1 )
                  subvec.at<double>(i) = vec.at<double>(k);
              else
                  for( j = 0; j < dims; j++ )
                      subvec.at<double>(i, j) = vec.at<double>(k, j);
          }
      return subvec;
  }
  
  class TrainDataImpl : public TrainData
  {
  public:
      typedef std::map<String, int> MapType;
  
      TrainDataImpl()
      {
          file = 0;
          clear();
      }
  
      virtual ~TrainDataImpl() { closeFile(); }
  
      int getLayout() const { return layout; }
      int getNSamples() const
      {
          return !sampleIdx.empty() ? (int)sampleIdx.total() :
                 layout == ROW_SAMPLE ? samples.rows : samples.cols;
      }
      int getNTrainSamples() const
      {
          return !trainSampleIdx.empty() ? (int)trainSampleIdx.total() : getNSamples();
      }
      int getNTestSamples() const
      {
          return !testSampleIdx.empty() ? (int)testSampleIdx.total() : 0;
      }
      int getNVars() const
      {
          return !varIdx.empty() ? (int)varIdx.total() : getNAllVars();
      }
      int getNAllVars() const
      {
          return layout == ROW_SAMPLE ? samples.cols : samples.rows;
      }
  
      Mat getSamples() const { return samples; }
      Mat getResponses() const { return responses; }
      Mat getMissing() const { return missing; }
      Mat getVarIdx() const { return varIdx; }
      Mat getVarType() const { return varType; }
      int getResponseType() const
      {
          return classLabels.empty() ? VAR_ORDERED : VAR_CATEGORICAL;
      }
      Mat getTrainSampleIdx() const { return !trainSampleIdx.empty() ? trainSampleIdx : sampleIdx; }
      Mat getTestSampleIdx() const { return testSampleIdx; }
      Mat getSampleWeights() const
      {
          return sampleWeights;
      }
      Mat getTrainSampleWeights() const
      {
          return getSubVector(sampleWeights, getTrainSampleIdx());
      }
      Mat getTestSampleWeights() const
      {
          Mat idx = getTestSampleIdx();
          return idx.empty() ? Mat() : getSubVector(sampleWeights, idx);
      }
      Mat getTrainResponses() const
      {
          return getSubVector(responses, getTrainSampleIdx());
      }
      Mat getTrainNormCatResponses() const
      {
          return getSubVector(normCatResponses, getTrainSampleIdx());
      }
      Mat getTestResponses() const
      {
          Mat idx = getTestSampleIdx();
          return idx.empty() ? Mat() : getSubVector(responses, idx);
      }
      Mat getTestNormCatResponses() const
      {
          Mat idx = getTestSampleIdx();
          return idx.empty() ? Mat() : getSubVector(normCatResponses, idx);
      }
      Mat getNormCatResponses() const { return normCatResponses; }
      Mat getClassLabels() const { return classLabels; }
      Mat getClassCounters() const { return classCounters; }
      int getCatCount(int vi) const
      {
          int n = (int)catOfs.total();
          CV_Assert( 0 <= vi && vi < n );
          Vec2i ofs = catOfs.at<Vec2i>(vi);
          return ofs[1] - ofs[0];
      }
  
      Mat getCatOfs() const { return catOfs; }
      Mat getCatMap() const { return catMap; }
  
      Mat getDefaultSubstValues() const { return missingSubst; }
  
      void closeFile() { if(file) fclose(file); file=0; }
      void clear()
      {
          closeFile();
          samples.release();
          missing.release();
          varType.release();
          responses.release();
          sampleIdx.release();
          trainSampleIdx.release();
          testSampleIdx.release();
          normCatResponses.release();
          classLabels.release();
          classCounters.release();
          catMap.release();
          catOfs.release();
          nameMap = MapType();
          layout = ROW_SAMPLE;
      }
  
      typedef std::map<int, int> CatMapHash;
  
      void setData(InputArray _samples, int _layout, InputArray _responses,
                   InputArray _varIdx, InputArray _sampleIdx, InputArray _sampleWeights,
                   InputArray _varType, InputArray _missing)
      {
          clear();
  
          CV_Assert(_layout == ROW_SAMPLE || _layout == COL_SAMPLE );
          samples = _samples.getMat();
          layout = _layout;
          responses = _responses.getMat();
          varIdx = _varIdx.getMat();
          sampleIdx = _sampleIdx.getMat();
          sampleWeights = _sampleWeights.getMat();
          varType = _varType.getMat();
          missing = _missing.getMat();
  
          int nsamples = layout == ROW_SAMPLE ? samples.rows : samples.cols;
          int ninputvars = layout == ROW_SAMPLE ? samples.cols : samples.rows;
          int i, noutputvars = 0;
  
          CV_Assert( samples.type() == CV_32F || samples.type() == CV_32S );
  
          if( !sampleIdx.empty() )
          {
              CV_Assert( (sampleIdx.checkVector(1, CV_32S, true) > 0 &&
                         checkRange(sampleIdx, true, 0, 0, nsamples-1)) ||
                         sampleIdx.checkVector(1, CV_8U, true) == nsamples );
              if( sampleIdx.type() == CV_8U )
                  sampleIdx = convertMaskToIdx(sampleIdx);
          }
  
          if( !sampleWeights.empty() )
          {
              CV_Assert( sampleWeights.checkVector(1, CV_32F, true) == nsamples );
          }
          else
          {
              sampleWeights = Mat::ones(nsamples, 1, CV_32F);
          }
  
          if( !varIdx.empty() )
          {
              CV_Assert( (varIdx.checkVector(1, CV_32S, true) > 0 &&
                         checkRange(varIdx, true, 0, 0, ninputvars)) ||
                         varIdx.checkVector(1, CV_8U, true) == ninputvars );
              if( varIdx.type() == CV_8U )
                  varIdx = convertMaskToIdx(varIdx);
              varIdx = varIdx.clone();
              std::sort(varIdx.ptr<int>(), varIdx.ptr<int>() + varIdx.total());
          }
  
          if( !responses.empty() )
          {
              CV_Assert( responses.type() == CV_32F || responses.type() == CV_32S );
              if( (responses.cols == 1 || responses.rows == 1) && (int)responses.total() == nsamples )
                  noutputvars = 1;
              else
              {
                  CV_Assert( (layout == ROW_SAMPLE && responses.rows == nsamples) ||
                             (layout == COL_SAMPLE && responses.cols == nsamples) );
                  noutputvars = layout == ROW_SAMPLE ? responses.cols : responses.rows;
              }
              if( !responses.isContinuous() || (layout == COL_SAMPLE && noutputvars > 1) )
              {
                  Mat temp;
                  transpose(responses, temp);
                  responses = temp;
              }
          }
  
          int nvars = ninputvars + noutputvars;
  
          if( !varType.empty() )
          {
              CV_Assert( varType.checkVector(1, CV_8U, true) == nvars &&
                         checkRange(varType, true, 0, VAR_ORDERED, VAR_CATEGORICAL+1) );
          }
          else
          {
              varType.create(1, nvars, CV_8U);
              varType = Scalar::all(VAR_ORDERED);
              if( noutputvars == 1 )
                  varType.at<uchar>(ninputvars) = (uchar)(responses.type() < CV_32F ? VAR_CATEGORICAL : VAR_ORDERED);
          }
  
          if( noutputvars > 1 )
          {
              for( i = 0; i < noutputvars; i++ )
                  CV_Assert( varType.at<uchar>(ninputvars + i) == VAR_ORDERED );
          }
  
          catOfs = Mat::zeros(1, nvars, CV_32SC2);
          missingSubst = Mat::zeros(1, nvars, CV_32F);
  
          vector<int> labels, counters, sortbuf, tempCatMap;
          vector<Vec2i> tempCatOfs;
          CatMapHash ofshash;
  
          AutoBuffer<uchar> buf(nsamples);
          Mat non_missing(layout == ROW_SAMPLE ? Size(1, nsamples) : Size(nsamples, 1), CV_8U, (uchar*)buf);
          bool haveMissing = !missing.empty();
          if( haveMissing )
          {
              CV_Assert( missing.size() == samples.size() && missing.type() == CV_8U );
          }
  
          // we iterate through all the variables. For each categorical variable we build a map
          // in order to convert input values of the variable into normalized values (0..catcount_vi-1)
          // often many categorical variables are similar, so we compress the map - try to re-use
          // maps for different variables if they are identical
          for( i = 0; i < ninputvars; i++ )
          {
              Mat values_i = layout == ROW_SAMPLE ? samples.col(i) : samples.row(i);
  
              if( varType.at<uchar>(i) == VAR_CATEGORICAL )
              {
                  preprocessCategorical(values_i, 0, labels, 0, sortbuf);
                  missingSubst.at<float>(i) = -1.f;
                  int j, m = (int)labels.size();
                  CV_Assert( m > 0 );
                  int a = labels.front(), b = labels.back();
                  const int* currmap = &labels[0];
                  int hashval = ((unsigned)a*127 + (unsigned)b)*127 + m;
                  CatMapHash::iterator it = ofshash.find(hashval);
                  if( it != ofshash.end() )
                  {
                      int vi = it->second;
                      Vec2i ofs0 = tempCatOfs[vi];
                      int m0 = ofs0[1] - ofs0[0];
                      const int* map0 = &tempCatMap[ofs0[0]];
                      if( m0 == m && map0[0] == a && map0[m0-1] == b )
                      {
                          for( j = 0; j < m; j++ )
                              if( map0[j] != currmap[j] )
                                  break;
                          if( j == m )
                          {
                              // re-use the map
                              tempCatOfs.push_back(ofs0);
                              continue;
                          }
                      }
                  }
                  else
                      ofshash[hashval] = i;
                  Vec2i ofs;
                  ofs[0] = (int)tempCatMap.size();
                  ofs[1] = ofs[0] + m;
                  tempCatOfs.push_back(ofs);
                  std::copy(labels.begin(), labels.end(), std::back_inserter(tempCatMap));
              }
              else
              {
                  tempCatOfs.push_back(Vec2i(0, 0));
                  /*Mat missing_i = layout == ROW_SAMPLE ? missing.col(i) : missing.row(i);
                  compare(missing_i, Scalar::all(0), non_missing, CMP_EQ);
                  missingSubst.at<float>(i) = (float)(mean(values_i, non_missing)[0]);*/
                  missingSubst.at<float>(i) = 0.f;
              }
          }
  
          if( !tempCatOfs.empty() )
          {
              Mat(tempCatOfs).copyTo(catOfs);
              Mat(tempCatMap).copyTo(catMap);
          }
  
          if( varType.at<uchar>(ninputvars) == VAR_CATEGORICAL )
          {
              preprocessCategorical(responses, &normCatResponses, labels, &counters, sortbuf);
              Mat(labels).copyTo(classLabels);
              Mat(counters).copyTo(classCounters);
          }
      }
  
      Mat convertMaskToIdx(const Mat& mask)
      {
          int i, j, nz = countNonZero(mask), n = mask.cols + mask.rows - 1;
          Mat idx(1, nz, CV_32S);
          for( i = j = 0; i < n; i++ )
              if( mask.at<uchar>(i) )
                  idx.at<int>(j++) = i;
          return idx;
      }
  
      struct CmpByIdx
      {
          CmpByIdx(const int* _data, int _step) : data(_data), step(_step) {}
          bool operator ()(int i, int j) const { return data[i*step] < data[j*step]; }
          const int* data;
          int step;
      };
  
      void preprocessCategorical(const Mat& data, Mat* normdata, vector<int>& labels,
                                 vector<int>* counters, vector<int>& sortbuf)
      {
          CV_Assert((data.cols == 1 || data.rows == 1) && (data.type() == CV_32S || data.type() == CV_32F));
          int* odata = 0;
          int ostep = 0;
  
          if(normdata)
          {
              normdata->create(data.size(), CV_32S);
              odata = normdata->ptr<int>();
              ostep = normdata->isContinuous() ? 1 : (int)normdata->step1();
          }
  
          int i, n = data.cols + data.rows - 1;
          sortbuf.resize(n*2);
          int* idx = &sortbuf[0];
          int* idata = (int*)data.ptr<int>();
          int istep = data.isContinuous() ? 1 : (int)data.step1();
  
          if( data.type() == CV_32F )
          {
              idata = idx + n;
              const float* fdata = data.ptr<float>();
              for( i = 0; i < n; i++ )
              {
                  if( fdata[i*istep] == MISSED_VAL )
                      idata[i] = -1;
                  else
                  {
                      idata[i] = cvRound(fdata[i*istep]);
                      CV_Assert( (float)idata[i] == fdata[i*istep] );
                  }
              }
              istep = 1;
          }
  
          for( i = 0; i < n; i++ )
              idx[i] = i;
  
          std::sort(idx, idx + n, CmpByIdx(idata, istep));
  
          int clscount = 1;
          for( i = 1; i < n; i++ )
              clscount += idata[idx[i]*istep] != idata[idx[i-1]*istep];
  
          int clslabel = -1;
          int prev = ~idata[idx[0]*istep];
          int previdx = 0;
  
          labels.resize(clscount);
          if(counters)
              counters->resize(clscount);
  
          for( i = 0; i < n; i++ )
          {
              int l = idata[idx[i]*istep];
              if( l != prev )
              {
                  clslabel++;
                  labels[clslabel] = l;
                  int k = i - previdx;
                  if( clslabel > 0 && counters )
                      counters->at(clslabel-1) = k;
                  prev = l;
                  previdx = i;
              }
              if(odata)
                  odata[idx[i]*ostep] = clslabel;
          }
          if(counters)
              counters->at(clslabel) = i - previdx;
      }
  
      bool loadCSV(const String& filename, int headerLines,
                   int responseStartIdx, int responseEndIdx,
                   const String& varTypeSpec, char delimiter, char missch)
      {
          const int M = 1000000;
          const char delimiters[3] = { ' ', delimiter, '\0' };
          int nvars = 0;
          bool varTypesSet = false;
  
          clear();
  
          file = fopen( filename.c_str(), "rt" );
  
          if( !file )
              return false;
  
          std::vector<char> _buf(M);
          std::vector<float> allresponses;
          std::vector<float> rowvals;
          std::vector<uchar> vtypes, rowtypes;
          bool haveMissed = false;
          char* buf = &_buf[0];
  
          int i, ridx0 = responseStartIdx, ridx1 = responseEndIdx;
          int ninputvars = 0, noutputvars = 0;
  
          Mat tempSamples, tempMissing, tempResponses;
          MapType tempNameMap;
          int catCounter = 1;
  
          // skip header lines
          int lineno = 0;
          for(;;lineno++)
          {
              if( !fgets(buf, M, file) )
                  break;
              if(lineno < headerLines )
                  continue;
              // trim trailing spaces
              int idx = (int)strlen(buf)-1;
              while( idx >= 0 && isspace(buf[idx]) )
                  buf[idx--] = '\0';
              // skip spaces in the beginning
              char* ptr = buf;
              while( *ptr != '\0' && isspace(*ptr) )
                  ptr++;
              // skip commented off lines
              if(*ptr == '#')
                  continue;
              rowvals.clear();
              rowtypes.clear();
  
              char* token = strtok(buf, delimiters);
              if (!token)
                  break;
  
              for(;;)
              {
                  float val=0.f; int tp = 0;
                  decodeElem( token, val, tp, missch, tempNameMap, catCounter );
                  if( tp == VAR_MISSED )
                      haveMissed = true;
                  rowvals.push_back(val);
                  rowtypes.push_back((uchar)tp);
                  token = strtok(NULL, delimiters);
                  if (!token)
                      break;
              }
  
              if( nvars == 0 )
              {
                  if( rowvals.empty() )
                      CV_Error(CV_StsBadArg, "invalid CSV format; no data found");
                  nvars = (int)rowvals.size();
                  if( !varTypeSpec.empty() && varTypeSpec.size() > 0 )
                  {
                      setVarTypes(varTypeSpec, nvars, vtypes);
                      varTypesSet = true;
                  }
                  else
                      vtypes = rowtypes;
  
                  ridx0 = ridx0 >= 0 ? ridx0 : ridx0 == -1 ? nvars - 1 : -1;
                  ridx1 = ridx1 >= 0 ? ridx1 : ridx0 >= 0 ? ridx0+1 : -1;
                  CV_Assert(ridx1 > ridx0);
                  noutputvars = ridx0 >= 0 ? ridx1 - ridx0 : 0;
                  ninputvars = nvars - noutputvars;
              }
              else
                  CV_Assert( nvars == (int)rowvals.size() );
  
              // check var types
              for( i = 0; i < nvars; i++ )
              {
                  CV_Assert( (!varTypesSet && vtypes[i] == rowtypes[i]) ||
                             (varTypesSet && (vtypes[i] == rowtypes[i] || rowtypes[i] == VAR_ORDERED)) );
              }
  
              if( ridx0 >= 0 )
              {
                  for( i = ridx1; i < nvars; i++ )
                      std::swap(rowvals[i], rowvals[i-noutputvars]);
                  for( i = ninputvars; i < nvars; i++ )
                      allresponses.push_back(rowvals[i]);
                  rowvals.pop_back();
              }
              Mat rmat(1, ninputvars, CV_32F, &rowvals[0]);
              tempSamples.push_back(rmat);
          }
  
          closeFile();
  
          int nsamples = tempSamples.rows;
          if( nsamples == 0 )
              return false;
  
          if( haveMissed )
              compare(tempSamples, MISSED_VAL, tempMissing, CMP_EQ);
  
          if( ridx0 >= 0 )
          {
              for( i = ridx1; i < nvars; i++ )
                  std::swap(vtypes[i], vtypes[i-noutputvars]);
              if( noutputvars > 1 )
              {
                  for( i = ninputvars; i < nvars; i++ )
                      if( vtypes[i] == VAR_CATEGORICAL )
                          CV_Error(CV_StsBadArg,
                                   "If responses are vector values, not scalars, they must be marked as ordered responses");
              }
          }
  
          if( !varTypesSet && noutputvars == 1 && vtypes[ninputvars] == VAR_ORDERED )
          {
              for( i = 0; i < nsamples; i++ )
                  if( allresponses[i] != cvRound(allresponses[i]) )
                      break;
              if( i == nsamples )
                  vtypes[ninputvars] = VAR_CATEGORICAL;
          }
  
          Mat(nsamples, noutputvars, CV_32F, &allresponses[0]).copyTo(tempResponses);
          setData(tempSamples, ROW_SAMPLE, tempResponses, noArray(), noArray(),
                  noArray(), Mat(vtypes).clone(), tempMissing);
          bool ok = !samples.empty();
          if(ok)
              std::swap(tempNameMap, nameMap);
          return ok;
      }
  
      void decodeElem( const char* token, float& elem, int& type,
                       char missch, MapType& namemap, int& counter ) const
      {
          char* stopstring = NULL;
          elem = (float)strtod( token, &stopstring );
          if( *stopstring == missch && strlen(stopstring) == 1 ) // missed value
          {
              elem = MISSED_VAL;
              type = VAR_MISSED;
          }
          else if( *stopstring != '\0' )
          {
              MapType::iterator it = namemap.find(token);
              if( it == namemap.end() )
              {
                  elem = (float)counter;
                  namemap[token] = counter++;
              }
              else
                  elem = (float)it->second;
              type = VAR_CATEGORICAL;
          }
          else
              type = VAR_ORDERED;
      }
  
      void setVarTypes( const String& s, int nvars, std::vector<uchar>& vtypes ) const
      {
          const char* errmsg = "type spec is not correct; it should have format \"cat\", \"ord\" or "
            "\"ord[n1,n2-n3,n4-n5,...]cat[m1-m2,m3,m4-m5,...]\", where n's and m's are 0-based variable indices";
          const char* str = s.c_str();
          int specCounter = 0;
  
          vtypes.resize(nvars);
  
          for( int k = 0; k < 2; k++ )
          {
              const char* ptr = strstr(str, k == 0 ? "ord" : "cat");
              int tp = k == 0 ? VAR_ORDERED : VAR_CATEGORICAL;
              if( ptr ) // parse ord/cat str
              {
                  char* stopstring = NULL;
  
                  if( ptr[3] == '\0' )
                  {
                      for( int i = 0; i < nvars; i++ )
                          vtypes[i] = (uchar)tp;
                      specCounter = nvars;
                      break;
                  }
  
                  if ( ptr[3] != '[')
                      CV_Error( CV_StsBadArg, errmsg );
  
                  ptr += 4; // pass "ord["
                  do
                  {
                      int b1 = (int)strtod( ptr, &stopstring );
                      if( *stopstring == 0 || (*stopstring != ',' && *stopstring != ']' && *stopstring != '-') )
                          CV_Error( CV_StsBadArg, errmsg );
                      ptr = stopstring + 1;
                      if( (stopstring[0] == ',') || (stopstring[0] == ']'))
                      {
                          CV_Assert( 0 <= b1 && b1 < nvars );
                          vtypes[b1] = (uchar)tp;
                          specCounter++;
                      }
                      else
                      {
                          if( stopstring[0] == '-')
                          {
                              int b2 = (int)strtod( ptr, &stopstring);
                              if ( (*stopstring == 0) || (*stopstring != ',' && *stopstring != ']') )
                                  CV_Error( CV_StsBadArg, errmsg );
                              ptr = stopstring + 1;
                              CV_Assert( 0 <= b1 && b1 <= b2 && b2 < nvars );
                              for (int i = b1; i <= b2; i++)
                                  vtypes[i] = (uchar)tp;
                              specCounter += b2 - b1 + 1;
                          }
                          else
                              CV_Error( CV_StsBadArg, errmsg );
  
                      }
                  }
                  while(*stopstring != ']');
  
                  if( stopstring[1] != '\0' && stopstring[1] != ',')
                      CV_Error( CV_StsBadArg, errmsg );
              }
          }
  
          if( specCounter != nvars )
              CV_Error( CV_StsBadArg, "type of some variables is not specified" );
      }
  
      void setTrainTestSplitRatio(double ratio, bool shuffle)
      {
          CV_Assert( 0. <= ratio && ratio <= 1. );
          setTrainTestSplit(cvRound(getNSamples()*ratio), shuffle);
      }
  
      void setTrainTestSplit(int count, bool shuffle)
      {
          int i, nsamples = getNSamples();
          CV_Assert( 0 <= count && count < nsamples );
  
          trainSampleIdx.release();
          testSampleIdx.release();
  
          if( count == 0 )
              trainSampleIdx = sampleIdx;
          else if( count == nsamples )
              testSampleIdx = sampleIdx;
          else
          {
              Mat mask(1, nsamples, CV_8U);
              uchar* mptr = mask.ptr();
              for( i = 0; i < nsamples; i++ )
                  mptr[i] = (uchar)(i < count);
              trainSampleIdx.create(1, count, CV_32S);
              testSampleIdx.create(1, nsamples - count, CV_32S);
              int j0 = 0, j1 = 0;
              const int* sptr = !sampleIdx.empty() ? sampleIdx.ptr<int>() : 0;
              int* trainptr = trainSampleIdx.ptr<int>();
              int* testptr = testSampleIdx.ptr<int>();
              for( i = 0; i < nsamples; i++ )
              {
                  int idx = sptr ? sptr[i] : i;
                  if( mptr[i] )
                      trainptr[j0++] = idx;
                  else
                      testptr[j1++] = idx;
              }
              if( shuffle )
                  shuffleTrainTest();
          }
      }
  
      void shuffleTrainTest()
      {
          if( !trainSampleIdx.empty() && !testSampleIdx.empty() )
          {
              int i, nsamples = getNSamples(), ntrain = getNTrainSamples(), ntest = getNTestSamples();
              int* trainIdx = trainSampleIdx.ptr<int>();
              int* testIdx = testSampleIdx.ptr<int>();
              RNG& rng = theRNG();
  
              for( i = 0; i < nsamples; i++)
              {
                  int a = rng.uniform(0, nsamples);
                  int b = rng.uniform(0, nsamples);
                  int* ptra = trainIdx;
                  int* ptrb = trainIdx;
                  if( a >= ntrain )
                  {
                      ptra = testIdx;
                      a -= ntrain;
                      CV_Assert( a < ntest );
                  }
                  if( b >= ntrain )
                  {
                      ptrb = testIdx;
                      b -= ntrain;
                      CV_Assert( b < ntest );
                  }
                  std::swap(ptra[a], ptrb[b]);
              }
          }
      }
  
      Mat getTrainSamples(int _layout,
                          bool compressSamples,
                          bool compressVars) const
      {
          if( samples.empty() )
              return samples;
  
          if( (!compressSamples || (trainSampleIdx.empty() && sampleIdx.empty())) &&
              (!compressVars || varIdx.empty()) &&
              layout == _layout )
              return samples;
  
          int drows = getNTrainSamples(), dcols = getNVars();
          Mat sidx = getTrainSampleIdx(), vidx = getVarIdx();
          const float* src0 = samples.ptr<float>();
          const int* sptr = !sidx.empty() ? sidx.ptr<int>() : 0;
          const int* vptr = !vidx.empty() ? vidx.ptr<int>() : 0;
          size_t sstep0 = samples.step/samples.elemSize();
          size_t sstep = layout == ROW_SAMPLE ? sstep0 : 1;
          size_t vstep = layout == ROW_SAMPLE ? 1 : sstep0;
  
          if( _layout == COL_SAMPLE )
          {
              std::swap(drows, dcols);
              std::swap(sptr, vptr);
              std::swap(sstep, vstep);
          }
  
          Mat dsamples(drows, dcols, CV_32F);
  
          for( int i = 0; i < drows; i++ )
          {
              const float* src = src0 + (sptr ? sptr[i] : i)*sstep;
              float* dst = dsamples.ptr<float>(i);
  
              for( int j = 0; j < dcols; j++ )
                  dst[j] = src[(vptr ? vptr[j] : j)*vstep];
          }
  
          return dsamples;
      }
  
      void getValues( int vi, InputArray _sidx, float* values ) const
      {
          Mat sidx = _sidx.getMat();
          int i, n = sidx.checkVector(1, CV_32S), nsamples = getNSamples();
          CV_Assert( 0 <= vi && vi < getNAllVars() );
          CV_Assert( n >= 0 );
          const int* s = n > 0 ? sidx.ptr<int>() : 0;
          if( n == 0 )
              n = nsamples;
  
          size_t step = samples.step/samples.elemSize();
          size_t sstep = layout == ROW_SAMPLE ? step : 1;
          size_t vstep = layout == ROW_SAMPLE ? 1 : step;
  
          const float* src = samples.ptr<float>() + vi*vstep;
          float subst = missingSubst.at<float>(vi);
          for( i = 0; i < n; i++ )
          {
              int j = i;
              if( s )
              {
                  j = s[i];
                  CV_Assert( 0 <= j && j < nsamples );
              }
              values[i] = src[j*sstep];
              if( values[i] == MISSED_VAL )
                  values[i] = subst;
          }
      }
  
      void getNormCatValues( int vi, InputArray _sidx, int* values ) const
      {
          float* fvalues = (float*)values;
          getValues(vi, _sidx, fvalues);
          int i, n = (int)_sidx.total();
          Vec2i ofs = catOfs.at<Vec2i>(vi);
          int m = ofs[1] - ofs[0];
  
          CV_Assert( m > 0 ); // if m==0, vi is an ordered variable
          const int* cmap = &catMap.at<int>(ofs[0]);
          bool fastMap = (m == cmap[m - 1] - cmap[0] + 1);
  
          if( fastMap )
          {
              for( i = 0; i < n; i++ )
              {
                  int val = cvRound(fvalues[i]);
                  int idx = val - cmap[0];
                  CV_Assert(cmap[idx] == val);
                  values[i] = idx;
              }
          }
          else
          {
              for( i = 0; i < n; i++ )
              {
                  int val = cvRound(fvalues[i]);
                  int a = 0, b = m, c = -1;
  
                  while( a < b )
                  {
                      c = (a + b) >> 1;
                      if( val < cmap[c] )
                          b = c;
                      else if( val > cmap[c] )
                          a = c+1;
                      else
                          break;
                  }
  
                  CV_DbgAssert( c >= 0 && val == cmap[c] );
                  values[i] = c;
              }
          }
      }
  
      void getSample(InputArray _vidx, int sidx, float* buf) const
      {
          CV_Assert(buf != 0 && 0 <= sidx && sidx < getNSamples());
          Mat vidx = _vidx.getMat();
          int i, n = vidx.checkVector(1, CV_32S), nvars = getNAllVars();
          CV_Assert( n >= 0 );
          const int* vptr = n > 0 ? vidx.ptr<int>() : 0;
          if( n == 0 )
              n = nvars;
  
          size_t step = samples.step/samples.elemSize();
          size_t sstep = layout == ROW_SAMPLE ? step : 1;
          size_t vstep = layout == ROW_SAMPLE ? 1 : step;
  
          const float* src = samples.ptr<float>() + sidx*sstep;
          for( i = 0; i < n; i++ )
          {
              int j = i;
              if( vptr )
              {
                  j = vptr[i];
                  CV_Assert( 0 <= j && j < nvars );
              }
              buf[i] = src[j*vstep];
          }
      }
  
      FILE* file;
      int layout;
      Mat samples, missing, varType, varIdx, responses, missingSubst;
      Mat sampleIdx, trainSampleIdx, testSampleIdx;
      Mat sampleWeights, catMap, catOfs;
      Mat normCatResponses, classLabels, classCounters;
      MapType nameMap;
  };
  
  Ptr<TrainData> TrainData::loadFromCSV(const String& filename,
                                        int headerLines,
                                        int responseStartIdx,
                                        int responseEndIdx,
                                        const String& varTypeSpec,
                                        char delimiter, char missch)
  {
      Ptr<TrainDataImpl> td = makePtr<TrainDataImpl>();
      if(!td->loadCSV(filename, headerLines, responseStartIdx, responseEndIdx, varTypeSpec, delimiter, missch))
          td.release();
      return td;
  }
  
  Ptr<TrainData> TrainData::create(InputArray samples, int layout, InputArray responses,
                                   InputArray varIdx, InputArray sampleIdx, InputArray sampleWeights,
                                   InputArray varType)
  {
      Ptr<TrainDataImpl> td = makePtr<TrainDataImpl>();
      td->setData(samples, layout, responses, varIdx, sampleIdx, sampleWeights, varType, noArray());
      return td;
  }
  
  }}
  
  /* End of file. */
  