text: /*
   * Copyright 2005-2014 ECMWF.
   *
   * This software is licensed under the terms of the Apache Licence Version 2.0
   * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
   *
   * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
   * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
   */
  
  /*
   *
   * Description: grib index
   *
   */
  
  #include "grib_api_internal.h"
  
  #define UNDEF_LONG -99999
  #define UNDEF_DOUBLE -99999
  
  #define NULL_MARKER 0
  #define NOT_NULL_MARKER 255
  
  
  /* #if GRIB_PTHREADS */
  #if 0
  static pthread_once_t once  = PTHREAD_ONCE_INIT;
  static pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER;
  static pthread_mutex_t mutex2 = PTHREAD_MUTEX_INITIALIZER;
  
  static void init() {
      pthread_mutexattr_t attr;
      pthread_mutexattr_init(&attr);
      pthread_mutexattr_settype(&attr,PTHREAD_MUTEX_RECURSIVE);
      pthread_mutex_init(&mutex1,&attr);
      pthread_mutex_init(&mutex2,&attr);
      pthread_mutexattr_destroy(&attr);
  
  }
  #endif
  
  
  static const char* mars_keys =
      "mars.date,mars.time,mars.expver,mars.stream,mars.class,mars.type,"
  	"mars.step,mars.param,mars.levtype,mars.levelist,mars.number,mars.iteration,"
  	"mars.domain,mars.fcmonth,mars.fcperiod,mars.hdate,mars.method,"
  	"mars.model,mars.origin,mars.quantile,mars.range,mars.refdate,mars.direction,mars.frequency";
  
  
  static int grib_filesid=0;
  static int index_count;
  
  static char* get_key(char** keys,int *type)
  {
      char* key=NULL;
      char* p=NULL;
  
      if (*keys == 0 || keys==NULL) return NULL;
      *type=GRIB_TYPE_UNDEFINED;
      p=*keys;
      while (*p == ' ') p++;
  
  
      while (*p != 0 && *p != ':' && *p != ',') p++;
      if ( *p == ':' ) {
          *type=grib_type_to_int(*(p+1));
          *p=0;
          p++;
          while (*p != 0 && *p != ',') {*(p++)=0;}
      } else *type=GRIB_TYPE_UNDEFINED;
      if (*p) {*p=0;p++;}
      key=*keys;
      *keys= *p==0 ? NULL : p;
      return key;
  }
  
  static int compare_long(const void* a,const void* b)
  {
      long* arg1 = (long*) a;
      long* arg2 = (long*) b;
      if( *arg1 == *arg2 ) return 0;
  
      return *arg1 < *arg2 ? -1 : 1;
  }
  
  static int compare_double(const void* a,const void* b)
  {
      double* arg1 = (double*) a;
      double* arg2 = (double*) b;
      if( *arg1 == *arg2 ) return 0;
  
      return *arg1 < *arg2 ? -1 : 1;
  }
  
  static int compare_string(const void* a,const void* b)
  {
      char* arg1 = *(char* const*) a;
      char* arg2 = *(char* const*) b;
  
      while (*arg1 != 0 && *arg2 != 0 && *arg1 == *arg2 ) {arg1++;arg2++;}
  
      if( *arg1 == *arg2 ) return 0;
  
      return *arg1 < *arg2 ? -1 : 1;
  }
  
  static int grib_index_keys_compress(grib_context* c,grib_index* index,int* compress)
  {
      grib_index_key *keys=index->keys->next;
      grib_index_key *prev=index->keys;
      int level=0;
  
      if (!keys) return 0;
  
      level=1;
      while (keys) {
          if (keys->values_count==1) {
              prev->next=keys->next;
              grib_context_free(c,keys->name);
              grib_context_free(c,keys);
              keys=prev->next;
              compress[level]=1;
              level++;
          } else {
              prev=keys;
              keys=keys->next;
              compress[level]=0;
              level++;
          }
      }
  
      if (index->keys->values_count==1) {
          keys=index->keys;
          index->keys=index->keys->next;
          grib_context_free(c,keys->name);
          grib_context_free(c,keys);
          compress[0]=1;
      } else compress[0]=0;
  
      return 0;
  }
  
  static int grib_index_fields_compress(grib_context* c,
          grib_field_tree* fields,grib_field_tree* prev,int level,int* compress)
  {
      if (!fields) return 0;
  
      if (!prev) {
          if (fields->next)
              grib_index_fields_compress(c,fields->next,0,level,compress);
          level++;
          return grib_index_fields_compress(c,fields->next_level,fields,level,compress);
      }
  
      if (compress[level]) {
          if (!fields->next_level) prev->field=fields->field;
  
          prev->next_level=fields->next_level;
          grib_context_free(c,fields->value);
          grib_context_free(c,fields);
          level++;
          grib_index_fields_compress(c,prev->next_level,prev,level,compress);
      } else {
          grib_field_tree* next;
          next=fields->next;
          level++;
          while (next) {
              grib_index_fields_compress(c,next->next_level,next,level,compress);
              next=next->next;
          }
          grib_index_fields_compress(c,fields->next_level,fields,level,compress);
      }
  
      return 0;
  }
  
  int grib_index_compress(grib_index* index)
  {
      int err=0;
      grib_context* c=index->context;
      int compress[200]={0,};
  
      if (!index->keys->next) return 0;
  
      err=grib_index_keys_compress(c,index,compress);
      if (err) return err;
  
      grib_index_fields_compress(c,index->fields,0,0,compress);
  
      if (!index->fields->next) {
          grib_field_tree* next_level=index->fields->next_level;
          grib_context_free(c,index->fields->value);
          grib_context_free(c,index->fields);
          index->fields=next_level;
      }
      return 0;
  }
  
  static grib_index_key* grib_index_new_key(grib_context* c,grib_index_key* keys,
          const char* key,int type,int *err)
  {
      grib_index_key *next=NULL,*current=NULL;
      grib_string_list* values=NULL;
  
      next=(grib_index_key*)grib_context_malloc_clear(c,sizeof(grib_index_key));
      if (!next) {
          grib_context_log(c,GRIB_LOG_ERROR,
                  "unable to allocate %d bytes",
                  sizeof(grib_index_key));
          *err=GRIB_OUT_OF_MEMORY;
          return NULL;
      }
      values=grib_context_malloc_clear(c,sizeof(grib_string_list));
      if (!values) {
          grib_context_log(c,GRIB_LOG_ERROR,
                  "unable to allocate %d bytes",
                  sizeof(grib_string_list));
          *err=GRIB_OUT_OF_MEMORY;
          return NULL;
      }
  
      next->values=values;
  
      if (!keys) {
          keys=next;
          current=keys;
      } else {
          current=keys;
          while (current->next) current=current->next;
          current->next=next;
          current=current->next;
      }
  
      current->type=type;
      current->name=grib_context_strdup(c,key);
      return keys;
  }
  
  int grib_read_uchar( FILE* fh,unsigned char *val)
  {
      if (fread(val,sizeof(unsigned char),1,fh) <1) {
          if (feof(fh)) return GRIB_END_OF_FILE;
          else return GRIB_IO_PROBLEM;
      }
      return GRIB_SUCCESS;
  }
  
  int grib_read_short( FILE* fh,short *val)
  {
      if (fread(val,sizeof(short),1,fh) <1) {
          if (feof(fh)) return GRIB_END_OF_FILE;
          else return GRIB_IO_PROBLEM;
      }
      return GRIB_SUCCESS;
  }
  
  int grib_read_long( FILE* fh,long *val)
  {
      if (fread(val,sizeof(long),1,fh) <1) {
          if (feof(fh)) return GRIB_END_OF_FILE;
          else return GRIB_IO_PROBLEM;
      }
      return GRIB_SUCCESS;
  }
  
  int grib_read_unsigned_long( FILE* fh,unsigned long *val)
  {
      if (fread(val,sizeof(long),1,fh) <1) {
          if (feof(fh)) return GRIB_END_OF_FILE;
          else return GRIB_IO_PROBLEM;
      }
      return GRIB_SUCCESS;
  }
  
  int grib_write_uchar(FILE* fh,unsigned char val)
  {
      if (fwrite(&val,sizeof(unsigned char),1,fh)<1)
          return GRIB_IO_PROBLEM;
      return GRIB_SUCCESS;
  }
  
  int grib_write_short(FILE* fh,short val)
  {
      if (fwrite(&val,sizeof(short),1,fh)<1)
          return GRIB_IO_PROBLEM;
      return GRIB_SUCCESS;
  }
  
  int grib_write_long(FILE* fh,long val)
  {
      if (fwrite(&val,sizeof(long),1,fh)<1)
          return GRIB_IO_PROBLEM;
      return GRIB_SUCCESS;
  }
  
  int grib_write_unsigned_long(FILE* fh,unsigned long val)
  {
      if (fwrite(&val,sizeof(long),1,fh)<1)
          return GRIB_IO_PROBLEM;
      return GRIB_SUCCESS;
  }
  
  int grib_write_string(FILE* fh,const char* s)
  {
      size_t len = strlen(s);
      grib_write_uchar(fh,(unsigned char)len);
      if (fwrite(s,1,len,fh)<len)
          return GRIB_IO_PROBLEM;
      return GRIB_SUCCESS;
  }
  
  int grib_write_identifier(FILE* fh)
  {
      return grib_write_string(fh,"GRBIDX1");
  }
  
  int grib_write_null_marker(FILE* fh)
  {
      return grib_write_uchar(fh,NULL_MARKER);
  }
  
  int grib_write_not_null_marker(FILE* fh)
  {
      return grib_write_uchar(fh,NOT_NULL_MARKER);
  }
  
  char *grib_read_string(grib_context* c,FILE* fh,int *err)
  {
      unsigned char len=0;
      char* s=NULL;
      *err = grib_read_uchar(fh,&len);
  
      if (*err) return NULL;
      s = grib_context_malloc_clear(c,len+1);
      if (fread(s,len,1,fh) < 1)
      {
          if (feof(fh)) *err=GRIB_END_OF_FILE;
          else *err=GRIB_IO_PROBLEM;
          return NULL;
      }
      s[len] = 0;
  
      return s;
  }
  
  static int grib_write_field(FILE* fh,grib_field* field)
  {
      int err;
      if (!field)
          return grib_write_null_marker(fh);
  
      err=grib_write_not_null_marker(fh);
      if (err) return err;
  
      err=grib_write_short(fh,field->file->id);
      if (err) return err;
  
      err=grib_write_unsigned_long(fh,field->offset);
      if (err) return err;
  
      err=grib_write_unsigned_long(fh,field->length);
      if (err) return err;
  
      err=grib_write_field(fh,field->next);
      if (err) return err;
  
      return GRIB_SUCCESS;
  }
  
  static grib_field* grib_read_field(grib_context* c, FILE* fh, grib_file** files, int *err)
  {
      grib_field* field=NULL;
      short file_id;
      unsigned char marker ;
      unsigned long offset;
      unsigned long length;
  
      *err = grib_read_uchar(fh,&marker);
      if(marker == NULL_MARKER) return NULL;
      if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}
  
      index_count++;
      field=grib_context_malloc(c,sizeof(grib_field));
      *err=grib_read_short(fh,&file_id);
      if (*err) return NULL;
  
      field->file=files[file_id];
  
      *err=grib_read_unsigned_long(fh,&offset);
      field->offset=offset;
      if (*err) return NULL;
  
      *err=grib_read_unsigned_long(fh,&length);
      field->length=length;
      if (*err) return NULL;
  
      field->next=grib_read_field(c,fh,files,err);
  
      return field;
  }
  
  static int grib_write_field_tree(FILE* fh,grib_field_tree* tree)
  {
      int err=0;
  
      if(!tree)
          return grib_write_null_marker(fh);
  
      err=grib_write_not_null_marker(fh);
      if (err) return err;
  
      err=grib_write_field(fh,tree->field);
      if (err) return err;
  
      err=grib_write_string(fh,tree->value);
      if (err) return err;
  
      err=grib_write_field_tree(fh,tree->next_level);
      if (err) return err;
  
      err=grib_write_field_tree(fh,tree->next);
      if (err) return err;
      return GRIB_SUCCESS;
  }
  
  grib_field_tree* grib_read_field_tree(grib_context* c, FILE* fh, grib_file** files, int *err)
  {
      grib_field_tree* tree=NULL;
      unsigned char marker=0;
      *err = grib_read_uchar(fh,&marker);
  
      if(marker == NULL_MARKER) return NULL;
      if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}
  
      tree = grib_context_malloc(c, sizeof(grib_field_tree));
      tree->field = grib_read_field(c,fh,files,err);
      if (*err) return NULL;
  
      tree->value = grib_read_string(c,fh,err);
      if (*err) return NULL;
  
      tree->next_level = grib_read_field_tree(c,fh,files,err);
      if (*err) return NULL;
  
      tree->next = grib_read_field_tree(c,fh,files,err);
      if (*err) return NULL;
  
      return tree;
  }
  
  grib_index* grib_index_new(grib_context* c,const char* key,int *err)
  {
      grib_index* index;
      grib_index_key* keys=NULL;
      char* q;
      int type;
      char* p;
  
      if (!strcmp(key,"mars"))
          return grib_index_new(c,mars_keys,err);
  
      p= grib_context_strdup(c,key);
      q=p;
  
      *err=0;
      if (!c) c=grib_context_get_default();
  
      index=(grib_index*)grib_context_malloc_clear(c,sizeof(grib_index));
      if (!index) {
          grib_context_log(c,GRIB_LOG_ERROR,"unable to create index");
          *err=GRIB_OUT_OF_MEMORY;
          return NULL;
      }
      index->context=c;
  
      while ((key=get_key(&p,&type))!=NULL) {
          keys=grib_index_new_key(c,keys,key,type,err);
          if (*err) return NULL;
      }
      index->keys=keys;
      index->fields=(grib_field_tree*)grib_context_malloc_clear(c,
              sizeof(grib_field_tree));
      if (!index->fields) {*err=GRIB_OUT_OF_MEMORY;return NULL;}
  
      grib_context_free(c,q);
      return index;
  }
  
  static void grib_index_values_delete(grib_context* c,grib_string_list* values)
  {
  
      if (!values) return;
  
      grib_index_values_delete(c,values->next);
      grib_context_free(c,values->value);
      grib_context_free(c,values);
  
      return ;
  }
  
  static void grib_index_key_delete(grib_context* c,grib_index_key* keys)
  {
      if (!keys) return;
  
      grib_index_key_delete(c,keys->next);
  
      grib_index_values_delete(c,keys->values);
      grib_index_values_delete(c,keys->current);
      grib_context_free(c,keys->name);
      grib_context_free(c,keys);
  
      return;
  }
  
  static long values_count=0;
  static grib_string_list* grib_read_key_values(grib_context* c,FILE* fh,int *err)
  {
      grib_string_list* values;
      unsigned char marker=0;
  
      *err = grib_read_uchar(fh,&marker);
      if(marker == NULL_MARKER) return NULL;
      if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}
  
      values_count++;
  
      values=grib_context_malloc_clear(c,sizeof(grib_string_list));
      values->value=grib_read_string(c,fh,err);
      if (*err) return NULL;
  
      values->next=grib_read_key_values(c,fh,err);
      if (*err) return NULL;
  
      return values;
  }
  
  static int grib_write_key_values(FILE* fh,grib_string_list* values)
  {
      int err=0;
  
      if (!values)
          return grib_write_null_marker(fh);
  
      err=grib_write_not_null_marker(fh);
      if (err) return err;
  
      err=grib_write_string(fh,values->value);
  
      err=grib_write_key_values(fh,values->next);
      if (err) return err;
  
      return GRIB_SUCCESS;
  }
  
  static grib_index_key* grib_read_index_keys(grib_context* c,FILE* fh,int *err)
  {
      grib_index_key* keys=NULL;
      unsigned char marker=0;
      unsigned char type=0;
  
      if (!c) c=grib_context_get_default();
  
      *err = grib_read_uchar(fh,&marker);
      if(marker == NULL_MARKER) return NULL;
      if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}
  
      keys= grib_context_malloc_clear(c,sizeof(grib_index_key));
      keys->name = grib_read_string(c,fh,err);
      if (*err) return NULL;
  
      *err = grib_read_uchar(fh,&type);
      keys->type=type;
      if (*err) return NULL;
  
      values_count=0;
      keys->values=grib_read_key_values(c,fh,err);
      if (*err) return NULL;
  
      keys->values_count=values_count;
      if (*err) return NULL;
  
  
      keys->next=grib_read_index_keys(c,fh,err);
      if (*err) return NULL;
  
      return keys;
  }
  
  static int grib_write_index_keys(FILE* fh,grib_index_key* keys)
  {
      int err=0;
  
      if (!keys)
          return grib_write_null_marker(fh);
  
      err=grib_write_not_null_marker(fh);
      if (err) return err;
  
      err=grib_write_string(fh,keys->name);
      if (err) return err;
  
      err=grib_write_uchar(fh,(unsigned char)keys->type);
      if (err) return err;
  
      grib_write_key_values(fh,keys->values);
  
      err=grib_write_index_keys(fh,keys->next);
      if (err) return err;
  
      return GRIB_SUCCESS;
  }
  
  static void grib_field_delete(grib_context* c,grib_field* field)
  {
      int err=0;
  
      if (!field) return;
  
      grib_field_delete(c,field->next);
  
      grib_file_close(field->file->name,&err);
      field->file=NULL;
  
      grib_context_free(c,field);
  
      return;
  
  }
  
  static void grib_field_tree_delete(grib_context* c,grib_field_tree* tree)
  {
      if(!tree)  return;
  
      grib_field_delete(c,tree->field);
      grib_context_free(c,tree->value);
  
      grib_field_tree_delete(c,tree->next_level);
  
      grib_field_tree_delete(c,tree->next);
  
      grib_context_free(c,tree);
  
      return;
  }
  
  void grib_index_delete(grib_index* index)
  {
      grib_file* file=index->files;
      grib_index_key_delete(index->context,index->keys);
      grib_field_tree_delete(index->context,index->fields);
      while (file) {
          grib_file* f=file;
          file=file->next;
          grib_file_delete(f);
      }
      grib_context_free(index->context,index);
  }
  
  static int grib_write_files(FILE* fh,grib_file* files)
  {
      int err;
      if (!files)
          return grib_write_null_marker(fh);
  
      err=grib_write_not_null_marker(fh);
      if (err) return err;
  
      err=grib_write_string(fh,files->name);
      if (err) return err;
  
      err=grib_write_short(fh,(short)files->id);
      if (err) return err;
  
      return grib_write_files(fh,files->next);
  }
  
  static grib_file* grib_read_files(grib_context *c, FILE* fh, int *err)
  {
      unsigned char marker=0;
      short id=0;
      grib_file* file;
      *err = grib_read_uchar(fh,&marker);
      if(marker == NULL_MARKER) return NULL;
      if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}
  
      file=grib_context_malloc(c,sizeof(grib_file));
      file->name=grib_read_string(c,fh,err);
      if (*err) return NULL;
  
      *err=grib_read_short(fh,&id);
      file->id=id;
      if (*err) return NULL;
  
      file->next=grib_read_files(c,fh,err);
      if (*err) return NULL;
  
      return file;
  }
  
  int grib_index_write(grib_index* index,const char* filename)
  {
      int err=0;
      FILE* fh;
      grib_file* files;
  
      fh=fopen(filename,"w");
      if (!fh) {
          grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
                  "Unable to write in file %s",filename);
          perror(filename);
          return GRIB_IO_PROBLEM;
      }
  
      err=grib_write_identifier(fh);
      if (err) {
          grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
                  "Unable to write in file %s",filename);
          perror(filename);
          return err;
      }
  
      if (!index) return grib_write_null_marker(fh);
  
      err=grib_write_not_null_marker(fh);
      if (err) return err;
  
      files=grib_file_pool_get_files();
      err=grib_write_files(fh,files);
      if (err) {
          grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
                  "Unable to write in file %s",filename);
          perror(filename);
          return err;
      }
  
  
      err=grib_write_index_keys(fh,index->keys);
      if (err) {
          grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
                  "Unable to write in file %s",filename);
          perror(filename);
          return err;
      }
  
      err=grib_write_field_tree(fh,index->fields);
      if (err) {
          grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
                  "Unable to write in file %s",filename);
          perror(filename);
          return err;
      }
  
      if (fclose(fh) != 0) {
          grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
                  "Unable to write in file %s",filename);
          perror(filename);
          return GRIB_IO_PROBLEM;
      }
  
      return err;
  }
  
  grib_index* grib_index_read(grib_context* c, const char* filename, int *err)
  {
      grib_file *file,*f;
      grib_file** files;
      grib_index* index=NULL;
      unsigned char marker=0;
      char* identifier=NULL;
      int max=0;
      FILE* fh;
  
      if (!c) c=grib_context_get_default();
  
      fh=fopen(filename,"rb");
      if (!fh) {
          grib_context* c = grib_context_get_default();
          grib_context_log(c,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
                  "Unable to write in file %s",filename);
          perror(filename);
          *err=GRIB_IO_PROBLEM;
          return NULL;
      }
  
      identifier=grib_read_string(c,fh,err);
      if (!identifier) return NULL;
      grib_context_free(c,identifier);
  
      *err = grib_read_uchar(fh,&marker);
      if(marker == NULL_MARKER) return NULL;
      if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}
  
      file = grib_read_files(c,fh,err);
      if (*err) return NULL;
  
      f=file;
      while (f) {
          if (max<f->id) max=f->id;
          f=f->next;
      }
  
      files=grib_context_malloc_clear(c,sizeof(grib_file)*(max+1));
  
      f=file;
      while (f) {
          grib_file_open(f->name,"r",err);
          if (*err) return NULL;
          files[f->id]=grib_get_file(f->name,err);
          f=f->next;
      }
  
      while (file) {
          f=file;
          file=file->next;
          grib_context_free(c,f->name);
          grib_context_free(c,f);
      }
  
      index=grib_context_malloc_clear(c,sizeof(grib_index));
      index->context=c;
  
      index->keys=grib_read_index_keys(c,fh,err);
      if (*err) return NULL;
  
      index_count=0;
      index->fields=grib_read_field_tree(c,fh,files,err);
      if (*err) return NULL;
  
      index->count=index_count;
  
      fclose(fh);
      return index;
  }
  
  int grib_index_search_same(grib_index* index, grib_handle* h)
  {
      int err=0;
      char buf[1024]={0,};
      size_t buflen=1024;
      grib_index_key* keys;
      long lval=0;
      double dval=0.0;
      grib_context* c;
  
      if (!index) return GRIB_NULL_INDEX;
      c=index->context;
  
      keys=index->keys;
  
      while (keys) {
          if (keys->type==GRIB_TYPE_UNDEFINED) {
              err=grib_get_native_type(h,keys->name,&(keys->type));
              if (err) keys->type=GRIB_TYPE_STRING;
          }
          buflen=1024;
          switch (keys->type) {
          case GRIB_TYPE_STRING:
              err=grib_get_string(h,keys->name,buf,&buflen);
              if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
              break;
          case GRIB_TYPE_LONG:
              err=grib_get_long(h,keys->name,&lval);
              if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
              else sprintf(buf,"%ld",lval);
              break;
          case GRIB_TYPE_DOUBLE:
              err=grib_get_double(h,keys->name,&dval);
              if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
              else sprintf(buf,"%g",dval);
              break;
          default :
              err=GRIB_WRONG_TYPE;
              return err;
          }
          if (err && err != GRIB_NOT_FOUND) {
              grib_context_log(c,GRIB_LOG_ERROR,
                      "unable to create index. \"%s\": %s",
                      keys->name,grib_get_error_message(err));
              return err;
          }
          sprintf(keys->value,"%s",buf);
          keys=keys->next;
      }
      grib_index_rewind(index);
      return 0;
  }
  
  int grib_index_add_file(grib_index* index, const char* filename)
  {
      double dval;
      size_t svallen;
      long length,lval;
      char buf[1024]={0,};
      int err=0;
      grib_file* indfile;
      grib_file* newfile;
  
      grib_index_key* index_key=NULL;
      grib_handle* h=NULL;
      grib_field* field;
      grib_field_tree* field_tree;
      grib_file* file=NULL;
      grib_context* c;
  
      if (!index) return GRIB_NULL_INDEX;
      c=index->context;
  
      file=grib_file_open(filename,"r",&err);
  
      if (!file || !file->handle) return err;
  
      if (!index->files) {
          grib_filesid++;
          newfile=grib_context_malloc_clear(c,sizeof(grib_file));
          newfile->id=grib_filesid;
          newfile->name=strdup(file->name);
          index->files=newfile;
      } else {
          indfile=index->files;
          while(indfile) {
              if (!strcmp(indfile->name,file->name)) return 0;
              indfile=indfile->next;
          }
          indfile=index->files;
          while(indfile->next) indfile=indfile->next;
          grib_filesid++;
          newfile=grib_context_malloc_clear(c,sizeof(grib_file));
          newfile->id=grib_filesid;
          newfile->name=strdup(file->name);
          indfile->next=newfile;
      }
  
      fseeko(file->handle,0,SEEK_SET);
  
      while ((h=grib_handle_new_from_file(c,file->handle,&err))!=NULL) {
          grib_string_list* v=0;
          index_key=index->keys;
          field_tree=index->fields;
          index_key->value[0]=0;
  
          /* process only GRIB for the moment*/
          svallen=1024;
          grib_get_string(h,"identifier",buf,&svallen);
          if (strcmp(buf,"GRIB")) {
              grib_handle_delete(h);
              return 0;
          }
  
          while (index_key) {
              if (index_key->type==GRIB_TYPE_UNDEFINED) {
                  err=grib_get_native_type(h,index_key->name,&(index_key->type));
                  if (err) index_key->type=GRIB_TYPE_STRING;
              }
              svallen=1024;
              switch (index_key->type) {
              case GRIB_TYPE_STRING:
                  err=grib_get_string(h,index_key->name,buf,&svallen);
                  if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
                  break;
              case GRIB_TYPE_LONG:
                  err=grib_get_long(h,index_key->name,&lval);
                  if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
                  else sprintf(buf,"%ld",lval);
                  break;
              case GRIB_TYPE_DOUBLE:
                  err=grib_get_double(h,index_key->name,&dval);
                  if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
                  else sprintf(buf,"%g",dval);
                  break;
              default :
                  err=GRIB_WRONG_TYPE;
                  return err;
              }
              if (err && err != GRIB_NOT_FOUND) {
                  grib_context_log(c,GRIB_LOG_ERROR,"unable to create index. \"%s\": %s",index_key->name,grib_get_error_message(err));
                  return err;
              }
  
              if (!index_key->values->value) {
                  index_key->values->value=grib_context_strdup(c,buf);
                  index_key->values_count++;
              } else {
                  v=index_key->values;
                  while (v->next && strcmp(v->value,buf)) v=v->next;
                  if (strcmp(v->value,buf)) {
                      index_key->values_count++;
                      if (v->next) v=v->next;
                      v->next=grib_context_malloc_clear(c,sizeof(grib_string_list));
                      v->next->value=grib_context_strdup(c,buf);
                  }
              }
  
              if (!field_tree->value) {
                  field_tree->value=grib_context_strdup(c,buf);
              } else {
                  while (field_tree->next &&
                          (field_tree->value==NULL ||
                                  strcmp(field_tree->value,buf)))
                      field_tree=field_tree->next;
  
                  if (!field_tree->value || strcmp(field_tree->value,buf)){
                      field_tree->next=
                              (grib_field_tree*)grib_context_malloc_clear(c,
                                      sizeof(grib_field_tree));
                      field_tree=field_tree->next;
                      field_tree->value=grib_context_strdup(c,buf);
                  }
              }
  
              if (index_key->next) {
                  if (!field_tree->next_level) {
                      field_tree->next_level=
                              grib_context_malloc_clear(c,sizeof(grib_field_tree));
                  }
                  field_tree=field_tree->next_level;
              }
              index_key=index_key->next;
          }
  
          field=grib_context_malloc_clear(c,sizeof(grib_field));
          field->file=file;
          index->count++;
          field->offset=h->offset;;
  
          err=grib_get_long(h,"totalLength",&length);
          if (err) return err;
          field->length=length;
  
  
          if (field_tree->field) {
              grib_field* pfield=field_tree->field;
              while (pfield->next) pfield=pfield->next;
              pfield->next=field;
          } else
              field_tree->field=field;
  
          if (h) grib_handle_delete(h);
  
      }
  
      grib_file_close(file->name,&err);
  
      if (err) return err;
      index->rewind=1;
      return GRIB_SUCCESS;
  }
  
  grib_index* grib_index_new_from_file(grib_context* c, char* filename, const char* keys, int *err)
  {
      grib_index* index=NULL;
  
      if(!c) c=grib_context_get_default();
  
      index=grib_index_new(c,keys,err);
  
      *err=grib_index_add_file(index,filename);
      if (*err) {
          grib_index_delete(index);
          return NULL;
      }
  
      return index;
  }
  
  int grib_index_get_size(grib_index* index,const char* key,size_t* size)
  {
      grib_index_key* k=index->keys;
      while (k && strcmp(k->name,key)) k=k->next;
      if (!k) return GRIB_NOT_FOUND;
      *size=k->values_count;
      return 0;
  }
  
  int grib_index_get_string(grib_index* index, const char* key, char** values, size_t *size)
  {
      grib_index_key* k=index->keys;
      grib_string_list* kv;
      int i=0;
      while (k && strcmp(k->name,key)) k=k->next;
      if (!k) return GRIB_NOT_FOUND;
      if (k->values_count>*size) return GRIB_ARRAY_TOO_SMALL;
      kv=k->values;
      while (kv) {
          values[i++]=grib_context_strdup(index->context,kv->value);
          kv=kv->next;
      }
      *size=k->values_count;
      qsort(values,*size,sizeof(char*),&compare_string);
  
      return GRIB_SUCCESS;
  }
  
  int grib_index_get_long(grib_index* index, const char* key, long* values, size_t *size)
  {
      grib_index_key* k=index->keys;
      grib_string_list* kv;
      int i=0;
      while (k && strcmp(k->name,key)) k=k->next;
      if (!k) return GRIB_NOT_FOUND;
      if (k->type != GRIB_TYPE_LONG) {
          grib_context_log(index->context,GRIB_LOG_ERROR,
                  "unable to get index %s as long");
          return GRIB_WRONG_TYPE;
      }
      if (k->values_count > *size) return GRIB_ARRAY_TOO_SMALL;
      kv=k->values;
      while (kv) {
          if (strcmp(kv->value,GRIB_KEY_UNDEF) )
              values[i++]=atol(kv->value);
          else
              values[i++]=UNDEF_LONG;
          kv=kv->next;
      }
      *size=k->values_count;
      qsort(values,*size,sizeof(long),&compare_long);
  
      return GRIB_SUCCESS;
  }
  
  int grib_index_get_double(grib_index* index,const char* key, double* values,size_t *size)
  {
      grib_index_key* k=index->keys;
      grib_string_list* kv;
      int i=0;
      while (k && strcmp(k->name,key)) k=k->next;
      if (!k) return GRIB_NOT_FOUND;
      if (k->type != GRIB_TYPE_DOUBLE) {
          grib_context_log(index->context,GRIB_LOG_ERROR,
                  "unable to get index %s as double");
          return GRIB_WRONG_TYPE;
      }
      if (k->values_count>*size) return GRIB_ARRAY_TOO_SMALL;
      kv=k->values;
      while (kv) {
          if (strcmp(kv->value,GRIB_KEY_UNDEF) )
              values[i++]=atof(kv->value);
          else
              values[i++]=UNDEF_DOUBLE;
  
          kv=kv->next;
      }
      *size=k->values_count;
      qsort(values,*size,sizeof(double),&compare_double);
  
      return GRIB_SUCCESS;
  }
  
  int grib_index_select_long(grib_index* index,const char* skey,long value)
  {
      grib_index_key* key=NULL;
      int err=GRIB_NOT_FOUND;
  
      if (!index) {
          grib_context* c=grib_context_get_default();
          grib_context_log(c,GRIB_LOG_ERROR,"null index pointer");
          return GRIB_INTERNAL_ERROR;
      }
      index->orderby=0;
      key=index->keys;
  
      while (key) {
          if (!strcmp(key->name,skey)) {
              err=0;
              break;
          }
          key=key->next;
      }
  
      if (err) {
          grib_context_log(index->context,GRIB_LOG_ERROR,
                  "key \"%s\" not found in index",skey);
          return err;
      }
  
      sprintf(key->value,"%ld",value);
      grib_index_rewind(index);
      return 0;
  }
  
  int grib_index_select_double(grib_index* index,const char* skey,double value)
  {
      grib_index_key* key=NULL;
      int err=GRIB_NOT_FOUND;
  
      if (!index) {
          grib_context* c=grib_context_get_default();
          grib_context_log(c,GRIB_LOG_ERROR,"null index pointer");
          return GRIB_INTERNAL_ERROR;
      }
      index->orderby=0;
      key=index->keys;
  
      while (key ) {
          if (!strcmp(key->name,skey)) {
              err=0;
              break;
          }
          key=key->next;
      }
  
      if (err) {
          grib_context_log(index->context,GRIB_LOG_ERROR,
                  "key \"%s\" not found in index",skey);
          return err;
      }
  
      sprintf(key->value,"%g",value);
      grib_index_rewind(index);
      return 0;
  }
  
  int grib_index_select_string(grib_index* index,const char* skey,char* value)
  {
      grib_index_key* key=NULL;
      int err=GRIB_NOT_FOUND;
  
      if (!index) {
          grib_context* c=grib_context_get_default();
          grib_context_log(c,GRIB_LOG_ERROR,"null index pointer");
          return GRIB_INTERNAL_ERROR;
      }
      index->orderby=0;
      key=index->keys;
  
      while (key ) {
          if (!strcmp(key->name,skey)) {
              err=0;
              break;
          }
          key=key->next;
      }
  
      if (err) {
          grib_context_log(index->context,GRIB_LOG_ERROR,
                  "key \"%s\" not found in index",skey);
          return err;
      }
  
      sprintf(key->value,"%s",value);
      grib_index_rewind(index);
      return 0;
  }
  
  grib_handle* grib_index_get_handle(grib_field* field,int *err)
  {
      grib_handle* h=NULL;
      grib_file_open(field->file->name,"r",err);
  
      if (*err!=GRIB_SUCCESS) return NULL;
  
      fseeko(field->file->handle,field->offset,SEEK_SET);
      h=grib_handle_new_from_file(0,field->file->handle,err);
      if (*err!=GRIB_SUCCESS) return NULL;
  
      grib_file_close(field->file->name,err);
      return h;
  }
  
  static int grib_index_execute(grib_index* index)
  {
      grib_index_key* keys = NULL;
      grib_field_tree* fields;
  
      if (!index) return GRIB_INTERNAL_ERROR;
      keys = index->keys;
  
      fields=index->fields;
      index->rewind=0;
  
      while (keys) {
          char* value;
          if (keys->value[0]) value=keys->value;
          else {
              grib_context_log(index->context,GRIB_LOG_ERROR,
                      "please select a value for index key \"%s\"",
                      keys->name);
              return GRIB_NOT_FOUND;
          }
  
          while (fields && strcmp(fields->value,value))
              fields=fields->next;
          if (fields && !strcmp(fields->value,value)) {
              if (fields->next_level) {
                  keys=keys->next;
                  fields=fields->next_level;
              } else {
                  index->current=index->fieldset;
                  while(index->current->next) index->current=index->current->next;
                  index->current->field=fields->field;
                  return 0;
              }
          } else return GRIB_END_OF_INDEX;
      }
  
      return 0;
  }
  
  void grib_index_dump(grib_index* index)
  {
  
  }
  
  char* grib_get_field_file(grib_index* index,off_t *offset)
  {
      char* file=NULL;
      if (index && index->current && index->current->field) {
          file=index->current->field->file->name;
          *offset=index->current->field->offset;
      }
      return file;
  }
  
  grib_handle* grib_handle_new_from_index(grib_index* index,int *err)
  {
      grib_index_key* keys;
      grib_field_list *fieldset,*next;
      grib_handle* h=NULL;
      grib_context* c=NULL;
  
      if (!index) return NULL;
      c=index->context;
      if (!index->rewind) {
          if (!index->current) {
              *err=GRIB_END_OF_INDEX;
              return NULL;
          }
  
          if (index->current->field->next)
              index->current->field=index->current->field->next;
          else if(index->current->next)
              index->current=index->current->next;
          else  {*err=GRIB_END_OF_INDEX;return NULL;}
  
          h=grib_index_get_handle(index->current->field,err);
          return h;
      }
  
      if (!index->fieldset) {
          index->fieldset=grib_context_malloc_clear(index->context,
                  sizeof(grib_field_list));
          if (!index->fieldset) {
              grib_context_log(index->context,GRIB_LOG_ERROR,
                      "unable to allocat %d bytes",
                      sizeof(grib_field_list));
              return NULL;
          }
          index->current=index->fieldset;
      } else {
          fieldset=index->fieldset;
          while(fieldset->next) {
              next=fieldset->next;
              grib_context_free(c,fieldset);
              fieldset=next;
          }
          fieldset->field=NULL;
          fieldset->next=NULL;
          index->fieldset=fieldset;
          index->current=fieldset;
      }
  
      *err=GRIB_END_OF_INDEX;
      h=NULL;
      keys=index->keys;
  
      if ((*err=grib_index_execute(index))==GRIB_SUCCESS) {
  
          if (!index->fieldset) {*err=GRIB_END_OF_INDEX;return NULL;}
          index->current=index->fieldset;
          h=grib_index_get_handle(index->current->field,err);
      }
      return h;
  }
  
  void grib_index_rewind(grib_index* index)
  {
      index->rewind=1;
  }
  
  static grib_index_key* search_key(grib_index_key* keys,grib_index_key* to_search)
  {
      if (!keys || !strcmp(keys->name,to_search->name)) return keys;
      return search_key(keys->next,to_search);
  }
  
  int grib_index_search(grib_index* index,grib_index_key* keys)
  {
      grib_index_key* ki=index->keys;
      grib_index_key* ks=keys;
  
      while (ks) {
          ki=search_key(ki,ks);
          if (!ki) {
              ki=index->keys;
              ki=search_key(ki,ks);
          }
          if (ki) sprintf(ki->value,"%s",ks->value);
          ks=ks->next;
      }
  
      grib_index_rewind(index);
      return 0;
  }
  