text: //  GloVe: Global Vectors for Word Representation
  //
  //  Copyright (c) 2014 The Board of Trustees of
  //  The Leland Stanford Junior University. All Rights Reserved.
  //
  //  Licensed under the Apache License, Version 2.0 (the "License");
  //  you may not use this file except in compliance with the License.
  //  You may obtain a copy of the License at
  //
  //      http://www.apache.org/licenses/LICENSE-2.0
  //
  //  Unless required by applicable law or agreed to in writing, software
  //  distributed under the License is distributed on an "AS IS" BASIS,
  //  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  //  See the License for the specific language governing permissions and
  //  limitations under the License.
  //
  //
  //  For more information, bug reports, fixes, contact:
  //    Jeffrey Pennington (jpennin@stanford.edu)
  //    GlobalVectors@googlegroups.com
  //    http://nlp.stanford.edu/projects/glove/
  
  
  #include <stdint.h>
  #include <stdio.h>
  #include <stdlib.h>
  #include <string.h>
  #include <math.h>
  #include <pthread.h>
  #include <time.h>
  
  #define _FILE_OFFSET_BITS 64
  #define MAX_STRING_LENGTH 1000
  #define TSIZE 1048576
  #define HSEED 1159241
  
  #define HASHFN  bitwisehash
  
  typedef double real;
  
  typedef struct cooccur_rec {
      int word1;
      int word2;
      real val;
  } CREC;
  
  typedef struct hashrec {
      char *word;
      long long count;
      struct hashrec *next;
  } HASHREC;
  
  int write_header=0; //0=no, 1=yes; writes vocab_size/vector_size as first line for use with some libraries, such as gensim.
  int verbose = 2; // 0, 1, or 2
  int use_unk_vec = 1; // 0 or 1
  int num_threads = 8; // pthreads
  int num_iter = 25; // Number of full passes through cooccurrence matrix
  int vector_size = 50; // Word vector size
  int save_gradsq = 0; // By default don't save squared gradient values
  int use_binary = 0; // 0: save as text files; 1: save as binary; 2: both. For binary, save both word and context word vectors.
  int model = 2; // For text file output only. 0: concatenate word and context vectors (and biases) i.e. save everything; 1: Just save word vectors (no bias); 2: Save (word + context word) vectors (no biases)
  int checkpoint_every = 0; // checkpoint the model for every checkpoint_every iterations. Do nothing if checkpoint_every <= 0
  real eta = 0.05; // Initial learning rate
  real alpha = 0.75, x_max = 100.0; // Weighting function parameters, not extremely sensitive to corpus, though may need adjustment for very small or very large corpora
  real *W, *gradsq, *cost;
  long long num_lines, *lines_per_thread, vocab_size;
  char *vocab_file, *input_file, *save_W_file, *save_gradsq_file;
  int seed;
  
  char init_word_file[MAX_STRING_LENGTH], init_context_file[MAX_STRING_LENGTH];
  
  int *freeze_hash;
  
  char freeze_vocab[MAX_STRING_LENGTH];
  long long freeze_iter = 0;
  
  long long global_iter=0;
  
  /* Efficient string comparison */
  int scmp( char *s1, char *s2 ) {
      while (*s1 != '\0' && *s1 == *s2) {s1++; s2++;}
      return(*s1 - *s2);
  }
  
  /* Simple bitwise hash function */
  unsigned int bitwisehash(char *word, int tsize, unsigned int seed) {
      char c;
      unsigned int h;
      h = seed;
      for (; (c =* word) != '\0'; word++) h ^= ((h << 5) + c + (h >> 2));
      return((unsigned int)((h&0x7fffffff) % tsize));
  }
  
  /* Create hash table, initialise pointers to NULL */
  HASHREC ** inithashtable() {
      int i;
      HASHREC **ht;
      ht = (HASHREC **) malloc( sizeof(HASHREC *) * TSIZE );
      for (i = 0; i < TSIZE; i++) ht[i] = (HASHREC *) NULL;
      return(ht);
  }
  
  /* Search hash table for given string, insert if not found */
  long long hashinsert(HASHREC **ht, char *w, long long cnt, int insert) {
      HASHREC     *htmp, *hprv;
      unsigned int hval = HASHFN(w, TSIZE, HSEED);
      long long ret = -1;
  
      for (hprv = NULL, htmp = ht[hval]; htmp != NULL && scmp(htmp->word, w) != 0; hprv = htmp, htmp = htmp->next);
      if (htmp == NULL) {
          if (insert > 0) {
              htmp = (HASHREC *) malloc( sizeof(HASHREC) );
              htmp->word = (char *) malloc( strlen(w) + 1 );
              strcpy(htmp->word, w);
              htmp->count = cnt;
              htmp->next = NULL;
              if ( hprv==NULL )
                  ht[hval] = htmp;
              else
                  hprv->next = htmp;
              ret = htmp->count;
          }
      }
      else {
          /* new records are not moved to front */
          if (insert > 0) {
              htmp->count = cnt;
              if (hprv != NULL) {
                  /* move to front on access */
                  hprv->next = htmp->next;
                  htmp->next = ht[hval];
                  ht[hval] = htmp;
              }
          }
          ret = htmp->count;
      }
      return ret;
  }
  
  void initialize_parameters() {
      srand(seed);
      long long a, b;
      vector_size++; // Temporarily increment to allocate space for bias
  
      /* Allocate space for word vectors and context word vectors, and correspodning gradsq */
      a = posix_memalign((void **)&W, 128, 2 * vocab_size * vector_size * sizeof(real)); // Might perform better than malloc
      if (W == NULL) {
          fprintf(stderr, "Error allocating memory for W\n");
          exit(1);
      }
      a = posix_memalign((void **)&gradsq, 128, 2 * vocab_size * vector_size * sizeof(real)); // Might perform better than malloc
      if (gradsq == NULL) {
          fprintf(stderr, "Error allocating memory for gradsq\n");
          exit(1);
      }
      for (b = 0; b < vector_size; b++) {
          for (a = 0; a < 2 * vocab_size; a++) {
              W[a * vector_size + b] = (rand() / (real)RAND_MAX - 0.5) / vector_size;
          }
      }
  
      for (b = 0; b < vector_size; b++) {
          for (a = 0; a < 2 * vocab_size; a++) {
              gradsq[a * vector_size + b] = 1.0; // So initial value of eta is equal to initial learning rate
          }
      }
  
      HASHREC **vocab_hash = inithashtable();
  
      char str[MAX_STRING_LENGTH + 1];
      char format[20];
      sprintf(format,"%%%ds",MAX_STRING_LENGTH);
      FILE *fid;
      fid = fopen(vocab_file, "rb");
      long long cnt = 0;
      long long tmp;
      while (fscanf(fid, format, str) != EOF) { // Insert all tokens into hashtable
          hashinsert(vocab_hash, str, cnt, 1);
          fscanf(fid, "%lld", &tmp);
          cnt += 1;
      }
      fclose(fid);
  
      long long w_size, w_length;
      fid = fopen(init_word_file, "r");
      if (fid == NULL) {
          if (init_word_file[0] != 0) {
              printf("Init word error\n");
              exit(1);
          }
      }
  
      if (fid != NULL) {
          fscanf (fid, "%lld", &w_size);
          fscanf (fid, "%lld", &w_length);
          if (vector_size != w_length) {
              fprintf(stderr, "Vector length doesn't match.\n");
              return;
          }
  
          char word[MAX_STRING_LENGTH];
          long long idx;
          real value;
          sprintf(format,"%%%ds",MAX_STRING_LENGTH);
          for (a = 0; a < w_size; a++) {
              if (fscanf(fid,format,word) == 0) return;
              idx = hashinsert(vocab_hash, word, -1, 0);
              for (b = 0; b < w_length; b++) {
                  if (fscanf(fid," %lf",&value) == 0) return;
                  if (idx >= 0) W[idx * vector_size + b] = value;
              }
          }
          fclose(fid);
      }
  
      long long c_size, c_length;
      fid = fopen(init_context_file, "r");
      if (fid == NULL) {
          if (init_context_file[0] != 0) {
              printf("Init context error\n");
              exit(1);
          }
      }
  
      if (fid != NULL) {
          fscanf (fid, "%lld", &c_size);
          fscanf (fid, "%lld", &c_length);
          if (vector_size != c_length) {
              fprintf(stderr, "Vector length doesn't match.\n");
              return;
          }
  
          char word[MAX_STRING_LENGTH];
          long long idx;
          real value;
  
          sprintf(format,"%%%ds",MAX_STRING_LENGTH);
          for (a = 0; a < w_size; a++) {
              if (fscanf(fid,format,word) == 0) return;
              idx = hashinsert(vocab_hash, word, -1, 0);
              for (b = 0; b < c_length; b++) {
                  if (fscanf(fid," %lf",&value) == 0) return;
                  if (idx >= 0) W[vocab_size * vector_size + idx * vector_size + b] = value;
              }
          }
          fclose(fid);
      }
  
      vector_size--;
  
  }
  
  inline real check_nan(real update) {
      if (isnan(update) || isinf(update)) {
          fprintf(stderr,"\ncaught NaN in update");
          return 0.;
      } else {
          return update;
      }
  }
  
  /* Train the GloVe model */
  void *glove_thread(void *vid) {
      long long a, b ,l1, l2;
      long long id = *(long long*)vid;
      CREC cr;
      real diff, fdiff, temp1, temp2;
      FILE *fin;
      fin = fopen(input_file, "rb");
      fseeko(fin, (num_lines / num_threads * id) * (sizeof(CREC)), SEEK_SET); //Threads spaced roughly equally throughout file
      cost[id] = 0;
  
      real* W_updates1 = (real*)malloc(vector_size * sizeof(real));
      real* W_updates2 = (real*)malloc(vector_size * sizeof(real));
      for (a = 0; a < lines_per_thread[id]; a++) {
          fread(&cr, sizeof(CREC), 1, fin);
          if (feof(fin)) break;
          if (cr.word1 < 1 || cr.word2 < 1) { continue; }
  
          /* Get location of words in W & gradsq */
          l1 = (cr.word1 - 1LL) * (vector_size + 1); // cr word indices start at 1
          l2 = ((cr.word2 - 1LL) + vocab_size) * (vector_size + 1); // shift by vocab_size to get separate vectors for context words
  
          /* Calculate cost, save diff for gradients */
          diff = 0;
          for (b = 0; b < vector_size; b++) diff += W[b + l1] * W[b + l2]; // dot product of word and context word vector
          diff += W[vector_size + l1] + W[vector_size + l2] - log(cr.val); // add separate bias for each word
          fdiff = (cr.val > x_max) ? diff : pow(cr.val / x_max, alpha) * diff; // multiply weighting function (f) with diff
  
          // Check for NaN and inf() in the diffs.
          if (isnan(diff) || isnan(fdiff) || isinf(diff) || isinf(fdiff)) {
              fprintf(stderr,"Caught NaN in diff for kdiff for thread. Skipping update");
              continue;
          }
  
          cost[id] += 0.5 * fdiff * diff; // weighted squared error
  
          /* Adaptive gradient updates */
          fdiff *= eta; // for ease in calculating gradient
          real W_updates1_sum = 0;
          real W_updates2_sum = 0;
          for (b = 0; b < vector_size; b++) {
              // learning rate times gradient for word vectors
              temp1 = fdiff * W[b + l2];
              temp2 = fdiff * W[b + l1];
              // adaptive updates
              W_updates1[b] = temp1 / sqrt(gradsq[b + l1]);
              W_updates2[b] = temp2 / sqrt(gradsq[b + l2]);
              W_updates1_sum += W_updates1[b];
              W_updates2_sum += W_updates2[b];
              gradsq[b + l1] += temp1 * temp1;
              gradsq[b + l2] += temp2 * temp2;
          }
          if (!isnan(W_updates1_sum) && !isinf(W_updates1_sum) && !isnan(W_updates2_sum) && !isinf(W_updates2_sum)) {
              for (b = 0; b < vector_size; b++) {
                  if (freeze_hash[cr.word1 - 1LL] == 0 || global_iter >= freeze_iter)
                      W[b + l1] -= W_updates1[b];
                  if (freeze_hash[cr.word2 - 1LL] == 0 || global_iter >= freeze_iter)
                      W[b + l2] -= W_updates2[b];
              }
          }
  
          // updates for bias terms
          if (freeze_hash[cr.word1 - 1LL] == 0 || global_iter >= freeze_iter)
              W[vector_size + l1] -= check_nan(fdiff / sqrt(gradsq[vector_size + l1]));
          if (freeze_hash[cr.word2 - 1LL] == 0 || global_iter >= freeze_iter)
              W[vector_size + l2] -= check_nan(fdiff / sqrt(gradsq[vector_size + l2]));
          fdiff *= fdiff;
          gradsq[vector_size + l1] += fdiff;
          gradsq[vector_size + l2] += fdiff;
  
      }
      free(W_updates1);
      free(W_updates2);
  
      fclose(fin);
      pthread_exit(NULL);
  }
  
  /* Save params to file */
  int save_params(int nb_iter) {
      /*
       * nb_iter is the number of iteration (= a full pass through the cooccurrence matrix).
       *   nb_iter > 0 => checkpointing the intermediate parameters, so nb_iter is in the filename of output file.
       *   else        => saving the final paramters, so nb_iter is ignored.
       */
  
      long long a, b;
      char format[20];
      char output_file[MAX_STRING_LENGTH], output_file_gsq[MAX_STRING_LENGTH];
  
      char output_word_vector_file_full[MAX_STRING_LENGTH];
      char output_context_vector_file_full[MAX_STRING_LENGTH];
  
      char *word = malloc(sizeof(char) * MAX_STRING_LENGTH + 1);
      FILE *fid, *fout, *fgs;
      FILE *foutw, *foutc;
  
      if (use_binary > 0) { // Save parameters in binary file
          if (nb_iter <= 0)
              sprintf(output_file,"%s.bin",save_W_file);
          else
              sprintf(output_file,"%s.%03d.bin",save_W_file,nb_iter);
  
          fout = fopen(output_file,"wb");
          if (fout == NULL) {fprintf(stderr, "Unable to open file %s.\n",save_W_file); return 1;}
          for (a = 0; a < 2 * (long long)vocab_size * (vector_size + 1); a++) fwrite(&W[a], sizeof(real), 1,fout);
          fclose(fout);
          if (save_gradsq > 0) {
              if (nb_iter <= 0)
                  sprintf(output_file_gsq,"%s.bin",save_gradsq_file);
              else
                  sprintf(output_file_gsq,"%s.%03d.bin",save_gradsq_file,nb_iter);
  
              fgs = fopen(output_file_gsq,"wb");
              if (fgs == NULL) {fprintf(stderr, "Unable to open file %s.\n",save_gradsq_file); return 1;}
              for (a = 0; a < 2 * (long long)vocab_size * (vector_size + 1); a++) fwrite(&gradsq[a], sizeof(real), 1,fgs);
              fclose(fgs);
          }
      }
      if (use_binary != 1) { // Save parameters in text file
          if (nb_iter <= 0) {
              sprintf(output_file,"%s.txt",save_W_file);
  
              sprintf(output_word_vector_file_full,"%s.w.txt", save_W_file);
              sprintf(output_context_vector_file_full,"%s.c.txt", save_W_file);
          } else {
              sprintf(output_file,"%s.%03d.txt",save_W_file,nb_iter);
  
              sprintf(output_word_vector_file_full,"%s.%03d.w.txt", save_W_file, nb_iter);
              sprintf(output_context_vector_file_full,"%s.%03d.c.txt", save_W_file, nb_iter);
          }
          if (save_gradsq > 0) {
              if (nb_iter <= 0)
                  sprintf(output_file_gsq,"%s.txt",save_gradsq_file);
              else
                  sprintf(output_file_gsq,"%s.%03d.txt",save_gradsq_file,nb_iter);
  
              fgs = fopen(output_file_gsq,"wb");
              if (fgs == NULL) {fprintf(stderr, "Unable to open file %s.\n",save_gradsq_file); return 1;}
          }
          fout = fopen(output_file,"wb");
          if (fout == NULL) {fprintf(stderr, "Unable to open file %s.\n",output_file); return 1;}
  
          foutw = fopen(output_word_vector_file_full,"wb");
          if (foutw == NULL) {fprintf(stderr, "Unable to open file %s.\n",output_word_vector_file_full); return 1;}
          foutc = fopen(output_context_vector_file_full,"wb");
          if (foutc == NULL) {fprintf(stderr, "Unable to open file %s.\n",output_context_vector_file_full); return 1;}
  
  
          fid = fopen(vocab_file, "r");
          sprintf(format,"%%%ds",MAX_STRING_LENGTH);
          if (fid == NULL) {fprintf(stderr, "Unable to open file %s.\n",vocab_file); return 1;}
          if (write_header) fprintf(fout, "%lld %d\n", vocab_size, vector_size);
  
          fprintf(foutw, "%lld %d\n", vocab_size, vector_size+1);
          fprintf(foutc, "%lld %d\n", vocab_size, vector_size+1);
  
          for (a = 0; a < vocab_size; a++) {
              if (fscanf(fid,format,word) == 0) return 1;
              // input vocab cannot contain special <unk> keyword
              if (strcmp(word, "<unk>") == 0) return 1;
              fprintf(fout, "%s",word);
              if (model == 0) { // Save all parameters (including bias)
                  for (b = 0; b < (vector_size + 1); b++) fprintf(fout," %lf", W[a * (vector_size + 1) + b]);
                  for (b = 0; b < (vector_size + 1); b++) fprintf(fout," %lf", W[(vocab_size + a) * (vector_size + 1) + b]);
              }
              if (model == 1) // Save only "word" vectors (without bias)
                  for (b = 0; b < vector_size; b++) fprintf(fout," %lf", W[a * (vector_size + 1) + b]);
              if (model == 2) // Save "word + context word" vectors (without bias)
                  for (b = 0; b < vector_size; b++) fprintf(fout," %lf", W[a * (vector_size + 1) + b] + W[(vocab_size + a) * (vector_size + 1) + b]);
              fprintf(fout,"\n");
  
              // Save word embedding and context embedding (including bias)
              fprintf(foutw, "%s",word);
              fprintf(foutc, "%s",word);
              for (b = 0; b < (vector_size + 1); b++) fprintf(foutw," %lf", W[a * (vector_size + 1) + b]);
              for (b = 0; b < (vector_size + 1); b++) fprintf(foutc," %lf", W[(vocab_size + a) * (vector_size + 1) + b]);
              fprintf(foutw,"\n");
              fprintf(foutc,"\n");
  
              if (save_gradsq > 0) { // Save gradsq
                  fprintf(fgs, "%s",word);
                  for (b = 0; b < (vector_size + 1); b++) fprintf(fgs," %lf", gradsq[a * (vector_size + 1) + b]);
                  for (b = 0; b < (vector_size + 1); b++) fprintf(fgs," %lf", gradsq[(vocab_size + a) * (vector_size + 1) + b]);
                  fprintf(fgs,"\n");
              }
              if (fscanf(fid,format,word) == 0) return 1; // Eat irrelevant frequency entry
          }
  
          if (use_unk_vec) {
              real* unk_vec = (real*)calloc((vector_size + 1), sizeof(real));
              real* unk_context = (real*)calloc((vector_size + 1), sizeof(real));
              word = "<unk>";
  
              int num_rare_words = vocab_size < 100 ? vocab_size : 100;
  
              for (a = vocab_size - num_rare_words; a < vocab_size; a++) {
                  for (b = 0; b < (vector_size + 1); b++) {
                      unk_vec[b] += W[a * (vector_size + 1) + b] / num_rare_words;
                      unk_context[b] += W[(vocab_size + a) * (vector_size + 1) + b] / num_rare_words;
                  }
              }
  
              fprintf(fout, "%s",word);
  
              fprintf(foutw, "%s",word);
              fprintf(foutc, "%s",word);
  
              if (model == 0) { // Save all parameters (including bias)
                  for (b = 0; b < (vector_size + 1); b++) fprintf(fout," %lf", unk_vec[b]);
                  for (b = 0; b < (vector_size + 1); b++) fprintf(fout," %lf", unk_context[b]);
              }
              if (model == 1) // Save only "word" vectors (without bias)
                  for (b = 0; b < vector_size; b++) fprintf(fout," %lf", unk_vec[b]);
              if (model == 2) // Save "word + context word" vectors (without bias)
                  for (b = 0; b < vector_size; b++) fprintf(fout," %lf", unk_vec[b] + unk_context[b]);
              fprintf(fout,"\n");
  
              for (b = 0; b < (vector_size + 1); b++) fprintf(foutw," %lf", unk_vec[b]);
              for (b = 0; b < (vector_size + 1); b++) fprintf(foutc," %lf", unk_context[b]);
              fprintf(foutw,"\n");
              fprintf(foutc,"\n");
  
              free(unk_vec);
              free(unk_context);
          }
  
          fclose(fid);
          fclose(fout);
  
          fclose(foutw);
          fclose(foutc);
          if (save_gradsq > 0) fclose(fgs);
      }
      return 0;
  }
  
  /* Train model */
  int train_glove() {
      long long a, file_size;
      int save_params_return_code;
      int b;
      FILE *fin;
      real total_cost = 0;
  
      freeze_hash = (int *)calloc(vocab_size, sizeof(int));
  
      HASHREC **vocab_hash = inithashtable();
  
      char str[MAX_STRING_LENGTH + 1];
      char format[20];
      sprintf(format,"%%%ds",MAX_STRING_LENGTH);
      FILE *fid;
      fid = fopen(vocab_file, "rb");
      long long cnt = 0;
      long long tmp;
      while (fscanf(fid, format, str) != EOF) { // Insert all tokens into hashtable
          hashinsert(vocab_hash, str, cnt, 1);
          fscanf(fid, "%lld", &tmp);
          cnt += 1;
      }
      fclose(fid);
  
      fid = fopen(freeze_vocab, "rb");
  
      if (fid == NULL) {
          if (freeze_vocab[0] != 0) {
              printf("Freeze vocab error\n");
              exit(1);
          }
      }
  
      if (fid != NULL) {
          long long idx;
          while (fscanf(fid, format, str) != EOF) { // Check all freezen tokens into hashtable
              idx = hashinsert(vocab_hash, str, -1, 0);
              fscanf(fid, "%lld", &tmp);
              if (idx >= 0) freeze_hash[idx] = 1;
          }
          fclose(fid);
      }
  
      fprintf(stderr, "TRAINING MODEL\n");
  
      fin = fopen(input_file, "rb");
      if (fin == NULL) {fprintf(stderr,"Unable to open cooccurrence file %s.\n",input_file); return 1;}
      fseeko(fin, 0, SEEK_END);
      file_size = ftello(fin);
      num_lines = file_size/(sizeof(CREC)); // Assuming the file isn't corrupt and consists only of CREC's
      fclose(fin);
      fprintf(stderr,"Read %lld lines.\n", num_lines);
      if (verbose > 1) fprintf(stderr,"Initializing parameters...");
      initialize_parameters();
      if (verbose > 1) fprintf(stderr,"done.\n");
      if (verbose > 0) fprintf(stderr,"vector size: %d\n", vector_size);
      if (verbose > 0) fprintf(stderr,"vocab size: %lld\n", vocab_size);
      if (verbose > 0) fprintf(stderr,"x_max: %lf\n", x_max);
      if (verbose > 0) fprintf(stderr,"alpha: %lf\n", alpha);
      pthread_t *pt = (pthread_t *)malloc(num_threads * sizeof(pthread_t));
      lines_per_thread = (long long *) malloc(num_threads * sizeof(long long));
  
      time_t rawtime;
      struct tm *info;
      char time_buffer[80];
      // Lock-free asynchronous SGD
      struct timespec start, finish;
      double elapsed;
      clock_gettime(CLOCK_MONOTONIC, &start);
      for (b = 0; b < num_iter; b++) {
          total_cost = 0;
          global_iter = b;
          for (a = 0; a < num_threads - 1; a++) lines_per_thread[a] = num_lines / num_threads;
          lines_per_thread[a] = num_lines / num_threads + num_lines % num_threads;
          long long *thread_ids = (long long*)malloc(sizeof(long long) * num_threads);
          for (a = 0; a < num_threads; a++) thread_ids[a] = a;
          for (a = 0; a < num_threads; a++) pthread_create(&pt[a], NULL, glove_thread, (void *)&thread_ids[a]);
          for (a = 0; a < num_threads; a++) pthread_join(pt[a], NULL);
          for (a = 0; a < num_threads; a++) total_cost += cost[a];
          free(thread_ids);
  
          time(&rawtime);
          info = localtime(&rawtime);
          strftime(time_buffer,80,"%x - %I:%M.%S%p", info);
          fprintf(stderr, "%s, iter: %03d, cost: %lf\n", time_buffer,  b+1, total_cost/num_lines);
  
          if (checkpoint_every > 0 && (b + 1) % checkpoint_every == 0) {
              fprintf(stderr,"    saving itermediate parameters for iter %03d...", b+1);
              save_params_return_code = save_params(b+1);
              if (save_params_return_code != 0)
                  return save_params_return_code;
              fprintf(stderr,"done.\n");
          }
  
      }
      clock_gettime(CLOCK_MONOTONIC, &finish);
      elapsed = (finish.tv_sec - start.tv_sec);
      elapsed += (finish.tv_nsec - start.tv_nsec) / 1000000000.0;
      printf("Average epoch time: %lfs\n", elapsed / num_iter);
  
      free(pt);
      free(lines_per_thread);
      return save_params(0);
  }
  
  int find_arg(char *str, int argc, char **argv) {
      int i;
      for (i = 1; i < argc; i++) {
          if (!scmp(str, argv[i])) {
              if (i == argc - 1) {
                  printf("No argument given for %s\n", str);
                  exit(1);
              }
              return i;
          }
      }
      return -1;
  }
  
  int main(int argc, char **argv) {
      int i;
      FILE *fid;
      vocab_file = malloc(sizeof(char) * MAX_STRING_LENGTH);
      input_file = malloc(sizeof(char) * MAX_STRING_LENGTH);
      save_W_file = malloc(sizeof(char) * MAX_STRING_LENGTH);
      save_gradsq_file = malloc(sizeof(char) * MAX_STRING_LENGTH);
      int result = 0;
  
      if (argc == 1) {
          printf("GloVe: Global Vectors for Word Representation, v0.2\n");
          printf("Author: Jeffrey Pennington (jpennin@stanford.edu)\n\n");
          printf("Usage options:\n");
          printf("\t-verbose <int>\n");
          printf("\t\tSet verbosity: 0, 1, or 2 (default)\n");
          printf("\t-write-header <int>\n");
          printf("\t\tIf 1, write vocab_size/vector_size as first line. Do nothing if 0 (default).\n");
          printf("\t-vector-size <int>\n");
          printf("\t\tDimension of word vector representations (excluding bias term); default 50\n");
          printf("\t-threads <int>\n");
          printf("\t\tNumber of threads; default 8\n");
          printf("\t-iter <int>\n");
          printf("\t\tNumber of training iterations; default 25\n");
          printf("\t-eta <float>\n");
          printf("\t\tInitial learning rate; default 0.05\n");
          printf("\t-alpha <float>\n");
          printf("\t\tParameter in exponent of weighting function; default 0.75\n");
          printf("\t-x-max <float>\n");
          printf("\t\tParameter specifying cutoff in weighting function; default 100.0\n");
          printf("\t-binary <int>\n");
          printf("\t\tSave output in binary format (0: text, 1: binary, 2: both); default 0\n");
          printf("\t-model <int>\n");
          printf("\t\tModel for word vector output (for text output only); default 2\n");
          printf("\t\t   0: output all data, for both word and context word vectors, including bias terms\n");
          printf("\t\t   1: output word vectors, excluding bias terms\n");
          printf("\t\t   2: output word vectors + context word vectors, excluding bias terms\n");
          printf("\t-input-file <file>\n");
          printf("\t\tBinary input file of shuffled cooccurrence data (produced by 'cooccur' and 'shuffle'); default cooccurrence.shuf.bin\n");
          printf("\t-vocab-file <file>\n");
          printf("\t\tFile containing vocabulary (truncated unigram counts, produced by 'vocab_count'); default vocab.txt\n");
          printf("\t-save-file <file>\n");
          printf("\t\tFilename, excluding extension, for word vector output; default vectors\n");
          printf("\t-gradsq-file <file>\n");
          printf("\t\tFilename, excluding extension, for squared gradient output; default gradsq\n");
          printf("\t-save-gradsq <int>\n");
          printf("\t\tSave accumulated squared gradients; default 0 (off); ignored if gradsq-file is specified\n");
          printf("\t-checkpoint-every <int>\n");
          printf("\t\tCheckpoint a  model every <int> iterations; default 0 (off)\n");
          printf("\t-seed <int>\n");
          printf("\t\tSeed for embedding initialization\n");
  
          printf("\t-init-word <file>\n");
          printf("\t\tInit word vectors from <file>, not random initialization.\n");
          printf("\t-init-context <file>\n");
          printf("\t\tInit context vectors from <file>, not random initialization.\n");
  
          printf("\t-freeze-vocab <file>\n");
          printf("\t\tFreezen vocab from <file>.\n");
          printf("\t-freeze-iter <file>\n");
          printf("\t\tNumber of iteration to freeze vocab.\n");
  
          printf("\t-use-unk-vec <int>\n");
          printf("\t\tGenerate word embedding for unknown word; default 1\n");        
  
          printf("\nExample usage:\n");
          printf("./glove -input-file cooccurrence.shuf.bin -vocab-file vocab.txt -save-file vectors -gradsq-file gradsq -verbose 2 -vector-size 100 -threads 16 -alpha 0.75 -x-max 100.0 -eta 0.05 -binary 2 -model 2\n\n");
          result = 0;
      } else {
          if ((i = find_arg((char *)"-write-header", argc, argv)) > 0) write_header = atoi(argv[i + 1]);
          if ((i = find_arg((char *)"-verbose", argc, argv)) > 0) verbose = atoi(argv[i + 1]);
          if ((i = find_arg((char *)"-vector-size", argc, argv)) > 0) vector_size = atoi(argv[i + 1]);
          if ((i = find_arg((char *)"-iter", argc, argv)) > 0) num_iter = atoi(argv[i + 1]);
          if ((i = find_arg((char *)"-threads", argc, argv)) > 0) num_threads = atoi(argv[i + 1]);
          cost = malloc(sizeof(real) * num_threads);
          if ((i = find_arg((char *)"-alpha", argc, argv)) > 0) alpha = atof(argv[i + 1]);
          if ((i = find_arg((char *)"-x-max", argc, argv)) > 0) x_max = atof(argv[i + 1]);
          if ((i = find_arg((char *)"-eta", argc, argv)) > 0) eta = atof(argv[i + 1]);
          if ((i = find_arg((char *)"-binary", argc, argv)) > 0) use_binary = atoi(argv[i + 1]);
          if ((i = find_arg((char *)"-model", argc, argv)) > 0) model = atoi(argv[i + 1]);
          if (model != 0 && model != 1) model = 2;
          if ((i = find_arg((char *)"-save-gradsq", argc, argv)) > 0) save_gradsq = atoi(argv[i + 1]);
          if ((i = find_arg((char *)"-vocab-file", argc, argv)) > 0) strcpy(vocab_file, argv[i + 1]);
          else strcpy(vocab_file, (char *)"vocab.txt");
          if ((i = find_arg((char *)"-save-file", argc, argv)) > 0) strcpy(save_W_file, argv[i + 1]);
          else strcpy(save_W_file, (char *)"vectors");
          if ((i = find_arg((char *)"-gradsq-file", argc, argv)) > 0) {
              strcpy(save_gradsq_file, argv[i + 1]);
              save_gradsq = 1;
          }
          else if (save_gradsq > 0) strcpy(save_gradsq_file, (char *)"gradsq");
          if ((i = find_arg((char *)"-input-file", argc, argv)) > 0) strcpy(input_file, argv[i + 1]);
          else strcpy(input_file, (char *)"cooccurrence.shuf.bin");
          if ((i = find_arg((char *)"-checkpoint-every", argc, argv)) > 0) checkpoint_every = atoi(argv[i + 1]);
          if ((i = find_arg((char *)"-seed", argc, argv)) > 0) seed = atoi(argv[i + 1]);
  
          if ((i = find_arg((char *)"-init-word", argc, argv)) > 0) strcpy(init_word_file, argv[i + 1]);
          if ((i = find_arg((char *)"-init-context", argc, argv)) > 0) strcpy(init_context_file, argv[i + 1]);
  
          if ((i = find_arg((char *)"-freeze-vocab", argc, argv)) > 0) strcpy(freeze_vocab, argv[i + 1]);
          if ((i = find_arg((char *)"-freeze-iter", argc, argv)) > 0) freeze_iter = atoi(argv[i + 1]);
  
          if ((i = find_arg((char *)"-use-unk-vec", argc, argv)) > 0) use_unk_vec = atoi(argv[i + 1]);
          if (use_unk_vec != 0 && use_unk_vec != 1)
              use_unk_vec = 0;
  
          vocab_size = 0;
          fid = fopen(vocab_file, "r");
          if (fid == NULL) {fprintf(stderr, "Unable to open vocab file %s.\n",vocab_file); return 1;}
          while ((i = getc(fid)) != EOF) if (i == '\n') vocab_size++; // Count number of entries in vocab_file
          fclose(fid);
          result = train_glove();
          free(cost);
      }
      free(vocab_file);
      free(input_file);
      free(save_W_file);
      free(save_gradsq_file);
      return result;
  }
  