title: Removing all duplicate lines from a file using C
body: <p>In this question:
  <a href="https://stackoverflow.com/questions/10189594/detecting-duplicate-lines-on-file-using-c">Detecting duplicate lines on file using c</a>
  i can detect duplicate lines, but how we can remove this lines from our file?</p>
  
  <p>Thanks.</p>
  
  <p>Edit : To add my code :</p>
  
  <pre><code>#include &lt;stdio.h&gt;
  #include &lt;stdlib.h&gt;
  #include &lt;string.h&gt;
  
  struct somehash {
      struct somehash *next;
          unsigned hash;
          char *mem;
  };
  
  #define THE_SIZE 100000
  
  struct somehash *table[THE_SIZE] = { NULL,};
  
  struct somehash **some_find(char *str, unsigned len);
  static unsigned some_hash(char *str, unsigned len);
  
  int main (void)
  {
      char buffer[100];
      struct somehash **pp;
      size_t len;
      FILE * pFileIn;
      FILE * pFileOut;
  
      pFileIn  = fopen("in.csv", "r");
      pFileOut  = fopen("out.csv", "w+");
  
      if (pFileIn==NULL) perror ("Error opening input file");
      if (pFileOut==NULL) perror ("Error opening output file");
  
      while (fgets(buffer, sizeof buffer, pFileIn)) {
              len = strlen(buffer);
              pp = some_find(buffer, len);
              if (*pp) { /* found */
                  fprintf(stderr, "Duplicate:%s\n", buffer);
                  }
              else    
          {       /* not found: create one */
                      fprintf(stdout, "%s", buffer);
                      fprintf(pFileOut, "%s", buffer);
                      *pp = malloc(sizeof **pp);
                      (*pp)-&gt;next = NULL;
                      (*pp)-&gt;hash = some_hash(buffer,len);
                      (*pp)-&gt;mem = malloc(1+len);
                      memcpy((*pp)-&gt;mem , buffer,  1+len);
                  }
          }
  
  return 0;
  }
  
  struct somehash **some_find(char *str, unsigned len)
  {
      unsigned hash;
      unsigned short slot;
      struct somehash **hnd;
  
      hash = some_hash(str,len);
      slot = hash % THE_SIZE;
      for (hnd = &amp;table[slot]; *hnd ; hnd = &amp;(*hnd)-&gt;next ) {
          if ( (*hnd)-&gt;hash != hash) continue;
              if ( strcmp((*hnd)-&gt;mem , str) ) continue;
                  break;
          }
  
      return hnd;
  }
  
  static unsigned some_hash(char *str, unsigned len)
  {
      unsigned val;
      unsigned idx;
  
      if (!len) len = strlen(str);
  
      val = 0;
      for(idx=0; idx &lt; len; idx++ )   {
              val ^= (val &gt;&gt; 2) ^ (val &lt;&lt; 5) ^ (val &lt;&lt; 13) ^ str[idx] ^ 0x80001801;
      }
  
      return val;
  }
  </code></pre>
  
  <p>But in the output file we got always the first occurrence!</p>
  
  <p>Edit 2: To clarify: the intent is to find all duplicates in an input file. When there is more than one instance of a line in the input, that line should not appear in the output <em>at all</em>. The intent is not just to remove duplicates <em>of</em> that line so each occurs only once, but to remove <em>all</em> instances of a line if that line is duplicated in the input.</p>