title: Golang Program leaking memory
body: <p>I am trying to write a simple program that behaves like find | grep in golang. I have the program all working using goroutines using the following pattern:</p>
  
  <p>goroutine (filech &lt;- each file as found)
  goroutine (store files in extension based category &lt;- grepch)</p>
  
  <p>goroutine for each filech file (grepch &lt;- if file contains string)</p>
  
  <p>This all works as expected, but when presented with a large number of files, the memory just keeps growing and growing. I have looked into some of the profiling tools offered by Go, but I couldn't figure out how to find my memory leak. I can say that the memory is being used up mostly by bytes.makeSlice.</p>
  
  <p>Can anyone look at the code below and see what I am doing wrong? Also, I would like to know what is wrong with my code, but I would also like to learn how to debug this on my own in the future, so if you could give detailed profiling instructions for a problem such as this, that would be greatly appreciated.</p>
  
  <pre><code>package main
  
  import (
      "flag"
      "fmt"
      "io/ioutil"
      "os"
      "regexp"
      "runtime/pprof"
      "strings"
      "sync"
  )
  
  var (
      topDir      string
      cProf       bool
      mProf       bool
      cProfFile   *os.File
      mProfFile   *os.File
      fileNames   []string
      fileTypes   []string
      fileLists   map[string][]string
      cMatch      = regexp.MustCompile(`(?i)^.*\.(?:c|h|cc|cpp|c\+\+|hpp)$`)
      javaMatch   = regexp.MustCompile(`(?i)^.*\.(?:java|js)$`)
      goMatch     = regexp.MustCompile(`(?i)^.*\.(?:go)$`)
      buildMatch  = regexp.MustCompile(`(?i)^.*\.(?:gradle|mk|mka)$`)
      buildMatch2 = regexp.MustCompile(`^.*/(?:Makefile[^/\\]*)$`)
      regMatch    = regexp.MustCompile(`(?i)(?:test|debug)`)
  )
  
  func init() {
      fileLists = make(map[string][]string)
  }
  
  func main() {
      flag.StringVar(&amp;topDir, "d", ".", "The top level directory to process (default is current directory)")
      flag.BoolVar(&amp;cProf, "c", false, "Include if you want to save the CPU profile")
      flag.BoolVar(&amp;mProf, "m", false, "Include if you want to save the MEM profile")
      flag.Parse()
  
      cProfFunc()
  
      getFilesChan := make(chan string, 1000)
      grepFilesChan := make(chan string, 100)
  
      go getFileNamesOverChan(topDir, getFilesChan)
  
      var fileResult string
  
      var grepWg sync.WaitGroup
      var categorizeWg sync.WaitGroup
  
      fileTypes = append(fileTypes, "C", "Java", "Go", "Build", "Uncategorized")
  
      categorizeWg.Add(1)
      go func(chan string) {
          var grepResult string
          for grepResult = range grepFilesChan {
              if grepResult != "" {
                  fmt.Printf("Found file %s with text\n", grepResult)
                  var fileType = getFileCategory(grepResult)
                  fileLists[fileType] = append(fileLists[fileType], grepResult)
              }
          }
          categorizeWg.Done()
      }(grepFilesChan)
  
      for fileResult = range getFilesChan {
          if fileResult != "" {
              fileNames = append(fileNames, fileResult)
              grepWg.Add(1)
              go func(file string, ch chan string) {
                  fmt.Printf("Grepping file %s\n", file)
                  grepOverChan(file, ch)
                  grepWg.Done()
              }(fileResult, grepFilesChan)
          }
  
      }
  
      grepWg.Wait()
      close(grepFilesChan)
      categorizeWg.Wait()
  
      printSummary()
      mProfFunc()
  
      defer pprof.StopCPUProfile()
      defer cProfFile.Close()
  }
  
  func cProfFunc() {
      if cProf {
          cProfFile, _ = os.Create("cpu_profile.pprof")
          //handle err
          _ = pprof.StartCPUProfile(cProfFile)
          //handle err
      }
  
  }
  
  func mProfFunc() {
      if mProf {
          mProfFile, _ = os.Create("mem_profile.pprof")
          //handle err
          _ = pprof.WriteHeapProfile(mProfFile)
          //handle err
          defer mProfFile.Close()
      }
  }
  
  func printSummary() {
      fmt.Printf("\n\nProcessed %d Files\n\n", len(fileNames))
  
      fmt.Println("")
      fmt.Println("Found text in the following files:")
  
      for _, fType := range fileTypes {
          fmt.Printf("Found text in %d %s Files\n", len(fileLists[fType]), fType)
      }
      /*
          for _, fType := range fileTypes {
              if len(fileLists[fType]) &gt; 0 {
                  fmt.Println("")
                  fmt.Printf("\t%s Files:\n", fType)
              }
  
              for _, fileName := range fileLists[fType] {
                  fmt.Printf("\t\t%s\n", fileName)
              }
          }
      */
  }
  
  func getFileNamesOverChan(directory string, ch chan string) {
      fmt.Printf("Finding files in directory %s\n", directory)
      var err error
      var dirInfo os.FileInfo
  
      dirInfo, err = os.Lstat(directory)
      if err != nil {
          close(ch)
          return
      }
  
      if !dirInfo.IsDir() {
          close(ch)
          return
      }
  
      recursiveGetFilesOverChan(directory, ch)
  
      close(ch)
  }
  
  func recursiveGetFilesOverChan(dir string, ch chan string) {
      dirFile, _ := os.Open(dir)
      //handle err
      defer dirFile.Close()
  
      dirFileInfo, _ := dirFile.Readdir(0)
      //handle err
  
      for _, file := range dirFileInfo {
          filePath := fmt.Sprintf("%s%c%s", dir, os.PathSeparator, file.Name())
          switch mode := file.Mode(); {
          case mode.IsDir():
              //is a directory ... recurse
              recursiveGetFilesOverChan(filePath, ch)
          case mode.IsRegular():
              //is a regular file ... send it if it is not a CVS or GIT file
              if !strings.Contains(filePath, "/CVS/") &amp;&amp; !strings.Contains(filePath, "/.git/") {
                  fmt.Printf("Found File %s\n", filePath)
                  ch &lt;- filePath
              }
          case mode&amp;os.ModeSymlink != 0:
              //is a symbolic link ... skip it
              continue
          case mode&amp;os.ModeNamedPipe != 0:
              //is a Named Pipe ... skip it
              continue
          }
      }
  }
  
  func getFileCategory(file string) string {
      var fileType string
  
      switch {
      case cMatch.MatchString(file):
          fileType = "C"
      case javaMatch.MatchString(file):
          fileType = "Java"
      case goMatch.MatchString(file):
          fileType = "Go"
      case buildMatch.MatchString(file):
          fileType = "Build"
      case buildMatch2.MatchString(file):
          fileType = "Build"
      default:
          fileType = "Uncategorized"
      }
      return fileType
  }
  
  func grepOverChan(f string, ch chan string) {
      fileBytes, _ := ioutil.ReadFile(f)
      if regMatch.Match(fileBytes) {
          ch &lt;- f
      }
  }
  </code></pre>