func HandleCombineImport(w http.ResponseWriter, r *http.Request) {
	log.Println("START handling", r.URL)
	log.Println("Proto", r.Proto, "TLS", r.TLS, "Host", r.Host)

	/*	if r.TLS == nil && r.Host != "localhost:8080" {
		log.Println("ERROR: Not handling non-https outside localhost", r.URL)
		return
	}*/

	w.Header().Add("Content-type", "application/json")

	ctx, bucket, _ := prepareContext()
	urls := readUrlFile(ctx, bucket)
	var combineImportData CombineImportData

	allFinalFilesExist := true
	finalFiles := make(map[int][]*storage.ObjectHandle)

	//var cLetters = [...]string{"c"}
	//for n := 2; n <= 2; n++ {
	//for _, letter := range cLetters {
	for n := 2; n <= 5; n++ {
		var newFinalFiles []*storage.ObjectHandle
		finalFiles[n] = newFinalFiles

		for _, letter := range validLetters {
			stringN := strconv.Itoa(n)
			finalFilename := getNgramTargetFilename(stringN, letter)
			finalObject := bucket.Object(finalFilename)

			// let's check if the file exists
			_, err := finalObject.Attrs(ctx)
			if err == nil {
				// log.Println("No need to process target file", targetFilename, "as it already exists and its size is", attrs.Size)
				finalFiles[n] = append(finalFiles[n], finalObject)
				continue
			} else {
				allFinalFilesExist = false
			}

			var sourceFiles []string
			var sourceObjects []*storage.ObjectHandle

			var totalSize int64 = 0
			allFilesExist := true
			log.Println("Checking if all source files exist for", finalFilename)

			for _, url := range urls {
				if isUrlForNgramAndLetter(url, stringN, letter) {
					sourceFilename := getNgramFilenameFromUrl(url, false)
					sourceObject := bucket.Object(sourceFilename)

					// let's check if the file exists
					attrs, err := sourceObject.Attrs(ctx)
					if err != nil {
						log.Println("The source file", sourceFilename, "does not exist")
						allFilesExist = false
						break
					} else {
						log.Println("The source file", sourceFilename, "exists and its size is", attrs.Size)
						totalSize += attrs.Size
					}

					sourceFiles = append(sourceFiles, sourceFilename)
					sourceObjects = append(sourceObjects, sourceObject)
				}
			}

			if allFilesExist {
				log.Println("All the source files exist and their total size is", totalSize, "- let's try composing them together")
				composer := finalObject.ComposerFrom(sourceObjects...)
				composer.ObjectAttrs.ContentType = "text/csv"
				_, err = composer.Run(ctx)
				if err != nil {
					log.Println("Error when composing:", err)
				} else {
					var combinedFile CombinedFile
					combinedFile.N = n
					combinedFile.Letter = letter
					combinedFile.SourceFiles = sourceFiles
					combinedFile.TargetFile = finalFilename
					combineImportData.CombinedFiles = append(combineImportData.CombinedFiles, combinedFile)

					log.Println("Composing successful for", finalFilename, " - deleting source files")
					for _, objectToDelete := range sourceObjects {
						err := objectToDelete.Delete(ctx)
						if err != nil {
							log.Println("Coudln't delete file", objectToDelete.ObjectName())
						}
					}
				}
			} else {
				log.Println("Not all source files exist for", finalFilename)
			}
		}
	}

	if allFinalFilesExist {
		log.Println("All target files exist, let's make a FINAL file")

		var allFinalNObjects []*storage.ObjectHandle
		allFinalNFilesOK := true
		for n := 2; n <= 5; n++ {
			finalNFilename := fmt.Sprintf("Final_%dgram.csv", n)
			FinalNObject := bucket.Object(finalNFilename)

			log.Printf("Composing %dgram: %s", n, finalNFilename)
			finalNComposer := FinalNObject.ComposerFrom(finalFiles[n]...)
			finalNComposer.ObjectAttrs.ContentType = "text/csv"
			_, err := finalNComposer.Run(ctx)
			if err != nil {
				log.Println("Error when composing the final N file:", err)
				allFinalNFilesOK = false
				break
			} else {
				allFinalNObjects = append(allFinalNObjects, FinalNObject)
			}
		}

		if allFinalNFilesOK {
			bigQueryImportFilename := "_ngram_BigQueryImport.csv"
			bigQueryImportObject := bucket.Object(bigQueryImportFilename)

			log.Printf("Composing FINAL file: %s", bigQueryImportFilename)

			bigQueryComposer := bigQueryImportObject.ComposerFrom(allFinalNObjects...)
			bigQueryComposer.ObjectAttrs.ContentType = "text/csv"
			_, err := bigQueryComposer.Run(ctx)
			if err != nil {
				log.Println("Error when composing the BigQuery import file:", err)
			} else {
				log.Printf("FINAL file composed!")

				var sourceFiles []string
				sourceFiles = append(sourceFiles, "(all final files)")
				var combinedFile CombinedFile
				combinedFile.N = 25
				combinedFile.Letter = "a-z"
				combinedFile.SourceFiles = sourceFiles
				combinedFile.TargetFile = bigQueryImportFilename
				combineImportData.CombinedFiles = append(combineImportData.CombinedFiles, combinedFile)
			}
		}
	}

	resultJson, err := json.Marshal(combineImportData)

	if err != nil {
		log.Fatal("Error when JSONing the result: ", err)
	}

	w.Write(resultJson)

	log.Println("DONE handling", r.URL)
}
// HandleCombineImport handles the /combine-import URL and does all the work
// It combines individual ngram files into one ngram file per letter using the Composer
HandleCombineImport handles the /combine-import URL and does all the work
It combines individual ngram files into one ngram file per letter using the Composer
// HandleCombineImport handles the / combine - import URL and does all the work // It combines individual ngram files into one ngram file per letter using the Composer
func HandleCombineImport(w http.ResponseWriter, r *http.Request) {
	log.Println("START handling", r.URL)
	log.Println("Proto", r.Proto, "TLS", r.TLS, "Host", r.Host)
	w.Header().Add("Content-type", "application/json")
	ctx, bucket, _ := prepareContext()
	urls := readUrlFile(ctx, bucket)
	var combineImportData CombineImportData
	allFinalFilesExist := true
	finalFiles := make(map[int][]*storage.ObjectHandle)
	for n := 2; n <= 5; n++ {
		var newFinalFiles []*storage.ObjectHandle
		finalFiles[n] = newFinalFiles
		for _, letter := range validLetters {
			stringN := strconv.Itoa(n)
			finalFilename := getNgramTargetFilename(stringN, letter)
			finalObject := bucket.Object(finalFilename)
			_, err := finalObject.Attrs(ctx)
			if err == nil {
				finalFiles[n] = append(finalFiles[n], finalObject)
				continue
			} else {
				allFinalFilesExist = false
			}
			var sourceFiles []string
			var sourceObjects []*storage.ObjectHandle
			var totalSize int64 = 0
			allFilesExist := true
			log.Println("Checking if all source files exist for", finalFilename)
			for _, url := range urls {
				if isUrlForNgramAndLetter(url, stringN, letter) {
					sourceFilename := getNgramFilenameFromUrl(url, false)
					sourceObject := bucket.Object(sourceFilename)
					attrs, err := sourceObject.Attrs(ctx)
					if err != nil {
						log.Println("The source file", sourceFilename, "does not exist")
						allFilesExist = false
						break
					} else {
						log.Println("The source file", sourceFilename, "exists and its size is", attrs.Size)
						totalSize += attrs.Size
					}
					sourceFiles = append(sourceFiles, sourceFilename)
					sourceObjects = append(sourceObjects, sourceObject)
				}
			}
			if allFilesExist {
				log.Println("All the source files exist and their total size is", totalSize, "- let's try composing them together")
				composer := finalObject.ComposerFrom(sourceObjects...)
				composer.ObjectAttrs.ContentType = "text/csv"
				_, err = composer.Run(ctx)
				if err != nil {
					log.Println("Error when composing:", err)
				} else {
					var combinedFile CombinedFile
					combinedFile.N = n
					combinedFile.Letter = letter
					combinedFile.SourceFiles = sourceFiles
					combinedFile.TargetFile = finalFilename
					combineImportData.CombinedFiles = append(combineImportData.CombinedFiles, combinedFile)
					log.Println("Composing successful for", finalFilename, " - deleting source files")
					for _, objectToDelete := range sourceObjects {
						err := objectToDelete.Delete(ctx)
						if err != nil {
							log.Println("Coudln't delete file", objectToDelete.ObjectName())
						}
					}
				}
			} else {
				log.Println("Not all source files exist for", finalFilename)
			}
		}
	}
	if allFinalFilesExist {
		log.Println("All target files exist, let's make a FINAL file")
		var allFinalNObjects []*storage.ObjectHandle
		allFinalNFilesOK := true
		for n := 2; n <= 5; n++ {
			finalNFilename := fmt.Sprintf("Final_%dgram.csv", n)
			FinalNObject := bucket.Object(finalNFilename)
			log.Printf("Composing %dgram: %s", n, finalNFilename)
			finalNComposer := FinalNObject.ComposerFrom(finalFiles[n]...)
			finalNComposer.ObjectAttrs.ContentType = "text/csv"
			_, err := finalNComposer.Run(ctx)
			if err != nil {
				log.Println("Error when composing the final N file:", err)
				allFinalNFilesOK = false
				break
			} else {
				allFinalNObjects = append(allFinalNObjects, FinalNObject)
			}
		}
		if allFinalNFilesOK {
			bigQueryImportFilename := "_ngram_BigQueryImport.csv"
			bigQueryImportObject := bucket.Object(bigQueryImportFilename)
			log.Printf("Composing FINAL file: %s", bigQueryImportFilename)
			bigQueryComposer := bigQueryImportObject.ComposerFrom(allFinalNObjects...)
			bigQueryComposer.ObjectAttrs.ContentType = "text/csv"
			_, err := bigQueryComposer.Run(ctx)
			if err != nil {
				log.Println("Error when composing the BigQuery import file:", err)
			} else {
				log.Printf("FINAL file composed!")
				var sourceFiles []string
				sourceFiles = append(sourceFiles, "(all final files)")
				var combinedFile CombinedFile
				combinedFile.N = 25
				combinedFile.Letter = "a-z"
				combinedFile.SourceFiles = sourceFiles
				combinedFile.TargetFile = bigQueryImportFilename
				combineImportData.CombinedFiles = append(combineImportData.CombinedFiles, combinedFile)
			}
		}
	}
	resultJson, err := json.Marshal(combineImportData)
	if err != nil {
		log.Fatal("Error when JSONing the result: ", err)
	}
	w.Write(resultJson)
	log.Println("DONE handling", r.URL)
}
func HandleCombineImport ( w http . ResponseWriter , r * http . Request ) { log . Println ( " " , r . URL ) log . Println ( " " , r . Proto , " " , r . TLS , " " , r . Host ) w . Header ( ) . Add ( " " , " " ) ctx , bucket , _ := prepareContext ( ) urls := readUrlFile ( ctx , bucket ) var combineImportData CombineImportData allFinalFilesExist := true finalFiles := make ( map [ int ] [ ] * storage . ObjectHandle ) for n := 2 ; n <= 5 ; n ++ { var newFinalFiles [ ] * storage . ObjectHandle finalFiles [ n ] = newFinalFiles for _ , letter := range validLetters { stringN := strconv . Itoa ( n ) finalFilename := getNgramTargetFilename ( stringN , letter ) finalObject := bucket . Object ( finalFilename ) _ , err := finalObject . Attrs ( ctx ) if err == nil { finalFiles [ n ] = append ( finalFiles [ n ] , finalObject ) continue } else { allFinalFilesExist = false } var sourceFiles [ ] string var sourceObjects [ ] * storage . ObjectHandle var totalSize int64 = 0 allFilesExist := true log . Println ( " " , finalFilename ) for _ , url := range urls { if isUrlForNgramAndLetter ( url , stringN , letter ) { sourceFilename := getNgramFilenameFromUrl ( url , false ) sourceObject := bucket . Object ( sourceFilename ) attrs , err := sourceObject . Attrs ( ctx ) if err != nil { log . Println ( " " , sourceFilename , " " ) allFilesExist = false break } else { log . Println ( " " , sourceFilename , " " , attrs . Size ) totalSize += attrs . Size } sourceFiles = append ( sourceFiles , sourceFilename ) sourceObjects = append ( sourceObjects , sourceObject ) } } if allFilesExist { log . Println ( " " , totalSize , " " ) composer := finalObject . ComposerFrom ( sourceObjects ... ) composer . ObjectAttrs . ContentType = " " _ , err = composer . Run ( ctx ) if err != nil { log . Println ( " " , err ) } else { var combinedFile CombinedFile combinedFile . N = n combinedFile . Letter = letter combinedFile . SourceFiles = sourceFiles combinedFile . TargetFile = finalFilename combineImportData . CombinedFiles = append ( combineImportData . CombinedFiles , combinedFile ) log . Println ( " " , finalFilename , " " ) for _ , objectToDelete := range sourceObjects { err := objectToDelete . Delete ( ctx ) if err != nil { log . Println ( " " , objectToDelete . ObjectName ( ) ) } } } } else { log . Println ( " " , finalFilename ) } } } if allFinalFilesExist { log . Println ( " " ) var allFinalNObjects [ ] * storage . ObjectHandle allFinalNFilesOK := true for n := 2 ; n <= 5 ; n ++ { finalNFilename := fmt . Sprintf ( " " , n ) FinalNObject := bucket . Object ( finalNFilename ) log . Printf ( " " , n , finalNFilename ) finalNComposer := FinalNObject . ComposerFrom ( finalFiles [ n ] ... ) finalNComposer . ObjectAttrs . ContentType = " " _ , err := finalNComposer . Run ( ctx ) if err != nil { log . Println ( " " , err ) allFinalNFilesOK = false break } else { allFinalNObjects = append ( allFinalNObjects , FinalNObject ) } } if allFinalNFilesOK { bigQueryImportFilename := " " bigQueryImportObject := bucket . Object ( bigQueryImportFilename ) log . Printf ( " " , bigQueryImportFilename ) bigQueryComposer := bigQueryImportObject . ComposerFrom ( allFinalNObjects ... ) bigQueryComposer . ObjectAttrs . ContentType = " " _ , err := bigQueryComposer . Run ( ctx ) if err != nil { log . Println ( " " , err ) } else { log . Printf ( " " ) var sourceFiles [ ] string sourceFiles = append ( sourceFiles , " " ) var combinedFile CombinedFile combinedFile . N = 25 combinedFile . Letter = " " combinedFile . SourceFiles = sourceFiles combinedFile . TargetFile = bigQueryImportFilename combineImportData . CombinedFiles = append ( combineImportData . CombinedFiles , combinedFile ) } } } resultJson , err := json . Marshal ( combineImportData ) if err != nil { log . Fatal ( " " , err ) } w . Write ( resultJson ) log . Println ( " " , r . URL ) }
HandleCombineImport handles the /combine-import URL and does all the work
It combines individual ngram files into one ngram file per letter using the Composer
HandleCombineImport handles the / combine - import URL and does all the work It combines individual ngram files into one ngram file per letter using the Composer
/*	if r.TLS == nil && r.Host != "localhost:8080" {
		log.Println("ERROR: Not handling non-https outside localhost", r.URL)
		return
	}*/ //var cLetters = [...]string{"c"} //for n := 2; n <= 2; n++ { //for _, letter := range cLetters { // let's check if the file exists // log.Println("No need to process target file", targetFilename, "as it already exists and its size is", attrs.Size) // let's check if the file exists