func prepareNgram(textBytes, yearBytes, matchesBytes, volumesBytes []byte, ngram *Ngram, previousNgram *Ngram) bool {
	text := string(textBytes)
	ngram.OriginalText = text
	isTheSame := false

	if previousNgram.OriginalText == ngram.OriginalText {
		isTheSame = true

		// yay, we don't have to process the text
		ngram.Text = previousNgram.Text
		ngram.Frequency = previousNgram.Frequency
	} else {
		ngram.Text = processNgram(text)
		ngram.Frequency = 0
	}

	// if the text is empty, it means it doesn't make sense and we don't want it; for example "B.B. --_."
	if ngram.Text != "" {
		year := int(yearBytes[0]-48)*1000 + int(yearBytes[1]-48)*100 + int(yearBytes[2]-48)*10 + int(yearBytes[3]-48)
		matches := convertAsciiNumberToInt(matchesBytes)
		volumes := convertAsciiNumberToInt(volumesBytes)

		yearBonus := 1
		if year > 1980 {
			yearBonus += (year - 1980) / 10
		}

		volumesBonus := 1 + volumes/10
		if volumesBonus > 4 {
			volumesBonus = 4
		}

		frequency := matches * yearBonus * volumesBonus
		ngram.Frequency += frequency
	}

	return isTheSame
}
// prepareNgram takes the data from one line, processes it and prepares an Ngram that it returns
// it also returns true if the ngram is the same as the previous one so that the calling function can handle it
// if there was no previous ngram, it returns false
prepareNgram takes the data from one line, processes it and prepares an Ngram that it returns
it also returns true if the ngram is the same as the previous one so that the calling function can handle it
if there was no previous ngram, it returns false
// prepareNgram takes the data from one line processes it and prepares an Ngram that it returns // it also returns true if the ngram is the same as the previous one so that the calling function can handle it // if there was no previous ngram it returns false
func prepareNgram(textBytes, yearBytes, matchesBytes, volumesBytes []byte, ngram *Ngram, previousNgram *Ngram) bool {
	text := string(textBytes)
	ngram.OriginalText = text
	isTheSame := false
	if previousNgram.OriginalText == ngram.OriginalText {
		isTheSame = true
		ngram.Text = previousNgram.Text
		ngram.Frequency = previousNgram.Frequency
	} else {
		ngram.Text = processNgram(text)
		ngram.Frequency = 0
	}
	if ngram.Text != "" {
		year := int(yearBytes[0]-48)*1000 + int(yearBytes[1]-48)*100 + int(yearBytes[2]-48)*10 + int(yearBytes[3]-48)
		matches := convertAsciiNumberToInt(matchesBytes)
		volumes := convertAsciiNumberToInt(volumesBytes)
		yearBonus := 1
		if year > 1980 {
			yearBonus += (year - 1980) / 10
		}
		volumesBonus := 1 + volumes/10
		if volumesBonus > 4 {
			volumesBonus = 4
		}
		frequency := matches * yearBonus * volumesBonus
		ngram.Frequency += frequency
	}
	return isTheSame
}
func prepareNgram ( textBytes , yearBytes , matchesBytes , volumesBytes [ ] byte , ngram * Ngram , previousNgram * Ngram ) bool { text := string ( textBytes ) ngram . OriginalText = text isTheSame := false if previousNgram . OriginalText == ngram . OriginalText { isTheSame = true ngram . Text = previousNgram . Text ngram . Frequency = previousNgram . Frequency } else { ngram . Text = processNgram ( text ) ngram . Frequency = 0 } if ngram . Text != " " { year := int ( yearBytes [ 0 ] - 48 ) * 1000 + int ( yearBytes [ 1 ] - 48 ) * 100 + int ( yearBytes [ 2 ] - 48 ) * 10 + int ( yearBytes [ 3 ] - 48 ) matches := convertAsciiNumberToInt ( matchesBytes ) volumes := convertAsciiNumberToInt ( volumesBytes ) yearBonus := 1 if year > 1980 { yearBonus += ( year - 1980 ) / 10 } volumesBonus := 1 + volumes / 10 if volumesBonus > 4 { volumesBonus = 4 } frequency := matches * yearBonus * volumesBonus ngram . Frequency += frequency } return isTheSame }
prepareNgram takes the data from one line, processes it and prepares an Ngram that it returns
it also returns true if the ngram is the same as the previous one so that the calling function can handle it
if there was no previous ngram, it returns false
prepareNgram takes the data from one line processes it and prepares an Ngram that it returns it also returns true if the ngram is the same as the previous one so that the calling function can handle it if there was no previous ngram it returns false
// yay, we don't have to process the text // if the text is empty, it means it doesn't make sense and we don't want it; for example "B.B. --_."