func processInvalidManifest(manifest io.Reader) ([][]string, error) {
	output := [][]string{}

	header := []string{}
	dateField := -1
	i := 0

	// Read over each line of the invalid manifest
	r := bufio.NewReader(manifest)
	for {
		line, err := r.ReadString('\n')
		if err == io.EOF && len(strings.TrimSpace(line)) == 0 {
			// Can't always break if you see an EOF. Specifically, in some
			// files, the final line has no valid ending (neither an EOF nor a
			// newline). In that case, since the entire line still remains
			// despite receiving an EOF signal, so you want to process that.
			// When you then loop around a final time, there will be nothing in
			// the line, just EOF, so it's safe (and necessary) to break.
			break
		} else if err == io.EOF {
			// We hit the end of the file, but we had non-empty data in the
			// line. See if we need to process it. But, after this, we're done.
			// drainBadBuffer = true
		} else if err != nil {
			return nil, err
		}

		// Split into columns, assuming commas are the delimiter
		if i == 0 {
			record := strings.Split(strings.TrimSpace(strings.Replace(line, "discription", "description", -1)), ",")

			header = append(header, record...)
			for i, col := range record {
				if col == "date" {
					dateField = i
					break
				}
			}

			if dateField == -1 {
				return nil, fmt.Errorf("No field named 'date' was found in the manifest")
			}

			output = append(output, header)
			i++

			continue
		}

		// Process non-header records
		record := strings.Split(strings.TrimSpace(line), ",")

		// For now, if there are fewer entries in the field than in the header,
		// we assume it's wrong (Could consider an invalidLayoutX that tries to
		// handle this)
		if len(record) < len(header) {
			log.Println("Skipping record due to being shorter than the header")
			spew.Fdump(os.Stderr, record)
			continue
		}

		// If the invalidLayout1 (basically, assuming a comma in the date field)
		// fails, then we will emit that to stderr and skip the bad record, but
		// continue otherwise.
		parsed, err := invalidLayout1(record, len(header), dateField)
		if err != nil {
			log.Println("Skipping record due to error in parsing:", err.Error())
			spew.Fdump(os.Stderr, record)

			continue
		}

		output = append(output, parsed)

		i++
	}

	return output, nil
}
// Many of the manifest.csv files are invalid. They use commas as delimiters and
// do not use quotes, and yet they have fields (like date) that use commas. Some
// are literally malformed, ending after 1-2 fields, or containing run-ons of
// multiple fields without proper line-breaks. So this requires some processing.
Many of the manifest.csv files are invalid. They use commas as delimiters and
do not use quotes, and yet they have fields (like date) that use commas. Some
are literally malformed, ending after 1-2 fields, or containing run-ons of
multiple fields without proper line-breaks. So this requires some processing.
// Many of the manifest . csv files are invalid . They use commas as delimiters and // do not use quotes and yet they have fields ( like date ) that use commas . Some // are literally malformed ending after 1 - 2 fields or containing run - ons of // multiple fields without proper line - breaks . So this requires some processing .
func processInvalidManifest(manifest io.Reader) ([][]string, error) {
	output := [][]string{}
	header := []string{}
	dateField := -1
	i := 0
	r := bufio.NewReader(manifest)
	for {
		line, err := r.ReadString('\n')
		if err == io.EOF && len(strings.TrimSpace(line)) == 0 {
			break
		} else if err == io.EOF {
		} else if err != nil {
			return nil, err
		}
		if i == 0 {
			record := strings.Split(strings.TrimSpace(strings.Replace(line, "discription", "description", -1)), ",")
			header = append(header, record...)
			for i, col := range record {
				if col == "date" {
					dateField = i
					break
				}
			}
			if dateField == -1 {
				return nil, fmt.Errorf("No field named 'date' was found in the manifest")
			}
			output = append(output, header)
			i++
			continue
		}
		record := strings.Split(strings.TrimSpace(line), ",")
		if len(record) < len(header) {
			log.Println("Skipping record due to being shorter than the header")
			spew.Fdump(os.Stderr, record)
			continue
		}
		parsed, err := invalidLayout1(record, len(header), dateField)
		if err != nil {
			log.Println("Skipping record due to error in parsing:", err.Error())
			spew.Fdump(os.Stderr, record)
			continue
		}
		output = append(output, parsed)
		i++
	}
	return output, nil
}
func processInvalidManifest ( manifest io . Reader ) ( [ ] [ ] string , error ) { output := [ ] [ ] string { } header := [ ] string { } dateField := - 1 i := 0 r := bufio . NewReader ( manifest ) for { line , err := r . ReadString ( '\n' ) if err == io . EOF && len ( strings . TrimSpace ( line ) ) == 0 { break } else if err == io . EOF { } else if err != nil { return nil , err } if i == 0 { record := strings . Split ( strings . TrimSpace ( strings . Replace ( line , " " , " " , - 1 ) ) , " " ) header = append ( header , record ... ) for i , col := range record { if col == " " { dateField = i break } } if dateField == - 1 { return nil , fmt . Errorf ( " " ) } output = append ( output , header ) i ++ continue } record := strings . Split ( strings . TrimSpace ( line ) , " " ) if len ( record ) < len ( header ) { log . Println ( " " ) spew . Fdump ( os . Stderr , record ) continue } parsed , err := invalidLayout1 ( record , len ( header ) , dateField ) if err != nil { log . Println ( " " , err . Error ( ) ) spew . Fdump ( os . Stderr , record ) continue } output = append ( output , parsed ) i ++ } return output , nil }
Many of the manifest.csv files are invalid.
Many of the manifest . csv files are invalid .
// Read over each line of the invalid manifest // Can't always break if you see an EOF. Specifically, in some // files, the final line has no valid ending (neither an EOF nor a // newline). In that case, since the entire line still remains // despite receiving an EOF signal, so you want to process that. // When you then loop around a final time, there will be nothing in // the line, just EOF, so it's safe (and necessary) to break. // We hit the end of the file, but we had non-empty data in the // line. See if we need to process it. But, after this, we're done. // drainBadBuffer = true // Split into columns, assuming commas are the delimiter // Process non-header records // For now, if there are fewer entries in the field than in the header, // we assume it's wrong (Could consider an invalidLayoutX that tries to // handle this) // If the invalidLayout1 (basically, assuming a comma in the date field) // fails, then we will emit that to stderr and skip the bad record, but // continue otherwise.