How do I read in a large flat file

Question

I have a flat file that has 339276 line of text in it for a size of 62.1 MB. I am attempting to read in all the lines, parse them based on some conditions I have and then insert

Answer 1

It's not clear that it's necessary to read in all the lines before parsing them and inserting them into a database. Try to avoid that.

You have a small file: "a flat file that has 339276 line of text in it for a size of 62.1 MB." For example,

package main

import (
    "bytes"
    "fmt"
    "io"
    "io/ioutil"
)

func readLines(filename string) ([]string, error) {
    var lines []string
    file, err := ioutil.ReadFile(filename)
    if err != nil {
        return lines, err
    }
    buf := bytes.NewBuffer(file)
    for {
        line, err := buf.ReadString('\n')
        if len(line) == 0 {
            if err != nil {
                if err == io.EOF {
                    break
                }
                return lines, err
            }
        }
        lines = append(lines, line)
        if err != nil && err != io.EOF {
            return lines, err
        }
    }
    return lines, nil
}

func main() {
    // a flat file that has 339276 lines of text in it for a size of 62.1 MB
    filename := "flat.file"
    lines, err := readLines(filename)
    fmt.Println(len(lines))
    if err != nil {
        fmt.Println(err)
        return
    }
}