Reading contents of csv file in node.js

Question

I am trying to implement a module in nodejs(just started working in nodejs) which has requirement below as

1. Upload .csv file.
2. Read content of the

Answer 1

There is an awesome node project which helped me a lot. You should check it out What we're going to use is their csv-parse module. It is able to get a stream as input and read it line by line without blocking the event loop, so basically while you are processing a file your server won't be stuck and other requests can still be processed normally.

Since you said you are just starting with nodejs, you should make a quick search and understand how midlewares work in the request handling process. As a simplification for request handling,a middleware is a function(req, res, next). With req you get request data. With res you can send the response, and next you send your req and res objects to the next middleware. This way you can process a request in parts and the last middleware of the flow will send response to the client (res.send(200) for example)

the Multer({...}) call returns a middleware function. When a request gets to this Middleware, multer will try to download any files the user send in the post request. When u say app.use(Multer({...})), you are asking multer to try and download files from ANY post requests that contains files. This is a security risk if not all your routes are expecting files to be uploaded.

Ok, that being said, here's a sample code I wrote to handle your use case:

//Important Security advice: 
//don't add multer as a middleware to all requests. 
//If you do this, people will be able to upload files
//in ALL YOUR 'post' handlers!!! 

var Multer = require('multer');
var Parse = require('csv-parse');
var fs = require('fs')

function parseCSVFile(sourceFilePath, columns, onNewRecord, handleError, done){
    var source = fs.createReadStream(sourceFilePath);

    var linesRead = 0;

    var parser = Parse({
        delimiter: ',', 
        columns:columns
    });

    parser.on("readable", function(){
        var record;
        while (record = parser.read()) {
            linesRead++;
            onNewRecord(record);
        }
    });

    parser.on("error", function(error){
        handleError(error)
    });

    parser.on("end", function(){
        done(linesRead);
    });

    source.pipe(parser);
}

//We will call this once Multer's middleware processed the request
//and stored file in req.files.fileFormFieldName

function parseFile(req, res, next){
    var filePath = req.files.file.path;
    console.log(filePath);
    function onNewRecord(record){
        console.log(record)
    }

    function onError(error){
        console.log(error)
    }

    function done(linesRead){
        res.send(200, linesRead)
    }

    var columns = true; 
    parseCSVFile(filePath, columns, onNewRecord, onError, done);

}

//this is the route handler with two middlewares. 
//First:  Multer middleware to download file. At some point,
//this middleware calls next() so process continues on to next middleware
//Second: use the file as you need

app.post('/upload', [Multer({dest:'./uploads'}), parseFile]);

I hope this helped. Make sure to understand how routes middlewares work in node: they are a key to good quality code.

Marcel