Streaming an uploaded file to an HTTP request

Question

My goal is to accept an uploaded file and stream it to Wistia using the the Wistia Upload API. I need to be able to add fields to the HTTP request, and I don't want the file to touch the disk. I'm using Node, Express, Request, and Busboy.

The code below has two console.log statements. The first returns [Error: not implemented] and the second returns [Error: form-data: not implemented]. I'm new to streaming in Node, so I'm probably doing something fundamentally wrong. Any help would be much appreciated.

app.use("/upload", function(req, res, next) {
    var writeStream = new stream.Writable();
    writeStream.on("error", function(error) {
        console.log(error);
    });
    var busboy = new Busboy({headers: req.headers});
    busboy.on("file", function(fieldname, file, filename, encoding, mimetype) {
        file.on("data", function(data) {
            writeStream.write(data);
        });
        file.on("end", function() {
            request.post({
                url: "https://upload.wistia.com",
                formData: {
                    api_password: "abc123",
                    file: new stream.Readable(writeStream)
                }
            }, function(error, response, body) {
                console.log(error);
            });
        });
    });
    req.pipe(busboy);
});

Answer 1

I am not to familiar with the busboy module, but there errors you are getting are from attempting to use un-implemented streams. Whenever you create a new readable or writable stream directly from the stream module you have to create the _read and _write methods respectively Stream Implementors (node.js api). To give you something to work with the following example is using multer for handling multipart requests, I think you'll find multer is easier to use than busboy.

var app = require('express')();
var fs = require('fs');
var request = require('request');
app.use(multer());

app.post("/upload", function(req, res, next) {
    // create a read stream
    var readable = fs.createReadStream(req.files.myfile.path);
    request.post({
        url: "https://upload.wistia.com",
        formData: {
            api_password: "abc123",
            file: readable
        }    
    }, function(err, res, body) {
        // send something to client
    })
});

I hope this helps unfortunately I am not familiar with busboy, but this should work with multer, and as I said before there problem is just that you are using un-implemented streams I'm sure there is a way to configure this operation with busboy if you wanted.

Answer 2

If you want to use multipart (another npm) here is a tutorial: http://qnimate.com/stream-file-uploads-to-storage-server-in-node-js/