How to pipe multiple readable streams, from multiple api requests, to a single writeable stream?

Question

- Desired Behaviour
- Actual Behaviour
- What I\'ve Tried
- Steps To Reproduce
- Research

---

Answer 1

Here are two solutions.

Solution 01

• uses Bluebird.mapSeries
• writes individual responses to temporary files
• puts them in a zip file (using archiver)
• sends zip file back to client to save
• deletes temporary files

It utilises Bluebird.mapSeries from BM's answer but instead of just mapping over the responses, requests and responses are handled within the map function. Also, it resolves promises on the writeable stream finish event, rather than the readable stream end event. Bluebird is helpful in that it pauses iteration within a map function until a response has been received and handled, and then moves on to the next iteration.

Given that the Bluebird map function produces clean audio files, rather than zipping the files, you could use a solution like in Terry Lennox's answer to combine multiple audio files into one audio file. My first attempt of that solution, using Bluebird and fluent-ffmpeg, produced a single file, but it was slightly lower quality - no doubt this could be tweaked in ffmpeg settings, but i didn't have time to do that.

// route handler
app.route("/api/:api_version/tts")
    .get(api_tts_get);

// route handler middleware
const api_tts_get = async (req, res) => {

    var query_parameters = req.query;

    var file_name = query_parameters.file_name;
    var text_string_array = text_string_array; // eg: https://pastebin.com/raw/JkK8ehwV

    var absolute_path = path.join(__dirname, "/src/temp_audio/", file_name);
    var relative_path = path.join("./src/temp_audio/", file_name); // path relative to server root

    // set up archiver
    var archive = archiver('zip', {
        zlib: { level: 9 } // sets the compression level  
    });
    var zip_write_stream = fs.createWriteStream(`${relative_path}.zip`);
    archive.pipe(zip_write_stream);

    await Bluebird.mapSeries(text_chunk_array, async function(text_chunk, index) {

        // check if last value of array  
        const isLastIndex = index === text_chunk_array.length - 1;

        return new Promise((resolve, reject) => {

            var textToSpeech = new TextToSpeechV1({
                iam_apikey: iam_apikey,
                url: tts_service_url
            });

            var synthesizeParams = {
                text: text_chunk,
                accept: 'audio/mp3',
                voice: 'en-US_AllisonV3Voice'
            };

            textToSpeech.synthesize(synthesizeParams, (err, audio) => {
                if (err) {
                    console.log("synthesize - an error occurred: ");
                    return reject(err);
                }

                // write individual files to disk  
                var file_name = `${relative_path}_${index}.mp3`;
                var write_stream = fs.createWriteStream(`${file_name}`);
                audio.pipe(write_stream);

                // on finish event of individual file write  
                write_stream.on('finish', function() {

                    // add file to archive  
                    archive.file(file_name, { name: `audio_${index}.mp3` });

                    // if not the last value of the array
                    if (isLastIndex === false) {
                        resolve();
                    } 
                    // if the last value of the array 
                    else if (isLastIndex === true) {
                        resolve();

                        // when zip file has finished writing,
                        // send it back to client, and delete temp files from server 
                        zip_write_stream.on('close', function() {

                            // download the zip file (using absolute_path)  
                            res.download(`${absolute_path}.zip`, (err) => {
                                if (err) {
                                    console.log(err);
                                }

                                // delete each audio file (using relative_path) 
                                for (let i = 0; i < text_chunk_array.length; i++) {
                                    fs.unlink(`${relative_path}_${i}.mp3`, (err) => {
                                        if (err) {
                                            console.log(err);
                                        }
                                        console.log(`AUDIO FILE ${i} REMOVED!`);
                                    });
                                }

                                // delete the zip file
                                fs.unlink(`${relative_path}.zip`, (err) => {
                                    if (err) {
                                        console.log(err);
                                    }
                                    console.log(`ZIP FILE REMOVED!`);
                                });

                            });


                        });

                        // from archiver readme examples  
                        archive.on('warning', function(err) {
                            if (err.code === 'ENOENT') {
                                // log warning
                            } else {
                                // throw error
                                throw err;
                            }
                        });

                        // from archiver readme examples  
                        archive.on('error', function(err) {
                            throw err;
                        });

                        // from archiver readme examples 
                        archive.finalize();
                    }
                });
            });

        });

    });

}

Solution 02

I was keen to find a solution that didn't use a library to "pause" within the map() iteration, so I:

• swapped the map() function for a for of loop
• used await before the api call, rather than wrapping it in a promise, and
• instead of using return new Promise() to contain the response handling, I used await new Promise() (gleaned from this answer)

This last change, magically, paused the loop until the archive.file() and audio.pipe(writestream) operations were completed - i'd like to better understand how that works.

// route handler
app.route("/api/:api_version/tts")
    .get(api_tts_get);

// route handler middleware
const api_tts_get = async (req, res) => {

    var query_parameters = req.query;

    var file_name = query_parameters.file_name;
    var text_string_array = text_string_array; // eg: https://pastebin.com/raw/JkK8ehwV

    var absolute_path = path.join(__dirname, "/src/temp_audio/", file_name);
    var relative_path = path.join("./src/temp_audio/", file_name); // path relative to server root

    // set up archiver
    var archive = archiver('zip', {
        zlib: { level: 9 } // sets the compression level  
    });
    var zip_write_stream = fs.createWriteStream(`${relative_path}.zip`);
    archive.pipe(zip_write_stream);

    for (const [index, text_chunk] of text_chunk_array.entries()) {

        // check if last value of array 
        const isLastIndex = index === text_chunk_array.length - 1;

        var textToSpeech = new TextToSpeechV1({
            iam_apikey: iam_apikey,
            url: tts_service_url
        });

        var synthesizeParams = {
            text: text_chunk,
            accept: 'audio/mp3',
            voice: 'en-US_AllisonV3Voice'
        };

        try {

            var audio_readable_stream = await textToSpeech.synthesize(synthesizeParams);

            await new Promise(function(resolve, reject) {

                // write individual files to disk 
                var file_name = `${relative_path}_${index}.mp3`;
                var write_stream = fs.createWriteStream(`${file_name}`);
                audio_readable_stream.pipe(write_stream);

                // on finish event of individual file write
                write_stream.on('finish', function() {

                    // add file to archive
                    archive.file(file_name, { name: `audio_${index}.mp3` });

                    // if not the last value of the array
                    if (isLastIndex === false) {
                        resolve();
                    } 
                    // if the last value of the array 
                    else if (isLastIndex === true) {
                        resolve();

                        // when zip file has finished writing,
                        // send it back to client, and delete temp files from server
                        zip_write_stream.on('close', function() {

                            // download the zip file (using absolute_path)  
                            res.download(`${absolute_path}.zip`, (err) => {
                                if (err) {
                                    console.log(err);
                                }

                                // delete each audio file (using relative_path)
                                for (let i = 0; i < text_chunk_array.length; i++) {
                                    fs.unlink(`${relative_path}_${i}.mp3`, (err) => {
                                        if (err) {
                                            console.log(err);
                                        }
                                        console.log(`AUDIO FILE ${i} REMOVED!`);
                                    });
                                }

                                // delete the zip file
                                fs.unlink(`${relative_path}.zip`, (err) => {
                                    if (err) {
                                        console.log(err);
                                    }
                                    console.log(`ZIP FILE REMOVED!`);
                                });

                            });


                        });

                        // from archiver readme examples  
                        archive.on('warning', function(err) {
                            if (err.code === 'ENOENT') {
                                // log warning
                            } else {
                                // throw error
                                throw err;
                            }
                        });

                        // from archiver readme examples  
                        archive.on('error', function(err) {
                            throw err;
                        });

                        // from archiver readme examples   
                        archive.finalize();
                    }
                });

            });

        } catch (err) {
            console.log("oh dear, there was an error: ");
            console.log(err);
        }
    }

}

Learning Experiences

Other issues that came up during this process are documented below:

Long requests time out when using node (and resend the request)...

// solution  
req.connection.setTimeout( 1000 * 60 * 10 ); // ten minutes

See: https://github.com/expressjs/express/issues/2512

---

400 errors caused by node max header size of 8KB (query string is included in header size)...

// solution (although probably not recommended - better to get text_string_array from server, rather than client) 
node --max-http-header-size 80000 app.js

See: https://github.com/nodejs/node/issues/24692