Is it required to clear Buffer in Node.js? If yes, How? If No, what is the issue here?

问题

I am making a thumbnail generator, and I am using Gearman to do the job in the background as soon as a photo is uploaded. I have a worker written in Node.js which basically does the thumbnail processing(using imagemagick for node) and inserting it into the database. The Client side is written in PHP, where it basically send the Photo_id and the base64 encoded string of the original image. The Node.js worker takes the input base64 encoded image, and processes it. Everything is working fine. The issue that I found out was that, when I call the Node.js worker via. PHP the first time, all 3 thumbnails(I am generating 3 thumbnails for every image) are generated, on the second time and third time as well also all 3 are generated, but on the fourth time only one thumbnail is generated, and after than the no matter how many times I call the worker, no thumbnail is getting generated.

What I think the issue might be is the Buffer getting filled, as I am using Node.js Buffer to convert base64 string to Binary and vice versa.

I searched for ways to clear the buffer, like pointing the buffer variable to null, using js delete function on the buffer variable. But, none of this seemed to help. Either the buffer is not getting cleared or the issue is not with the buffer at all.

Please go through the code, and let me know what the issue might be.

PHP Gearman Client

<?php
$client= new GearmanClient();
$client->addServer('127.0.0.1',4731);
print $client->do("infinite", "[\"12246\", \"Base 64 image string in here\"]");
?>

Node.js Gearman Client

var fs = require('fs');
var db = require('./class.photo.js');
var im = require('imagemagick');
var Gearman = require('node-gearman/lib/gearman.js');
var gearman = new Gearman("127.0.0.1", 4731);
var db = new db();

function initializePic(id,base, req_type, callback) {
    console.log("Initializing thumbnail resize on " + id);

        var rawbase = base.split(",")[1];
        rawbase = rawbase.replace("\n", "");
        var buff = new Buffer(rawbase, 'base64');
        console.log(buff.length);
        var bin = buff.toString('binary');
        buff = null;

        console.time("image_" + req_type);
        imageResize(id, bin, properties[req_type].img_dest_width, properties[req_type].img_dest_height, imageRequestEnum[req_type], function (res) {
            if (res) {
                console.timeEnd(req_type);
                console.timeEnd("image_" + req_type);
            }
        });
}

function imageResize(id, bin, w, h, s, callback) {
    var fileName = id + '_' + w + '_' + h + '_thumb.jpg'
    console.log("Initializing resize process with filename " + fileName);
    im.resize({
        srcData: bin,
//        dstPath: fileName,
        width: w,
        height: h + '\!',
        quality: 0.7,
        strip: false,
        progressive: true,
    }, function (err, stdout, stderr) {
        if (err)
            throw err;
        if (stderr)
            throw stderr;

        var buff = new Buffer(stdout, "binary");
        console.log(buff.length);
        var b64 = buff.toString('base64');
        buff = null;

        if (b64) {
            b64 = "data:image/jpeg;base64," + b64;
            db.insertThumb(id, b64, s, function (res) {
                if (res) {
                    callback(true);
                }
            });
        }
    });
}


gearman.connect();

gearman.registerWorker("infinite", function (payload, worker) {
    var payload = payload.toString();
    var json = JSON.parse(payload);
    var id = json[0]; //getting the photo_id from the payload
    var base = json[1]; //Getting the base64 image data from the payload

    worker.end();
    console.time("user_image_small");
    initializePic(id,base, "user_image_small");
    console.time("user_image_medium");
    initializePic(id,base, "user_image_medium");
    console.time("profile_photo");
    initializePic(id,base, "profile_photo");
});

Node.js Console response

Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_50_50_thumb.jpg
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_90_90_thumb.jpg
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_245_245_thumb.jpg
profile_photo: 151ms
image_profile_photo: 150ms
user_image_medium: 211ms
image_user_image_medium: 210ms
user_image_small: 218ms
image_user_image_small: 217ms
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_50_50_thumb.jpg
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_90_90_thumb.jpg
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_245_245_thumb.jpg
user_image_small: 98ms
image_user_image_small: 96ms
profile_photo: 142ms
image_profile_photo: 141ms
user_image_medium: 147ms
image_user_image_medium: 145ms
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_50_50_thumb.jpg
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_90_90_thumb.jpg
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_245_245_thumb.jpg
user_image_small: 90ms
image_user_image_small: 89ms
user_image_medium: 141ms
image_user_image_medium: 140ms
profile_photo: 138ms
image_profile_photo: 137ms
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_50_50_thumb.jpg
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_90_90_thumb.jpg
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_245_245_thumb.jpg
user_image_small: 94ms
image_user_image_small: 93ms
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_50_50_thumb.jpg
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_90_90_thumb.jpg
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_245_245_thumb.jpg
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_50_50_thumb.jpg
Initializing thumbnail resize on 12246
Initializing resize process with filename 12246_90_90_thumb.jpg
Initializing thumbnail resize on 12246

Notice in the last few tries, its not actually creating any thumbnail. Displaying the time taken basically means the thumbnail creation process is complete.

I am not sure what the issue might be. Looking for some help.

回答1:

After much troubleshooting, found out that the issue was with the node-mysql package. Using the mysql package for node.js solved the issue. Check my other question and answer for more details.

回答2:

In my opinion, the problem is that you are closing worker before the actual completion of the work.

Something like

gearman.registerWorker("infinite", function (payload, worker) {
    var payload = payload.toString();
    var json = JSON.parse(payload);
    var id = json[0]; //getting the photo_id from the payload
    var base = json[1]; //Getting the base64 image data from the payload

    // not actually work with your code right now, but you got the idea
    Promise.all([        
        initializePic(id, base, "user_image_small"),
        initializePic(id, base, "user_image_medium"),
        initializePic(id, base, "profile_photo")
    ]).then(() => {
        // should be called after all processing is done
        worker.end();
    });
});

来源：https://stackoverflow.com/questions/36258653/is-it-required-to-clear-buffer-in-node-js-if-yes-how-if-no-what-is-the-issue

标签

javascript

php

node.js

ImageMagick

gearman