问题
I have multiple Bull Queues in my NodeJS
project which will run if previous queue is executed successfully.
I'm trying to verify some email addresses here.
Check the Email format (formatQueue)
Email Existence using
npm email-existence
package (existenceQueue)
The formatQueue
is less time taking process, which wil run the RegEx
and validate the Email
format. but The email-existence
package takes around 5-10 seconds to complete.
formatQueue
and existenceQueue
works properly if there are less jobs
like 20-100. but when I Add more than that around 1000 jobs at a time, existenceQueue
failes with below error
myemail@email.com job stalled more than allowable limit
I checked the issue HERE and HERE, I thought the process is taking too long to respond, so added limiter
as refered HERE. But that does not help me.
If a job in any of the queue fails, Its not processing the next job. It will stop there and the other jobs will stay in waiting
state.
My code is something similar to below code. please help me with the issue.
Queue.js
var formatQueue = new Queue('format', "redis-db-url");
var existenceQueue = new Queue('existence', "redis-db-url");
// ------------ function for adding to queue ------------
module.exports.addToQueue = (emails) => {
emails.forEach(element => {
formatQueue.add(element, { attempts: 3, backoff: 1000 });
});
}
// ------------ Queue Process -------------
// Format Test Process
formatQueue.process(function(job, done){
FormatTest.validate(job.data, (err, data) => {
if(err) done();
else{
job.data = data;
done();
}
});
});
// Existence Test Process
formatQueue.process(function(job, done){
ExistenceTest.validate(job.data, (err, data) => {
if(err) done();
else{
job.data = data;
done();
}
});
});
// ------------ On Cmplete Handlers ------------
formatQueue.on('completed', function(job){
if(job.data.is_well_format){
existenceQueue.add(job.data, { attempts: 3, backoff: 1000 });
}else QueueModel.lastStep(job.data)
});
existenceQueue.on('completed', function(job){
QueueModel.lastStep(job.data)
});
// ------------ To update the emaile ------------
module.exports.lastStep = (data) => {
Emails.updateEmail(data, (err, updated) => {
if(!err) {
formatQueue.clean('completed');
existenceQueue.clean('completed');
}
})
}
--------- Update ---------
The processor was taking too much time to respond so the job was getting stalled
or getting failed since i was using timeout.
I'm trying to run the process
in different processor
file itsef as its in bull
documentation, I've added the file as below.
// -------- Queue.js ----------
formatQueue.process(__dirname+"/processors/format-worker.js");
// On Cmplete Handler
formatQueue.on('completed', function(job, result){
console.log(result, "Format-Complete-job"); // result is undefined
if(job.data.is_well_format){
existenceQueue.add(job.data, { attempts: 3, backoff: 1000 });
}else QueueModel.lastStep(job.data)
});
// -------- Queue.js ends ---------
//format-worker.js
Validator = require("../../validators");
module.exports = (job) => {
Validator.Format.validate(job.data, (data) => {
job.data = data;
return Promise.resolve(data);
});
}
Now On Job complete which i was using before, I used to get job data with updated job parameters. Now I'm not getting updated job data. and the second parameter which is there in the documentation i.e result
is undefined
.
Now how can I get the updated job data in this case.
回答1:
Try repeatable jobs
var formatQueue = new Queue('format', "redis-db-url");
var existenceQueue = new Queue('existence', "redis-db-url");
// ------------ function for adding to queue ------------
module.exports.addToQueue = (emails) => {
emails.forEach(element => {
let jobOptions = {
repeat: {
every: 10 * 1000, // Run job every 10 seconds for example
limit: 3 // Maximum number of times a job can repeat.
},
jobId: someUniqueId, // important do not forget this
removeOnComplete: true, // removes job from queue on success (if required)
removeOnFail: true // removes job from queue on failure (if required)
}
formatQueue.add(element, jobOptions);
});
}
// ------------ Queue Process -------------
// Format Test Process
formatQueue.process(function (job, done) {
FormatTest.validate(job.data, (err, data) => {
if (err) {
// Done with error
done(true);
} else {
job.data = data;
// Done without any error
done(false);
}
});
});
// Existence Test Process
existenceQueue.process(function (job, done) {
ExistenceTest.validate(job.data, (err, data) => {
if (err) {
// Done with error
done(true);
} else {
job.data = data;
// Done without any error
done(false);
}
});
});
// ------------ On Complete Handlers ------------
formatQueue.on('completed', function (job) {
if (job.data.is_well_format) {
let jobOptions = {
repeat: {
every: 10 * 1000, // Run job every 10 seconds for example
limit: 3 // Maximum number of times a job can repeat.
},
jobId: someUniqueId, // important do not forget this
removeOnComplete: true, // removes job from queue on success (if required)
removeOnFail: true // removes job from queue on failure (if required)
}
existenceQueue.add(job.data, jobOptions);
} else QueueModel.lastStep(job.data)
});
existenceQueue.on('completed', function (job) {
QueueModel.lastStep(job.data)
});
// ------------ To update the email ------------
module.exports.lastStep = (data) => {
Emails.updateEmail(data, (err, updated) => {
if (!err) {
formatQueue.clean('completed');
existenceQueue.clean('completed');
}
})
}
来源:https://stackoverflow.com/questions/61565588/nodejs-bull-stop-the-queue-jobs-on-a-job-failed