How to call a Python function from Node.js

Question

I have an Express Node.js application, but I also have a machine learning algorithm to use in Python. Is there a way I can call Python functions from my Node.js application

Answer 1

Most of previous answers call the success of the promise in the on("data"), it is not the proper way to do it because if you receive a lot of data you will only get the first part. Instead you have to do it on the end event.

const { spawn } = require('child_process');
const pythonDir = (__dirname + "/../pythonCode/"); // Path of python script folder
const python = pythonDir + "pythonEnv/bin/python"; // Path of the Python interpreter

/** remove warning that you don't care about */
function cleanWarning(error) {
    return error.replace(/Detector is not able to detect the language reliably.\n/g,"");
}

function callPython(scriptName, args) {
    return new Promise(function(success, reject) {
        const script = pythonDir + scriptName;
        const pyArgs = [script, JSON.stringify(args) ]
        const pyprog = spawn(python, pyArgs );
        let result = "";
        let resultError = "";
        pyprog.stdout.on('data', function(data) {
            result += data.toString();
        });

        pyprog.stderr.on('data', (data) => {
            resultError += cleanWarning(data.toString());
        });

        pyprog.stdout.on("end", function(){
            if(resultError == "") {
                success(JSON.parse(result));
            }else{
                console.error(`Python error, you can reproduce the error with: \n${python} ${script} ${pyArgs.join(" ")}`);
                const error = new Error(resultError);
                console.error(error);
                reject(resultError);
            }
        })
   });
}
module.exports.callPython = callPython;

Call:

const pythonCaller = require("../core/pythonCaller");
const result = await pythonCaller.callPython("preprocessorSentiment.py", {"thekeyYouwant": value});

python:

try:
    argu = json.loads(sys.argv[1])
except:
    raise Exception("error while loading argument")

Answer 2

You can now use RPC libraries that support Python and Javascript such as zerorpc

From their front page:

Node.js Client

var zerorpc = require("zerorpc");

var client = new zerorpc.Client();
client.connect("tcp://127.0.0.1:4242");

client.invoke("hello", "RPC", function(error, res, more) {
    console.log(res);
});

Python Server

import zerorpc

class HelloRPC(object):
    def hello(self, name):
        return "Hello, %s" % name

s = zerorpc.Server(HelloRPC())
s.bind("tcp://0.0.0.0:4242")
s.run()

Answer 3

The Boa is good for your needs, see the example which extends Python tensorflow keras.Sequential class in JavaScript.

const fs = require('fs');
const boa = require('@pipcook/boa');
const { tuple, enumerate } = boa.builtins();

const tf = boa.import('tensorflow');
const tfds = boa.import('tensorflow_datasets');

const { keras } = tf;
const { layers } = keras;

const [
  [ train_data, test_data ],
  info
] = tfds.load('imdb_reviews/subwords8k', boa.kwargs({
  split: tuple([ tfds.Split.TRAIN, tfds.Split.TEST ]),
  with_info: true,
  as_supervised: true
}));

const encoder = info.features['text'].encoder;
const padded_shapes = tuple([
  [ null ], tuple([])
]);
const train_batches = train_data.shuffle(1000)
  .padded_batch(10, boa.kwargs({ padded_shapes }));
const test_batches = test_data.shuffle(1000)
  .padded_batch(10, boa.kwargs({ padded_shapes }));

const embedding_dim = 16;
const model = keras.Sequential([
  layers.Embedding(encoder.vocab_size, embedding_dim),
  layers.GlobalAveragePooling1D(),
  layers.Dense(16, boa.kwargs({ activation: 'relu' })),
  layers.Dense(1, boa.kwargs({ activation: 'sigmoid' }))
]);

model.summary();
model.compile(boa.kwargs({
  optimizer: 'adam',
  loss: 'binary_crossentropy',
  metrics: [ 'accuracy' ]
}));

The complete example is at: https://github.com/alibaba/pipcook/blob/master/example/boa/tf2/word-embedding.js

I used Boa in another project Pipcook, which is to address the machine learning problems for JavaScript developers, we implemented ML/DL models upon the Python ecosystem(tensorflow,keras,pytorch) by the boa library.

Answer 4

The python-shell module by extrabacon is a simple way to run Python scripts from Node.js with basic, but efficient inter-process communication and better error handling.

Installation: npm install python-shell.

Running a simple Python script:

var PythonShell = require('python-shell');

PythonShell.run('my_script.py', function (err) {
  if (err) throw err;
  console.log('finished');
});

Running a Python script with arguments and options:

var PythonShell = require('python-shell');

var options = {
  mode: 'text',
  pythonPath: 'path/to/python',
  pythonOptions: ['-u'],
  scriptPath: 'path/to/my/scripts',
  args: ['value1', 'value2', 'value3']
};

PythonShell.run('my_script.py', options, function (err, results) {
  if (err) 
    throw err;
  // Results is an array consisting of messages collected during execution
  console.log('results: %j', results);
});

For the full documentation and source code, check out https://github.com/extrabacon/python-shell

Answer 5

/*eslint-env es6*/
/*global require*/
/*global console*/
var express = require('express'); 
var app = express();

// Creates a server which runs on port 3000 and  
// can be accessed through localhost:3000
app.listen(3000, function() { 
    console.log('server running on port 3000'); 
} ) 

app.get('/name', function(req, res) {

    console.log('Running');

    // Use child_process.spawn method from  
    // child_process module and assign it 
    // to variable spawn 
    var spawn = require("child_process").spawn;   
    // Parameters passed in spawn - 
    // 1. type_of_script 
    // 2. list containing Path of the script 
    //    and arguments for the script  

    // E.g : http://localhost:3000/name?firstname=Levente
    var process = spawn('python',['apiTest.py', 
                        req.query.firstname]);

    // Takes stdout data from script which executed 
    // with arguments and send this data to res object
    var output = '';
    process.stdout.on('data', function(data) {

        console.log("Sending Info")
        res.end(data.toString('utf8'));
    });

    console.log(output);
}); 

This worked for me. Your python.exe must be added to you path variables for this code snippet. Also, make sure your python script is in your project folder.

Answer 6

Easiest way I know of is to use "child_process" package which comes packaged with node.

Then you can do something like:

const spawn = require("child_process").spawn;
const pythonProcess = spawn('python',["path/to/script.py", arg1, arg2, ...]);

Then all you have to do is make sure that you import sys in your python script, and then you can access arg1 using sys.argv[1], arg2 using sys.argv[2], and so on.

To send data back to node just do the following in the python script:

print(dataToSendBack)
sys.stdout.flush()

And then node can listen for data using:

pythonProcess.stdout.on('data', (data) => {
    // Do something with the data returned from python script
});

Since this allows multiple arguments to be passed to a script using spawn, you can restructure a python script so that one of the arguments decides which function to call, and the other argument gets passed to that function, etc.

Hope this was clear. Let me know if something needs clarification.