Skip to content

Commit 401bb52

Browse files
Maxim-MazurokMaxim-Mazurok
committed
no leaks and turbo speeds!
1 parent 7521e1c commit 401bb52

10 files changed

+170
-99
lines changed

‎README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ Updated replacement for [vader-sentiment](https://www.npmjs.com/package/vader-se
66

77
[vader-sentiment](https://www.npmjs.com/package/vader-sentiment) is outdated and doesn't support emoji - vital element of comments nowadays.
88

9+
Also, spawning python process from nodejs is slow. This module can analyze ~60,000 strings per second by re-using the same python instance, and doesn't leak memory.
10+
911
## Getting Started for dev
1012

1113
1. Run `nvm i`

‎binding.cc

Lines changed: 72 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -8,112 +8,87 @@ const char *executableFolder() {
88
return folderPath.c_str();
99
}
1010

11-
double MainFunc(const char *arg) {
12-
PyObject *pName = NULL, *importlib = NULL, *importlib__import_module = NULL;
13-
PyObject *vaderSentiment = NULL, *pFunc = NULL, *pArgs = NULL, *pArgs2 = NULL;
14-
PyObject *pModule2 = NULL, *importlib__import_module__args = NULL, *analyser = NULL;
15-
PyObject *result = NULL;
16-
PyObject *pModule2_path = NULL, *pModule2_path_insert = NULL;
17-
18-
double score = 1.0;
19-
20-
Py_Initialize();
21-
22-
pName = PyUnicode_DecodeFSDefault("importlib");
23-
if (!pName) goto error;
24-
25-
importlib = PyImport_Import(pName);
26-
Py_DECREF(pName);
27-
if (!importlib) goto error;
28-
29-
pModule2 = PyImport_Import(PyUnicode_DecodeFSDefault("sys"));
30-
if (!pModule2) goto error;
31-
32-
pArgs2 = PyTuple_New(2);
33-
if (!pArgs2) goto error;
34-
35-
PyTuple_SET_ITEM(pArgs2, 0, PyLong_FromLong(0));
36-
PyTuple_SET_ITEM(pArgs2, 1, PyUnicode_DecodeFSDefault(executableFolder()));
37-
38-
pModule2_path = PyObject_GetAttrString(pModule2, "path");
39-
if (!pModule2_path) goto error;
40-
41-
pModule2_path_insert = PyObject_GetAttrString(pModule2_path, "insert");
42-
if (!pModule2_path_insert) goto error;
43-
44-
if (PyObject_CallObject(pModule2_path_insert, pArgs2) == NULL) {
45-
goto error;
11+
// Define a constant for the default/error value
12+
const double DEFAULT_ERROR_VALUE = 0.0;
13+
14+
// Global variables to hold Python objects and initialization state
15+
static PyObject* analyser = NULL;
16+
static bool isPythonInitialized = false;
17+
18+
void InitializePython() {
19+
if (!isPythonInitialized) {
20+
Py_Initialize();
21+
22+
PyObject *pName = PyUnicode_DecodeFSDefault("importlib");
23+
PyObject *importlib = PyImport_Import(pName);
24+
Py_DECREF(pName);
25+
if (!importlib) {
26+
PyErr_Print();
27+
fprintf(stderr, "Failed to load \"importlib\" module\n");
28+
return;
29+
}
30+
31+
PyObject *pModule2 = PyImport_Import(PyUnicode_DecodeFSDefault("sys"));
32+
PyObject *pArgs2 = PyTuple_New(2);
33+
PyTuple_SET_ITEM(pArgs2, 0, PyLong_FromLong(0));
34+
PyTuple_SET_ITEM(pArgs2, 1, PyUnicode_DecodeFSDefault(executableFolder()));
35+
36+
PyObject *pModule2_path = PyObject_GetAttrString(pModule2, "path");
37+
PyObject *pModule2_path_insert = PyObject_GetAttrString(pModule2_path, "insert");
38+
PyObject *insertResult = PyObject_CallObject(pModule2_path_insert, pArgs2);
39+
Py_DECREF(insertResult);
40+
Py_DECREF(pArgs2);
41+
Py_DECREF(pModule2_path_insert);
42+
Py_DECREF(pModule2_path);
43+
Py_DECREF(pModule2);
44+
45+
PyObject *importlib__import_module = PyObject_GetAttrString(importlib, "import_module");
46+
PyObject *importlib__import_module__args = PyTuple_New(1);
47+
PyTuple_SET_ITEM(importlib__import_module__args, 0, PyUnicode_DecodeFSDefault("vaderSentiment-master.vaderSentiment.vaderSentiment"));
48+
49+
PyObject *vaderSentiment = PyObject_CallObject(importlib__import_module, importlib__import_module__args);
50+
Py_DECREF(importlib__import_module__args);
51+
Py_DECREF(importlib__import_module);
52+
Py_DECREF(importlib);
53+
54+
if (vaderSentiment) {
55+
PyObject *pFunc = PyObject_GetAttrString(vaderSentiment, "SentimentIntensityAnalyzer");
56+
if (pFunc && PyCallable_Check(pFunc)) {
57+
PyObject *pArgs = PyTuple_New(0);
58+
analyser = PyObject_CallObject(pFunc, pArgs);
59+
Py_DECREF(pArgs);
60+
}
61+
Py_XDECREF(pFunc);
62+
Py_DECREF(vaderSentiment);
63+
}
64+
65+
if (!analyser) {
66+
PyErr_Print();
67+
fprintf(stderr, "Failed to create SentimentIntensityAnalyzer\n");
68+
}
69+
70+
isPythonInitialized = true;
4671
}
72+
}
4773

48-
importlib__import_module = PyObject_GetAttrString(importlib, "import_module");
49-
if (!importlib__import_module) goto error;
50-
51-
importlib__import_module__args = PyTuple_New(1);
52-
if (!importlib__import_module__args) goto error;
53-
54-
PyTuple_SET_ITEM(importlib__import_module__args, 0,
55-
PyUnicode_DecodeFSDefault("vaderSentiment-master.vaderSentiment.vaderSentiment"));
56-
vaderSentiment = PyObject_CallObject(importlib__import_module, importlib__import_module__args);
57-
if (!vaderSentiment) goto error;
58-
59-
pFunc = PyObject_GetAttrString(vaderSentiment, "SentimentIntensityAnalyzer");
60-
if (!pFunc || !PyCallable_Check(pFunc)) {
61-
if (PyErr_Occurred()) PyErr_Print();
62-
fprintf(stderr, "Cannot find function \"%s\"\n", "SentimentIntensityAnalyzer");
63-
goto error;
74+
double MainFunc(const char *arg) {
75+
if (!isPythonInitialized) {
76+
InitializePython();
6477
}
6578

66-
pArgs = PyTuple_New(0);
67-
if (!pArgs) goto error;
68-
69-
analyser = PyObject_CallObject(pFunc, pArgs);
70-
Py_DECREF(pArgs);
71-
if (!analyser) goto error;
79+
if (!analyser) {
80+
return DEFAULT_ERROR_VALUE;
81+
}
7282

73-
result = PyObject_CallMethod(analyser, "polarity_scores", "(s)", arg);
83+
PyObject *result = PyObject_CallMethod(analyser, "polarity_scores", "(s)", arg);
7484
if (!result) {
7585
PyErr_Print();
76-
fprintf(stderr, "Call failed\n");
77-
goto error;
86+
fprintf(stderr, "Call to polarity_scores failed\n");
87+
return DEFAULT_ERROR_VALUE;
7888
}
7989

80-
score = PyFloat_AsDouble(PyDict_GetItemString(result, "compound"));
81-
90+
double score = PyFloat_AsDouble(PyDict_GetItemString(result, "compound"));
8291
Py_DECREF(result);
83-
Py_DECREF(analyser);
84-
Py_DECREF(importlib__import_module__args);
85-
Py_DECREF(pFunc);
86-
Py_DECREF(vaderSentiment);
87-
Py_DECREF(importlib__import_module);
88-
Py_DECREF(importlib);
89-
Py_DECREF(pModule2);
90-
Py_DECREF(pModule2_path);
91-
Py_DECREF(pModule2_path_insert);
92-
Py_DECREF(pArgs2);
93-
94-
if (Py_FinalizeEx() < 0) {
95-
return 120;
96-
}
97-
98-
return score;
99-
100-
error:
101-
Py_XDECREF(result);
102-
Py_XDECREF(analyser);
103-
Py_XDECREF(importlib__import_module__args);
104-
Py_XDECREF(pFunc);
105-
Py_XDECREF(vaderSentiment);
106-
Py_XDECREF(importlib__import_module);
107-
Py_XDECREF(importlib);
108-
Py_XDECREF(pModule2);
109-
Py_XDECREF(pModule2_path);
110-
Py_XDECREF(pModule2_path_insert);
111-
Py_XDECREF(pArgs2);
112-
113-
if (Py_FinalizeEx() < 0) {
114-
return 120;
115-
}
116-
11792
return score;
11893
}
11994

@@ -138,4 +113,4 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) {
138113
return exports;
139114
}
140115

141-
NODE_API_MODULE(hello, Init);
116+
NODE_API_MODULE(NODE_GYP_MODULE_NAME, Init);

‎prebuilds/linux-x64/node.napi.node

-3.58 MB
Binary file not shown.

‎prebuilds/linux-x64/vader-sentiment-cpython.node

0 Bytes
Binary file not shown.

‎test-fork.js

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
const { fork } = require("child_process");
2+
const text = process.argv[2] || "Woohooo 😍 ✌️";
3+
let i = 0;
4+
5+
function runService(workerData) {
6+
return new Promise((resolve, reject) => {
7+
const worker = fork("./worker-fork.js", [workerData]);
8+
9+
worker.on("message", resolve);
10+
worker.on("error", reject);
11+
worker.on("exit", (code) => {
12+
if (code !== 0) {
13+
reject(new Error(`Worker stopped with exit code ${code}`));
14+
}
15+
});
16+
});
17+
}
18+
19+
const startDateTime = new Date();
20+
21+
const next = async () => {
22+
try {
23+
console.log(`calling with "${text}"`);
24+
const result = await runService(text);
25+
i++;
26+
const iterationsPerSecond = i / ((new Date() - startDateTime) / 1000);
27+
console.log({ result, i, iterationsPerSecond });
28+
} catch (err) {
29+
console.error(err);
30+
}
31+
// global.gc();
32+
next();
33+
};
34+
35+
next();

‎test-worker.js

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
const { Worker } = require("worker_threads");
2+
const text = process.argv[2] || "Woohooo 😍 ✌️";
3+
let i = 0;
4+
5+
function runService(workerData) {
6+
return new Promise((resolve, reject) => {
7+
const worker = new Worker("./worker-worker.js", { workerData });
8+
worker.on("message", resolve);
9+
worker.on("error", reject);
10+
worker.on("exit", (code) => {
11+
if (code !== 0) {
12+
reject(new Error(`Worker stopped with exit code ${code}`));
13+
}
14+
});
15+
});
16+
}
17+
18+
const startDateTime = new Date();
19+
20+
const next = async () => {
21+
try {
22+
console.log(`calling with "${text}"`);
23+
const result = await runService(text);
24+
i++;
25+
const iterationsPerSecond = i / ((new Date() - startDateTime) / 1000);
26+
console.log({ result, i, iterationsPerSecond });
27+
} catch (err) {
28+
console.error(err);
29+
}
30+
// global.gc();
31+
next();
32+
};
33+
34+
next();

‎test.js

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,14 @@
11
const vader = require("./index");
22
const text = process.argv[2] || "Woohooo 😍 ✌️";
3-
console.log(`calling with "${text}"`);
4-
console.log({ result: vader(text) });
3+
4+
let i = 0;
5+
const startDateTime = new Date();
6+
7+
while (true) {
8+
i++;
9+
const result = vader(text);
10+
if (i % 10_000 === 0) {
11+
const iterationsPerSecond = i / ((new Date() - startDateTime) / 1000);
12+
console.log({ result, i, iterationsPerSecond });
13+
}
14+
}

‎worker-fork.js

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
const vader = require("./index");
2+
3+
const workerData = process.argv[2];
4+
const result = vader(workerData);
5+
6+
process.send(result);
7+
process.exit(0);

‎worker-worker.js

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
const { parentPort, workerData } = require("worker_threads");
2+
const vader = require("./index");
3+
4+
parentPort.postMessage(vader(workerData));

‎worker.js

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
const { parentPort, workerData } = require("worker_threads");
2+
const vader = require("./index");
3+
4+
parentPort.postMessage(vader(workerData));

0 commit comments

Comments
 (0)