Skip to content

Commit 82ced72

Browse files
glasstigerpuzpuzpuz
glasstiger
and
puzpuzpuz
authored
feat(nodejs): set copyBuffer to 'true' by default (#18)
* feat(nodejs): set copyBuffer to 'true' by default, add type checks for sender options * feat(nodejs): worker threads example * version bump * Update src/sender.js Co-authored-by: Andrei Pechkurov <37772591+puzpuzpuz@users.noreply.github.com> --------- Co-authored-by: Andrei Pechkurov <37772591+puzpuzpuz@users.noreply.github.com>
1 parent c88ddd1 commit 82ced72

14 files changed

+443
-62
lines changed

‎README.md

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,3 +124,77 @@ async function run(): Promise<number> {
124124

125125
run().then(value => console.log(value)).catch(err => console.log(err));
126126
```
127+
128+
### Worker threads example
129+
```javascript
130+
const { Sender } = require("@questdb/nodejs-client");
131+
const { Worker, isMainThread, parentPort, workerData } = require('worker_threads');
132+
133+
// fake venue
134+
// generates random prices for a ticker for max 5 seconds, then the feed closes
135+
function* venue(ticker) {
136+
let end = false;
137+
setTimeout(() => { end = true; }, rndInt(5000));
138+
while (!end) {
139+
yield {"ticker": ticker, "price": Math.random()};
140+
}
141+
}
142+
143+
// market data feed simulator
144+
// uses the fake venue to deliver price updates to the feed handler (onTick() callback)
145+
async function subscribe(ticker, onTick) {
146+
const feed = venue(workerData.ticker);
147+
let tick;
148+
while (tick = feed.next().value) {
149+
await onTick(tick);
150+
await sleep(rndInt(30));
151+
}
152+
}
153+
154+
async function run() {
155+
if (isMainThread) {
156+
const tickers = ["t1", "t2", "t3", "t4"];
157+
// main thread to start a worker thread for each ticker
158+
for (let ticker in tickers) {
159+
const worker = new Worker(__filename, { workerData: { ticker: ticker } })
160+
.on('error', (err) => { throw err; })
161+
.on('exit', () => { console.log(`${ticker} thread exiting...`); })
162+
.on('message', (msg) => { console.log("Ingested " + msg.count + " prices for ticker " + msg.ticker); });
163+
}
164+
} else {
165+
// it is important that each worker has a dedicated sender object
166+
// threads cannot share the sender because they would write into the same buffer
167+
const sender = new Sender({ bufferSize: 4096 });
168+
await sender.connect({ port: 9009, host: "localhost" });
169+
170+
// subscribe for the market data of the ticker assigned to the worker
171+
// ingest each price update into the database using the sender
172+
let count = 0;
173+
await subscribe(workerData.ticker, async (tick) => {
174+
sender
175+
.table("prices")
176+
.symbol("ticker", tick.ticker)
177+
.floatColumn("price", tick.price)
178+
.atNow();
179+
await sender.flush();
180+
count++;
181+
});
182+
183+
// let the main thread know how many prices were ingested
184+
parentPort.postMessage({"ticker": workerData.ticker, "count": count});
185+
186+
// close the connection to the database
187+
await sender.close();
188+
}
189+
}
190+
191+
function sleep(ms) {
192+
return new Promise(resolve => setTimeout(resolve, ms));
193+
}
194+
195+
function rndInt(limit) {
196+
return Math.floor((Math.random() * limit) + 1);
197+
}
198+
199+
run().catch((err) => console.log(err));
200+
```

‎docs/Sender.html

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -120,14 +120,18 @@ <h5>Parameters:</h5>
120120
Properties of the object:
121121
<ul>
122122
<li>bufferSize: <i>number</i> - Size of the buffer used by the sender to collect rows, provided in bytes. <br>
123-
Optional, defaults to 8192 bytes </li>
124-
<li>copyBuffer: <i>boolean</i> - If true a new buffer will be created for every flush() call and the data to be sent to the server will be copied into this new buffer. <br>
125-
Setting the flag could result in performance degradation, use this flag only if calls to the client cannot be serialised. <br>
126-
Optional, defaults to false </li>
123+
Optional, defaults to 8192 bytes. <br>
124+
If the value passed is not a number, the setting is ignored. </li>
125+
<li>copyBuffer: <i>boolean</i> - By default a new buffer is created for every flush() call, and the data to be sent to the server is copied into this new buffer.
126+
Setting the flag to <i>false</i> results in reusing the same buffer instance for each flush() call. Use this flag only if calls to the client are serialised. <br>
127+
Optional, defaults to <i>true</i>. <br>
128+
If the value passed is not a boolean, the setting is ignored. </li>
127129
<li>jwk: <i>{x: string, y: string, kid: string, kty: string, d: string, crv: string}</i> - JsonWebKey for authentication. <br>
128-
If not provided, client is not authenticated and server might reject the connection depending on configuration.</li>
130+
If not provided, client is not authenticated and server might reject the connection depending on configuration. <br>
131+
No type checks performed on the object passed. </li>
129132
<li>log: <i>(level: 'error'|'warn'|'info'|'debug', message: string) => void</i> - logging function. <br>
130-
If not provided, default logging is used which writes to the console with logging level 'info'.</li>
133+
If not provided, default logging is used which writes to the console with logging level <i>info</i>. <br>
134+
If not a function passed, the setting is ignored. </li>
131135
</ul>
132136
</p></td>
133137
</tr>
@@ -327,7 +331,7 @@ <h5>Parameters:</h5>
327331

328332
<dt class="tag-source">Source:</dt>
329333
<dd class="tag-source"><ul class="dummy"><li>
330-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line356">line 356</a>
334+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line361">line 361</a>
331335
</li></ul></dd>
332336

333337

@@ -416,7 +420,7 @@ <h4 class="name" id="atNow"><span class="type-signature"></span>atNow<span class
416420

417421
<dt class="tag-source">Source:</dt>
418422
<dd class="tag-source"><ul class="dummy"><li>
419-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line375">line 375</a>
423+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line380">line 380</a>
420424
</li></ul></dd>
421425

422426

@@ -576,7 +580,7 @@ <h5>Parameters:</h5>
576580

577581
<dt class="tag-source">Source:</dt>
578582
<dd class="tag-source"><ul class="dummy"><li>
579-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line287">line 287</a>
583+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line292">line 292</a>
580584
</li></ul></dd>
581585

582586

@@ -687,7 +691,7 @@ <h4 class="name" id="close"><span class="type-signature">(async) </span>close<sp
687691

688692
<dt class="tag-source">Source:</dt>
689693
<dd class="tag-source"><ul class="dummy"><li>
690-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line167">line 167</a>
694+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line172">line 172</a>
691695
</li></ul></dd>
692696

693697

@@ -882,7 +886,7 @@ <h5>Parameters:</h5>
882886

883887
<dt class="tag-source">Source:</dt>
884888
<dd class="tag-source"><ul class="dummy"><li>
885-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line111">line 111</a>
889+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line116">line 116</a>
886890
</li></ul></dd>
887891

888892

@@ -1064,7 +1068,7 @@ <h5>Parameters:</h5>
10641068

10651069
<dt class="tag-source">Source:</dt>
10661070
<dd class="tag-source"><ul class="dummy"><li>
1067-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line302">line 302</a>
1071+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line307">line 307</a>
10681072
</li></ul></dd>
10691073

10701074

@@ -1175,7 +1179,7 @@ <h4 class="name" id="flush"><span class="type-signature">(async) </span>flush<sp
11751179

11761180
<dt class="tag-source">Source:</dt>
11771181
<dd class="tag-source"><ul class="dummy"><li>
1178-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line180">line 180</a>
1182+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line185">line 185</a>
11791183
</li></ul></dd>
11801184

11811185

@@ -1357,7 +1361,7 @@ <h5>Parameters:</h5>
13571361

13581362
<dt class="tag-source">Source:</dt>
13591363
<dd class="tag-source"><ul class="dummy"><li>
1360-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line318">line 318</a>
1364+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line323">line 323</a>
13611365
</li></ul></dd>
13621366

13631367

@@ -1468,7 +1472,7 @@ <h4 class="name" id="reset"><span class="type-signature"></span>reset<span class
14681472

14691473
<dt class="tag-source">Source:</dt>
14701474
<dd class="tag-source"><ul class="dummy"><li>
1471-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line97">line 97</a>
1475+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line102">line 102</a>
14721476
</li></ul></dd>
14731477

14741478

@@ -1629,7 +1633,7 @@ <h5>Parameters:</h5>
16291633

16301634
<dt class="tag-source">Source:</dt>
16311635
<dd class="tag-source"><ul class="dummy"><li>
1632-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line82">line 82</a>
1636+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line87">line 87</a>
16331637
</li></ul></dd>
16341638

16351639

@@ -1789,7 +1793,7 @@ <h5>Parameters:</h5>
17891793

17901794
<dt class="tag-source">Source:</dt>
17911795
<dd class="tag-source"><ul class="dummy"><li>
1792-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line270">line 270</a>
1796+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line275">line 275</a>
17931797
</li></ul></dd>
17941798

17951799

@@ -1971,7 +1975,7 @@ <h5>Parameters:</h5>
19711975

19721976
<dt class="tag-source">Source:</dt>
19731977
<dd class="tag-source"><ul class="dummy"><li>
1974-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line245">line 245</a>
1978+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line250">line 250</a>
19751979
</li></ul></dd>
19761980

19771981

@@ -2130,7 +2134,7 @@ <h5>Parameters:</h5>
21302134

21312135
<dt class="tag-source">Source:</dt>
21322136
<dd class="tag-source"><ul class="dummy"><li>
2133-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line224">line 224</a>
2137+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line229">line 229</a>
21342138
</li></ul></dd>
21352139

21362140

@@ -2312,7 +2316,7 @@ <h5>Parameters:</h5>
23122316

23132317
<dt class="tag-source">Source:</dt>
23142318
<dd class="tag-source"><ul class="dummy"><li>
2315-
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line338">line 338</a>
2319+
<a href="src_sender.js.html">src/sender.js</a>, <a href="src_sender.js.html#line343">line 343</a>
23162320
</li></ul></dd>
23172321

23182322

@@ -2386,7 +2390,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-@q
23862390
<br class="clear">
23872391

23882392
<footer>
2389-
Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 3.6.11</a> on Wed May 17 2023 18:51:54 GMT+0100 (British Summer Time)
2393+
Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 3.6.11</a> on Fri May 26 2023 10:30:37 GMT+0100 (British Summer Time)
23902394
</footer>
23912395

23922396
<script> prettyPrint(); </script>

‎docs/index.html

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,78 @@ <h3>TypeScript example</h3>
159159
}
160160

161161
run().then(value => console.log(value)).catch(err => console.log(err));
162+
</code></pre>
163+
<h3>Worker threads example</h3>
164+
<pre class="prettyprint source lang-javascript"><code>const { Sender } = require(&quot;@questdb/nodejs-client&quot;);
165+
const { Worker, isMainThread, parentPort, workerData } = require('worker_threads');
166+
167+
// fake venue
168+
// generates random prices for a ticker for max 5 seconds, then the feed closes
169+
function* venue(ticker) {
170+
let end = false;
171+
setTimeout(() => { end = true; }, rndInt(5000));
172+
while (!end) {
173+
yield {&quot;ticker&quot;: ticker, &quot;price&quot;: Math.random()};
174+
}
175+
}
176+
177+
// market data feed simulator
178+
// uses the fake venue to deliver price updates to the feed handler (onTick() callback)
179+
async function subscribe(ticker, onTick) {
180+
const feed = venue(workerData.ticker);
181+
let tick;
182+
while (tick = feed.next().value) {
183+
await onTick(tick);
184+
await sleep(rndInt(30));
185+
}
186+
}
187+
188+
async function run() {
189+
if (isMainThread) {
190+
const tickers = [&quot;t1&quot;, &quot;t2&quot;, &quot;t3&quot;, &quot;t4&quot;];
191+
// main thread to start a worker thread for each ticker
192+
for (let ticker in tickers) {
193+
const worker = new Worker(__filename, { workerData: { ticker: ticker } })
194+
.on('error', (err) => { throw err; })
195+
.on('exit', () => { console.log(`${ticker} thread exiting...`); })
196+
.on('message', (msg) => { console.log(&quot;Ingested &quot; + msg.count + &quot; prices for ticker &quot; + msg.ticker); });
197+
}
198+
} else {
199+
// it is important that each worker has a dedicated sender object
200+
// threads cannot share the sender because they would write into the same buffer
201+
const sender = new Sender({ bufferSize: 4096 });
202+
await sender.connect({ port: 9009, host: &quot;localhost&quot; });
203+
204+
// subscribe for the market data of the ticker assigned to the worker
205+
// ingest each price update into the database using the sender
206+
let count = 0;
207+
await subscribe(workerData.ticker, async (tick) => {
208+
sender
209+
.table(&quot;prices&quot;)
210+
.symbol(&quot;ticker&quot;, tick.ticker)
211+
.floatColumn(&quot;price&quot;, tick.price)
212+
.atNow();
213+
await sender.flush();
214+
count++;
215+
});
216+
217+
// let the main thread know how many prices were ingested
218+
parentPort.postMessage({&quot;ticker&quot;: workerData.ticker, &quot;count&quot;: count});
219+
220+
// close the connection to the database
221+
await sender.close();
222+
}
223+
}
224+
225+
function sleep(ms) {
226+
return new Promise(resolve => setTimeout(resolve, ms));
227+
}
228+
229+
function rndInt(limit) {
230+
return Math.floor((Math.random() * limit) + 1);
231+
}
232+
233+
run().catch((err) => console.log(err));
162234
</code></pre></article>
163235
</section>
164236

@@ -176,7 +248,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-@q
176248
<br class="clear">
177249

178250
<footer>
179-
Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 3.6.11</a> on Wed May 17 2023 18:51:54 GMT+0100 (British Summer Time)
251+
Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 3.6.11</a> on Fri May 26 2023 10:30:37 GMT+0100 (British Summer Time)
180252
</footer>
181253

182254
<script> prettyPrint(); </script>

‎docs/index.js.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-@q
5353
<br class="clear">
5454

5555
<footer>
56-
Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 3.6.11</a> on Wed May 17 2023 18:51:54 GMT+0100 (British Summer Time)
56+
Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 3.6.11</a> on Fri May 26 2023 10:30:37 GMT+0100 (British Summer Time)
5757
</footer>
5858

5959
<script> prettyPrint(); </script>

‎docs/module-@questdb_nodejs-client.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ <h2><a href="index.html">Home</a></h2><h3>Modules</h3><ul><li><a href="module-@q
156156
<br class="clear">
157157

158158
<footer>
159-
Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 3.6.11</a> on Wed May 17 2023 18:51:54 GMT+0100 (British Summer Time)
159+
Documentation generated by <a href="https://github.com/jsdoc/jsdoc">JSDoc 3.6.11</a> on Fri May 26 2023 10:30:37 GMT+0100 (British Summer Time)
160160
</footer>
161161

162162
<script> prettyPrint(); </script>

0 commit comments

Comments
 (0)