feat: sign record's public url (#358)

* feat: sign public url

* refactor: compose url with URL()

* refactor: clean up

* chore: added test for publicUrl

* fix: use apiPublicBaseUrl to generate public url

* fix: improve test

* refactor: clean up

* refactor: improved test

* refactor: clean up

* fix: CI

---------

Co-authored-by: Jan Buchar <Teyras@gmail.com>

Author: danpoletaev

Diff for: package-lock.json

@@ -67,7 +67,7 @@
         "@typescript-eslint/eslint-plugin": "^7.0.0",
         "@typescript-eslint/parser": "^7.0.0",
         "commitlint": "^19.3.0",
-        "crawlee": "^3.11.5",
+        "crawlee": "^3.13.0",
         "eslint": "^8.57.0",
         "fs-extra": "^11.2.0",
         "gen-esm-wrapper": "^1.1.3",

Diff for: package.json

@@ -58,15 +58,15 @@
         "@apify/input_secrets": "^1.1.40",
         "@apify/log": "^2.4.3",
         "@apify/timeout": "^0.3.0",
-        "@apify/utilities": "^2.9.3",
-        "@crawlee/core": "^3.9.0",
-        "@crawlee/types": "^3.9.0",
-        "@crawlee/utils": "^3.9.0",
-        "apify-client": "^2.12.0",
+        "@apify/utilities": "^2.13.0",
+        "@crawlee/core": "^3.13.0",
+        "@crawlee/types": "^3.13.0",
+        "@crawlee/utils": "^3.13.0",
+        "apify-client": "^2.12.1",
         "fs-extra": "^11.2.0",
         "ow": "^0.28.2",
         "semver": "^7.5.4",
         "tslib": "^2.6.2",
         "ws": "^8.18.0"
     }
-}
+}

Diff for: packages/apify/package.json

@@ -16,6 +16,9 @@ export interface ConfigurationOptions extends CoreConfigurationOptions {
     actorRunId?: string;
     actorTaskId?: string;
     apiBaseUrl?: string;
+    // apiBaseUrl is the internal API URL, accessible only within the platform(private network),
+    // while apiPublicBaseUrl is the public API URL, available externally(through internet).
+    apiPublicBaseUrl?: string;
     containerPort?: number;
     containerUrl?: string;
     proxyHostname?: string;
@@ -139,6 +142,7 @@ export class Configuration extends CoreConfiguration {
         APIFY_ACTOR_EVENTS_WS_URL: 'actorEventsWsUrl',
         APIFY_ACTOR_ID: 'actorId',
         APIFY_API_BASE_URL: 'apiBaseUrl',
+        APIFY_API_PUBLIC_BASE_URL: 'apiPublicBaseUrl',
         APIFY_IS_AT_HOME: 'isAtHome',
         APIFY_ACTOR_RUN_ID: 'actorRunId',
         APIFY_ACTOR_TASK_ID: 'actorTaskId',
@@ -183,6 +187,7 @@ export class Configuration extends CoreConfiguration {
         defaultRequestQueueId: LOCAL_ACTOR_ENV_VARS[ACTOR_ENV_VARS.DEFAULT_REQUEST_QUEUE_ID],
         inputKey: 'INPUT',
         apiBaseUrl: 'https://api.apify.com',
+        apiPublicBaseUrl: 'https://api.apify.com',
         proxyStatusUrl: 'http://proxy.apify.com',
         proxyHostname: LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_HOSTNAME],
         proxyPort: +LOCAL_APIFY_ENV_VARS[APIFY_ENV_VARS.PROXY_PORT],

Diff for: packages/apify/src/configuration.ts

@@ -1,3 +1,4 @@
+import { createHmacSignature } from '@apify/utilities';
 import type { StorageManagerOptions } from '@crawlee/core';
 import { KeyValueStore as CoreKeyValueStore } from '@crawlee/core';
 
@@ -15,11 +16,18 @@ export class KeyValueStore extends CoreKeyValueStore {
      * access the value in the remote key-value store.
      */
     override getPublicUrl(key: string): string {
-        if (!(this.config as Configuration).get('isAtHome') && getPublicUrl) {
+        const config = this.config as Configuration;
+        if (!config.get('isAtHome') && getPublicUrl) {
             return getPublicUrl.call(this, key);
         }
 
-        return `https://api.apify.com/v2/key-value-stores/${this.id}/records/${key}`;
+        const publicUrl = new URL(`${config.get('apiPublicBaseUrl')}/v2/key-value-stores/${this.id}/records/${key}`);
+
+        if (this.storageObject?.urlSigningSecretKey) {
+            publicUrl.searchParams.append('signature', createHmacSignature(this.storageObject.urlSigningSecretKey as string, key));
+        }
+
+        return publicUrl.toString();
     }
 
     /**

Diff for: packages/apify/src/key_value_store.ts

@@ -0,0 +1,19 @@
+import { Actor, log } from 'apify';
+
+await Actor.init();
+
+const { data } = await Actor.getInput();
+
+await Actor.setValue('public-record-key', JSON.stringify(data), { contentType: `application/json` });
+
+const defaultKeyValueStore = await Actor.openKeyValueStore();
+const publicUrl = defaultKeyValueStore.getPublicUrl('public-record-key');
+
+// Here we store the url itself
+await Actor.setValue('urlToPublicData', publicUrl);
+
+log.info('Generated public url', { publicUrl });
+
+await Actor.pushData({ publicUrl });
+
+await Actor.exit();

Diff for: test/e2e/sdk/publicUrl/src/main.mjs

@@ -0,0 +1,19 @@
+import assert from 'node:assert/strict';
+
+import { ApifyClient } from 'apify';
+
+const PUBLIC_DATA = { exposedData: 'test' };
+
+const client = new ApifyClient({
+    token: process.env.APIFY_TOKEN,
+});
+
+const actor = client.actor(process.argv[2]);
+
+const run = await actor.call({ data: PUBLIC_DATA }, { waitSecs: 15 });
+assert.equal(run.exitCode, 0);
+
+const publicUrl = await client.keyValueStore(run.defaultKeyValueStoreId).getRecord('urlToPublicData');
+const data = await fetch(publicUrl.value).then((res) => res.json());
+
+assert.deepEqual(data, PUBLIC_DATA);