diff --git a/.gitignore b/.gitignore index 57e91da..3d7d4f0 100644 --- a/.gitignore +++ b/.gitignore @@ -125,4 +125,6 @@ lib */.output e2e/esmCompatibility/.output src/e2e/esmCompatibility/.output -**/0x \ No newline at end of file +**/0x + +**/*.db \ No newline at end of file diff --git a/src/packages/dumbo/src/benchmarks/index.ts b/src/packages/dumbo/src/benchmarks/index.ts index e563c37..424051f 100644 --- a/src/packages/dumbo/src/benchmarks/index.ts +++ b/src/packages/dumbo/src/benchmarks/index.ts @@ -3,11 +3,16 @@ import 'dotenv/config'; import Benchmark from 'benchmark'; import pg from 'pg'; import { rawSql, single } from '..'; -import { defaultPostgreSQLConenctionString, dumbo } from '../pg'; +import { + defaultPostgreSQLConenctionString, + dumbo, + PostgreSQLConnectionString, +} from '../pg'; -const connectionString = +const connectionString = PostgreSQLConnectionString( process.env.BENCHMARK_POSTGRESQL_CONNECTION_STRING ?? - defaultPostgreSQLConenctionString; + defaultPostgreSQLConenctionString, +); const pooled = process.env.BENCHMARK_CONNECTION_POOLED === 'true'; diff --git a/src/packages/dumbo/src/core/connections/connection.ts b/src/packages/dumbo/src/core/connections/connection.ts index c4be4e6..c8e6cc8 100644 --- a/src/packages/dumbo/src/core/connections/connection.ts +++ b/src/packages/dumbo/src/core/connections/connection.ts @@ -1,5 +1,6 @@ import type { ConnectorType } from '../connectors'; import { + createDeferredExecutor, sqlExecutor, type DbSQLExecutor, type WithSQLExecutor, @@ -14,7 +15,7 @@ export interface Connection< Connector extends ConnectorType = ConnectorType, DbClient = unknown, > extends WithSQLExecutor, - DatabaseTransactionFactory { + DatabaseTransactionFactory { connector: Connector; open: () => Promise; close: () => Promise; @@ -85,3 +86,53 @@ export const createConnection = < return typedConnection; }; + +export const createDeferredConnection = ( + connector: Connector, + importConnection: () => Promise>, +): Connection => { + const getConnection = importConnection(); + + const execute = createDeferredExecutor(async () => { + const conn = await getConnection; + return conn.execute; + }); + + const connection: Connection = { + connector, + execute, + + open: async (): Promise => { + const conn = await getConnection; + return conn.open(); + }, + + close: async (): Promise => { + if (getConnection) { + const conn = await getConnection; + await conn.close(); + } + }, + + transaction: () => { + const transaction = getConnection.then((c) => c.transaction()); + + return { + connector, + connection, + execute: createDeferredExecutor( + async () => (await transaction).execute, + ), + begin: async () => (await transaction).begin(), + commit: async () => (await transaction).commit(), + rollback: async () => (await transaction).rollback(), + }; + }, + withTransaction: async (handle) => { + const connection = await getConnection; + return connection.withTransaction(handle); + }, + }; + + return connection; +}; diff --git a/src/packages/dumbo/src/core/connections/pool.ts b/src/packages/dumbo/src/core/connections/pool.ts index 1f9ba9f..805a690 100644 --- a/src/packages/dumbo/src/core/connections/pool.ts +++ b/src/packages/dumbo/src/core/connections/pool.ts @@ -1,4 +1,7 @@ +import { createDeferredConnection } from '../connections'; +import type { ConnectorType } from '../connectors'; import { + createDeferredExecutor, executeInNewConnection, sqlExecutorInNewConnection, type WithSQLExecutor, @@ -72,3 +75,39 @@ export const createConnectionPool = < return result as ConnectionPoolType; }; + +export const createDeferredConnectionPool = ( + connector: Connector, + importPool: () => Promise>>, +): ConnectionPool> => { + let poolPromise: Promise>> | null = null; + + const getPool = async (): Promise>> => { + if (poolPromise) return poolPromise; + try { + return (poolPromise = importPool()); + } catch (error) { + throw new Error( + `Failed to import connection pool: ${error instanceof Error ? error.message : String(error)}`, + ); + } + }; + + return createConnectionPool({ + connector, + execute: createDeferredExecutor(async () => { + const connection = await getPool(); + return connection.execute; + }), + close: async () => { + if (!poolPromise) return; + const pool = await poolPromise; + await pool.close(); + poolPromise = null; + }, + getConnection: () => + createDeferredConnection(connector, async () => + (await getPool()).connection(), + ), + }); +}; diff --git a/src/packages/dumbo/src/core/execute/execute.ts b/src/packages/dumbo/src/core/execute/execute.ts index fbc292d..2dc0081 100644 --- a/src/packages/dumbo/src/core/execute/execute.ts +++ b/src/packages/dumbo/src/core/execute/execute.ts @@ -154,3 +154,56 @@ export const executeInNewConnection = async < await connection.close(); } }; + +export const createDeferredExecutor = ( + importExecutor: () => Promise, +): SQLExecutor => { + let executor: SQLExecutor | null = null; + + const getExecutor = async (): Promise => { + if (!executor) { + try { + executor = await importExecutor(); + } catch (error) { + throw new Error( + `Failed to import SQL executor: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + return executor; + }; + + return { + query: async ( + sql: SQL, + options?: SQLQueryOptions, + ): Promise> => { + const exec = await getExecutor(); + return exec.query(sql, options); + }, + + batchQuery: async ( + sqls: SQL[], + options?: SQLQueryOptions, + ): Promise[]> => { + const exec = await getExecutor(); + return exec.batchQuery(sqls, options); + }, + + command: async ( + sql: SQL, + options?: SQLCommandOptions, + ): Promise> => { + const exec = await getExecutor(); + return exec.command(sql, options); + }, + + batchCommand: async ( + sqls: SQL[], + options?: SQLCommandOptions, + ): Promise[]> => { + const exec = await getExecutor(); + return exec.batchCommand(sqls, options); + }, + }; +}; diff --git a/src/packages/dumbo/src/core/index.ts b/src/packages/dumbo/src/core/index.ts index be1f66d..f90219d 100644 --- a/src/packages/dumbo/src/core/index.ts +++ b/src/packages/dumbo/src/core/index.ts @@ -1,4 +1,5 @@ -import type { Connection, ConnectionPool } from './connections'; +import type { DatabaseConnectionString } from '../storage/all'; +import { type Connection, type ConnectionPool } from './connections'; import type { ConnectorType } from './connectors'; export * from './connections'; @@ -11,11 +12,13 @@ export * from './serializer'; export * from './sql'; export * from './tracing'; -export type DumboOptions = { - connector?: Connector; -}; - export type Dumbo< Connector extends ConnectorType = ConnectorType, ConnectionType extends Connection = Connection, > = ConnectionPool; + +export type DumboConnectionOptions< + T extends DatabaseConnectionString = DatabaseConnectionString, +> = { + connectionString: string | T; +}; diff --git a/src/packages/dumbo/src/index.ts b/src/packages/dumbo/src/index.ts index 4b0e041..a43824a 100644 --- a/src/packages/dumbo/src/index.ts +++ b/src/packages/dumbo/src/index.ts @@ -1 +1,2 @@ export * from './core'; +export * from './storage/all'; diff --git a/src/packages/dumbo/src/pg.ts b/src/packages/dumbo/src/pg.ts index d27f15f..f165bce 100644 --- a/src/packages/dumbo/src/pg.ts +++ b/src/packages/dumbo/src/pg.ts @@ -1,3 +1,2 @@ -export * from './core'; export * from './storage/postgresql/core'; export * from './storage/postgresql/pg'; diff --git a/src/packages/dumbo/src/sqlite3.ts b/src/packages/dumbo/src/sqlite3.ts index d0bb16e..9d11f1d 100644 --- a/src/packages/dumbo/src/sqlite3.ts +++ b/src/packages/dumbo/src/sqlite3.ts @@ -1,3 +1,2 @@ -export * from './core'; export * from './storage/sqlite/core'; export * from './storage/sqlite/sqlite3'; diff --git a/src/packages/dumbo/src/storage/all/connections/connectionString.ts b/src/packages/dumbo/src/storage/all/connections/connectionString.ts new file mode 100644 index 0000000..59d05fd --- /dev/null +++ b/src/packages/dumbo/src/storage/all/connections/connectionString.ts @@ -0,0 +1,38 @@ +import type { ConnectorTypeParts } from '../../../core'; +import type { PostgreSQLConnectionString } from '../../postgresql/core'; +import type { SQLiteConnectionString } from '../../sqlite/core'; + +export type DatabaseConnectionString = + | PostgreSQLConnectionString + | SQLiteConnectionString; + +export const parseConnectionString = ( + // eslint-disable-next-line @typescript-eslint/no-redundant-type-constituents + connectionString: DatabaseConnectionString | string, +): ConnectorTypeParts => { + if ( + connectionString.startsWith('postgresql://') || + connectionString.startsWith('postgres://') + ) { + return { + databaseType: 'PostgreSQL', + driverName: 'pg', + }; + } + + if ( + connectionString.startsWith('file:') || + connectionString === ':memory:' || + connectionString.startsWith('/') || + connectionString.startsWith('./') + ) { + return { + databaseType: 'SQLite', + driverName: 'sqlite3', + }; + } + + throw new Error( + `Unsupported database connection string: ${connectionString}`, + ); +}; diff --git a/src/packages/dumbo/src/storage/all/connections/index.ts b/src/packages/dumbo/src/storage/all/connections/index.ts new file mode 100644 index 0000000..f6e062b --- /dev/null +++ b/src/packages/dumbo/src/storage/all/connections/index.ts @@ -0,0 +1 @@ +export * from './connectionString'; diff --git a/src/packages/dumbo/src/storage/all/index.ts b/src/packages/dumbo/src/storage/all/index.ts new file mode 100644 index 0000000..da79584 --- /dev/null +++ b/src/packages/dumbo/src/storage/all/index.ts @@ -0,0 +1,76 @@ +import { + type Connection, + type ConnectionPool, + type ConnectorType, + createDeferredConnectionPool, + type DumboConnectionOptions, +} from '../../core'; +import type { PostgreSQLConnectionString } from '../postgresql/core'; +import type { SQLiteConnectionString } from '../sqlite/core'; +import { + type DatabaseConnectionString, + parseConnectionString, +} from './connections'; + +export * from './connections'; + +// Helper type to infer the connector type based on connection string +export type InferConnector = + T extends PostgreSQLConnectionString + ? ConnectorType<'PostgreSQL', 'pg'> + : T extends SQLiteConnectionString + ? ConnectorType<'SQLite', 'sqlite3'> + : never; + +// Helper type to infer the connection type based on connection string +export type InferConnection = + T extends PostgreSQLConnectionString + ? Connection> + : T extends SQLiteConnectionString + ? Connection> + : never; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const importDrivers: Record Promise> = { + 'PostgreSQL:pg': () => import('../postgresql/pg'), + 'SQLite:sqlite3': () => import('../sqlite/sqlite3'), +}; + +export function dumbo< + ConnectionString extends DatabaseConnectionString, + DatabaseOptions extends DumboConnectionOptions, +>(options: DatabaseOptions): ConnectionPool> { + const { connectionString } = options; + + const { databaseType, driverName } = parseConnectionString(connectionString); + + const connector: InferConnection['connector'] = + `${databaseType}:${driverName}` as InferConnection['connector']; + + const importDriver = importDrivers[connector]; + if (!importDriver) { + throw new Error(`Unsupported connector: ${connector}`); + } + + const importAndCreatePool = async () => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const module = await importDriver(); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const poolFactory: (options: { + connectionString: string; + }) => ConnectionPool> = + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + 'dumbo' in module ? module.dumbo : undefined; + + if (poolFactory === undefined) + throw new Error(`No pool factory found for connector: ${connector}`); + + return poolFactory({ connector, ...options }); + }; + + return createDeferredConnectionPool( + connector, + importAndCreatePool, + ) as ConnectionPool>; +} diff --git a/src/packages/dumbo/src/storage/postgresql/core/connections/connectionString.ts b/src/packages/dumbo/src/storage/postgresql/core/connections/connectionString.ts index 51dca74..c06ec9f 100644 --- a/src/packages/dumbo/src/storage/postgresql/core/connections/connectionString.ts +++ b/src/packages/dumbo/src/storage/postgresql/core/connections/connectionString.ts @@ -1,8 +1,26 @@ import pgcs from 'pg-connection-string'; import { defaultPostgreSqlDatabase } from '../schema'; -export const defaultPostgreSQLConenctionString = - 'postgresql://postgres@localhost:5432/postgres'; +export const defaultPostgreSQLConenctionString: PostgreSQLConnectionString = + 'postgresql://postgres@localhost:5432/postgres' as PostgreSQLConnectionString; export const getDatabaseNameOrDefault = (connectionString: string) => pgcs.parse(connectionString).database ?? defaultPostgreSqlDatabase; + +export type PostgreSQLConnectionString = + | `postgresql://${string}` + | `postgres://${string}`; + +export const PostgreSQLConnectionString = ( + connectionString: string, +): PostgreSQLConnectionString => { + if ( + !connectionString.startsWith('postgresql://') && + !connectionString.startsWith('postgres://') + ) { + throw new Error( + `Invalid PostgreSQL connection string: ${connectionString}. It should start with "postgresql://".`, + ); + } + return connectionString as PostgreSQLConnectionString; +}; diff --git a/src/packages/dumbo/src/storage/postgresql/core/schema/migrations.int.spec.ts b/src/packages/dumbo/src/storage/postgresql/core/schema/migrations.int.spec.ts index 47923e3..b9fef05 100644 --- a/src/packages/dumbo/src/storage/postgresql/core/schema/migrations.int.spec.ts +++ b/src/packages/dumbo/src/storage/postgresql/core/schema/migrations.int.spec.ts @@ -4,10 +4,10 @@ import { } from '@testcontainers/postgresql'; import assert from 'assert'; import { after, before, beforeEach, describe, it } from 'node:test'; -import { tableExists } from '..'; +import { PostgreSQLConnectionString, tableExists } from '..'; import { type Dumbo } from '../../../..'; import { count, rawSql, sql } from '../../../../core'; -import { type SQLMigration, MIGRATIONS_LOCK_ID } from '../../../../core/schema'; +import { MIGRATIONS_LOCK_ID, type SQLMigration } from '../../../../core/schema'; import { dumbo } from '../../../../pg'; import { acquireAdvisoryLock, releaseAdvisoryLock } from '../locks'; import { runPostgreSQLMigrations } from './migrations'; @@ -15,11 +15,11 @@ import { runPostgreSQLMigrations } from './migrations'; void describe('Migration Integration Tests', () => { let pool: Dumbo; let postgres: StartedPostgreSqlContainer; - let connectionString: string; + let connectionString: PostgreSQLConnectionString; before(async () => { postgres = await new PostgreSqlContainer().start(); - connectionString = postgres.getConnectionUri(); + connectionString = PostgreSQLConnectionString(postgres.getConnectionUri()); pool = dumbo({ connectionString }); }); diff --git a/src/packages/dumbo/src/storage/postgresql/pg/connections/connection.generic.int.spec.ts b/src/packages/dumbo/src/storage/postgresql/pg/connections/connection.generic.int.spec.ts new file mode 100644 index 0000000..0b81b2d --- /dev/null +++ b/src/packages/dumbo/src/storage/postgresql/pg/connections/connection.generic.int.spec.ts @@ -0,0 +1,216 @@ +import { + PostgreSqlContainer, + type StartedPostgreSqlContainer, +} from '@testcontainers/postgresql'; +import { after, before, describe, it } from 'node:test'; +import pg from 'pg'; +import { nodePostgresPool } from '.'; +import { dumbo } from '..'; +import { rawSql } from '../../../../core'; +import { endPool, getPool } from './pool'; + +void describe('Node Postgresql', () => { + let postgres: StartedPostgreSqlContainer; + let connectionString: string; + + before(async () => { + postgres = await new PostgreSqlContainer().start(); + connectionString = postgres.getConnectionUri(); + }); + + after(async () => { + await postgres.stop(); + }); + + void describe('nodePostgresPool', () => { + void it('connects using default pool', async () => { + const pool = dumbo({ connectionString }); + const connection = await pool.connection(); + + try { + await connection.execute.query(rawSql('SELECT 1')); + } catch (error) { + console.log(error); + } finally { + await connection.close(); + await pool.close(); + } + }); + + void it('connects using ambient pool', async () => { + const nativePool = getPool(connectionString); + const pool = nodePostgresPool({ connectionString, pool: nativePool }); + const connection = await pool.connection(); + + try { + await connection.execute.query(rawSql('SELECT 1')); + } finally { + await connection.close(); + await pool.close(); + await endPool({ connectionString }); + } + }); + + void it('connects using client', async () => { + const pool = nodePostgresPool({ + connectionString, + pooled: false, + }); + const connection = await pool.connection(); + + try { + await connection.execute.query(rawSql('SELECT 1')); + } finally { + await connection.close(); + await pool.close(); + } + }); + + void it('connects using ambient client', async () => { + const existingClient = new pg.Client({ connectionString }); + await existingClient.connect(); + + const pool = nodePostgresPool({ + connectionString, + client: existingClient, + }); + const connection = await pool.connection(); + + try { + await connection.execute.query(rawSql('SELECT 1')); + } finally { + await connection.close(); + await pool.close(); + await existingClient.end(); + } + }); + + void it('connects using connected ambient connected connection', async () => { + const ambientPool = nodePostgresPool({ connectionString }); + const ambientConnection = await ambientPool.connection(); + await ambientConnection.open(); + + const pool = nodePostgresPool({ + connectionString, + connection: ambientConnection, + }); + + try { + await pool.execute.query(rawSql('SELECT 1')); + } finally { + await pool.close(); + await ambientConnection.close(); + await ambientPool.close(); + } + }); + + void it('connects using connected ambient not-connected connection', async () => { + const ambientPool = nodePostgresPool({ connectionString }); + const ambientConnection = await ambientPool.connection(); + + const pool = nodePostgresPool({ + connectionString, + connection: ambientConnection, + }); + + try { + await pool.execute.query(rawSql('SELECT 1')); + } finally { + await pool.close(); + await ambientConnection.close(); + await ambientPool.close(); + } + }); + + void it('connects using ambient connected connection with transaction', async () => { + const ambientPool = nodePostgresPool({ connectionString }); + const ambientConnection = await ambientPool.connection(); + await ambientConnection.open(); + + try { + await ambientConnection.withTransaction(async () => { + const pool = nodePostgresPool({ + connectionString, + connection: ambientConnection, + }); + try { + await pool.execute.query(rawSql('SELECT 1')); + + return { success: true, result: undefined }; + } finally { + await pool.close(); + } + }); + } finally { + await ambientConnection.close(); + await ambientPool.close(); + } + }); + + void it('connects using ambient not-connected connection with transaction', async () => { + const ambientPool = nodePostgresPool({ connectionString }); + const ambientConnection = await ambientPool.connection(); + + try { + await ambientConnection.withTransaction(async () => { + const pool = nodePostgresPool({ + connectionString, + connection: ambientConnection, + }); + try { + await pool.execute.query(rawSql('SELECT 1')); + + return { success: true, result: undefined }; + } finally { + await pool.close(); + } + }); + } finally { + await ambientConnection.close(); + await ambientPool.close(); + } + }); + + void it('connects using ambient connection in withConnection scope', async () => { + const ambientPool = nodePostgresPool({ connectionString }); + try { + await ambientPool.withConnection(async (ambientConnection) => { + const pool = nodePostgresPool({ + connectionString, + connection: ambientConnection, + }); + try { + await pool.execute.query(rawSql('SELECT 1')); + + return { success: true, result: undefined }; + } finally { + await pool.close(); + } + }); + } finally { + await ambientPool.close(); + } + }); + + void it('connects using ambient connection in withConnection and withTransaction scope', async () => { + const ambientPool = nodePostgresPool({ connectionString }); + try { + await ambientPool.withConnection((ambientConnection) => + ambientConnection.withTransaction(async () => { + const pool = nodePostgresPool({ + connectionString, + connection: ambientConnection, + }); + try { + await pool.execute.query(rawSql('SELECT 1')); + } finally { + await pool.close(); + } + }), + ); + } finally { + await ambientPool.close(); + } + }); + }); +}); diff --git a/src/packages/dumbo/src/storage/postgresql/pg/connections/transaction.ts b/src/packages/dumbo/src/storage/postgresql/pg/connections/transaction.ts index 025aed7..2755413 100644 --- a/src/packages/dumbo/src/storage/postgresql/pg/connections/transaction.ts +++ b/src/packages/dumbo/src/storage/postgresql/pg/connections/transaction.ts @@ -20,7 +20,7 @@ export const nodePostgresTransaction = ( getClient: Promise, options?: { close: (client: DbClient, error?: unknown) => Promise }, - ): DatabaseTransaction => ({ + ): DatabaseTransaction => ({ connection: connection(), connector: NodePostgresConnectorType, begin: async () => { diff --git a/src/packages/dumbo/src/storage/postgresql/pg/index.ts b/src/packages/dumbo/src/storage/postgresql/pg/index.ts index f29bb33..8ddbd11 100644 --- a/src/packages/dumbo/src/storage/postgresql/pg/index.ts +++ b/src/packages/dumbo/src/storage/postgresql/pg/index.ts @@ -1,4 +1,4 @@ -import type { Dumbo, DumboOptions } from '../../../core'; +import type { Dumbo } from '../../../core'; import { type NodePostgresConnection, type NodePostgresConnector, @@ -11,8 +11,7 @@ export type PostgresConnector = NodePostgresConnector; export type PostgresPool = NodePostgresPool; export type PostgresConnection = NodePostgresConnection; -export type PostgresPoolOptions = DumboOptions & - NodePostgresPoolOptions; +export type PostgresPoolOptions = NodePostgresPoolOptions; export const postgresPool = nodePostgresPool; export const connectionPool = postgresPool; diff --git a/src/packages/dumbo/src/storage/sqlite/core/connections/connectionString.ts b/src/packages/dumbo/src/storage/sqlite/core/connections/connectionString.ts new file mode 100644 index 0000000..79369c9 --- /dev/null +++ b/src/packages/dumbo/src/storage/sqlite/core/connections/connectionString.ts @@ -0,0 +1,21 @@ +export type SQLiteConnectionString = + | `file:${string}` + | `:memory:` + | `/${string}` + | `./${string}`; + +export const SQLiteConnectionString = ( + connectionString: string, +): SQLiteConnectionString => { + if ( + !connectionString.startsWith('file:') && + connectionString !== ':memory:' && + !connectionString.startsWith('/') && + !connectionString.startsWith('./') + ) { + throw new Error( + `Invalid SQLite connection string: ${connectionString}. It should start with "file:", ":memory:", "/", or "./".`, + ); + } + return connectionString as SQLiteConnectionString; +}; diff --git a/src/packages/dumbo/src/storage/sqlite/core/connections/index.ts b/src/packages/dumbo/src/storage/sqlite/core/connections/index.ts index c85ccf2..733dcfe 100644 --- a/src/packages/dumbo/src/storage/sqlite/core/connections/index.ts +++ b/src/packages/dumbo/src/storage/sqlite/core/connections/index.ts @@ -1,10 +1,16 @@ -import { sqliteSQLExecutor, type SQLiteConnectorType } from '..'; +import { + SQLiteConnectionString, + sqliteSQLExecutor, + type SQLiteConnectorType, + type SQLiteFileNameOrConnectionString, +} from '..'; import { createConnection, type Connection } from '../../../../core'; import { sqliteTransaction } from '../transactions'; export type Parameters = object | string | bigint | number | boolean | null; export type SQLiteClient = { + connect: () => Promise; close: () => Promise; command: (sql: string, values?: Parameters[]) => Promise; query: (sql: string, values?: Parameters[]) => Promise; @@ -127,9 +133,8 @@ export function sqliteConnection< } export type InMemorySQLiteDatabase = ':memory:'; -export const InMemorySQLiteDatabase = ':memory:'; +export const InMemorySQLiteDatabase = SQLiteConnectionString(':memory:'); -export type SQLiteClientOptions = { - // eslint-disable-next-line @typescript-eslint/no-redundant-type-constituents - fileName: InMemorySQLiteDatabase | string | undefined; -}; +export type SQLiteClientOptions = SQLiteFileNameOrConnectionString; + +export * from './connectionString'; diff --git a/src/packages/dumbo/src/storage/sqlite/core/pool/pool.ts b/src/packages/dumbo/src/storage/sqlite/core/pool/pool.ts index 24e66f2..20d8ee7 100644 --- a/src/packages/dumbo/src/storage/sqlite/core/pool/pool.ts +++ b/src/packages/dumbo/src/storage/sqlite/core/pool/pool.ts @@ -2,6 +2,7 @@ import { InMemorySQLiteDatabase, sqliteClientProvider, sqliteConnection, + SQLiteConnectionString, type SQLiteClient, type SQLiteClientConnection, type SQLiteConnectorType, @@ -53,19 +54,26 @@ export const sqliteAmbientConnectionPool = < export const sqliteSingletonClientPool = < ConnectorType extends SQLiteConnectorType = SQLiteConnectorType, ->(options: { - connector: ConnectorType; - fileName: string; - database?: string | undefined; -}): SQLiteAmbientClientPool => { - const { connector, fileName } = options; +>( + options: { + connector: ConnectorType; + database?: string | undefined; + } & SQLiteFileNameOrConnectionString, +): SQLiteAmbientClientPool => { + const { connector } = options; let connection: SQLiteClientConnection | undefined = undefined; const getConnection = () => { if (connection) return connection; - const connect = sqliteClientProvider(connector).then((sqliteClient) => - sqliteClient({ fileName }), + const connect = sqliteClientProvider(connector).then( + async (sqliteClient) => { + const client = sqliteClient(options); + + await client.connect(); + + return client; + }, ); return (connection = sqliteConnection({ @@ -91,18 +99,25 @@ export const sqliteSingletonClientPool = < export const sqliteAlwaysNewClientPool = < ConnectorType extends SQLiteConnectorType = SQLiteConnectorType, ->(options: { - connector: ConnectorType; - fileName: string; - database?: string | undefined; -}): SQLiteAmbientClientPool => { - const { connector, fileName } = options; +>( + options: { + connector: ConnectorType; + database?: string | undefined; + } & SQLiteFileNameOrConnectionString, +): SQLiteAmbientClientPool => { + const { connector } = options; return createConnectionPool({ connector: connector, getConnection: () => { - const connect = sqliteClientProvider(connector).then((sqliteClient) => - sqliteClient({ fileName }), + const connect = sqliteClientProvider(connector).then( + async (sqliteClient) => { + const client = sqliteClient(options); + + await client.connect(); + + return client; + }, ); return sqliteConnection({ @@ -145,11 +160,22 @@ export const sqliteAmbientClientPool = < }); }; +export type SQLiteFileNameOrConnectionString = + | { + // eslint-disable-next-line @typescript-eslint/no-redundant-type-constituents + fileName: string | SQLiteConnectionString; + connectionString?: never; + } + | { + // eslint-disable-next-line @typescript-eslint/no-redundant-type-constituents + connectionString: string | SQLiteConnectionString; + fileName?: never; + }; + export type SQLitePoolPooledOptions< ConnectorType extends SQLiteConnectorType = SQLiteConnectorType, > = { connector: ConnectorType; - fileName: string; pooled?: true; singleton?: boolean; }; @@ -159,20 +185,17 @@ export type SQLitePoolNotPooledOptions< > = | { connector: ConnectorType; - fileName: string; pooled?: false; client: SQLiteClient; singleton?: true; } | { connector: ConnectorType; - fileName: string; pooled?: boolean; singleton?: boolean; } | { connector: ConnectorType; - fileName: string; connection: | SQLitePoolClientConnection | SQLiteClientConnection; @@ -187,12 +210,13 @@ export type SQLitePoolOptions< | SQLitePoolNotPooledOptions ) & { serializer?: JSONSerializer; -}; +} & SQLiteFileNameOrConnectionString; export function sqlitePool< ConnectorType extends SQLiteConnectorType = SQLiteConnectorType, >( - options: SQLitePoolNotPooledOptions, + options: SQLitePoolNotPooledOptions & + SQLiteFileNameOrConnectionString, ): SQLiteAmbientClientPool; export function sqlitePool< ConnectorType extends SQLiteConnectorType = SQLiteConnectorType, @@ -201,7 +225,7 @@ export function sqlitePool< ): | SQLiteAmbientClientPool | SQLiteAmbientConnectionPool { - const { fileName, connector } = options; + const { connector } = options; // TODO: Handle dates and bigints // setSQLiteTypeParser(serializer ?? JSONSerializer); @@ -215,8 +239,12 @@ export function sqlitePool< connection: options.connection, }); - if (options.singleton === true || options.fileName == InMemorySQLiteDatabase) - return sqliteSingletonClientPool({ connector, fileName }); + if ( + options.singleton === true || + options.fileName === InMemorySQLiteDatabase || + options.connectionString === InMemorySQLiteDatabase + ) + return sqliteSingletonClientPool(options); - return sqliteAlwaysNewClientPool({ connector, fileName }); + return sqliteAlwaysNewClientPool(options); } diff --git a/src/packages/dumbo/src/storage/sqlite/core/transactions/index.ts b/src/packages/dumbo/src/storage/sqlite/core/transactions/index.ts index 9abdc2a..54c533d 100644 --- a/src/packages/dumbo/src/storage/sqlite/core/transactions/index.ts +++ b/src/packages/dumbo/src/storage/sqlite/core/transactions/index.ts @@ -22,7 +22,7 @@ export const sqliteTransaction = ( getClient: Promise, options?: { close: (client: DbClient, error?: unknown) => Promise }, - ): DatabaseTransaction => ({ + ): DatabaseTransaction => ({ connection: connection(), connector, begin: async () => { diff --git a/src/packages/dumbo/src/storage/sqlite/sqlite3/connections/connection.int.generic.spec.ts b/src/packages/dumbo/src/storage/sqlite/sqlite3/connections/connection.int.generic.spec.ts new file mode 100644 index 0000000..f0756ec --- /dev/null +++ b/src/packages/dumbo/src/storage/sqlite/sqlite3/connections/connection.int.generic.spec.ts @@ -0,0 +1,302 @@ +import assert from 'assert'; +import fs from 'fs'; +import { afterEach, describe, it } from 'node:test'; +import path from 'path'; +import { fileURLToPath } from 'url'; +import { rawSql } from '../../../../core'; +import { dumbo } from '../../../all'; +import { InMemorySQLiteDatabase, SQLiteConnectionString } from '../../core'; +import { sqlite3Client } from './connection'; + +void describe('Node SQLite pool', () => { + const inMemoryfileName = InMemorySQLiteDatabase; + + const testDatabasePath = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + ); + const fileName = path.resolve(testDatabasePath, 'test.db'); + const connectionString = SQLiteConnectionString(`file:${fileName}`); + + const testCases = [ + { + testName: 'in-memory', + connectionString: inMemoryfileName, + }, + { testName: 'file', connectionString }, + ]; + + afterEach(() => { + if (!fs.existsSync(fileName)) { + return; + } + try { + fs.unlinkSync(fileName); + } catch (error) { + console.log('Error deleting file:', error); + } + }); + + void describe(`in-memory database`, () => { + void it('returns the singleton connection', async () => { + const pool = dumbo({ + connectionString: inMemoryfileName, + }); + const connection = await pool.connection(); + const otherConnection = await pool.connection(); + + try { + // Won't work for now as it's lazy loaded + // assert.strictEqual(connection, otherConnection); + + const client = await connection.open(); + const otherClient = await otherConnection.open(); + assert.strictEqual(client, otherClient); + } finally { + await connection.close(); + await otherConnection.close(); + await pool.close(); + } + }); + }); + + void describe(`file-based database`, () => { + void it('returns the new connection each time', async () => { + const pool = dumbo({ + connectionString, + }); + const connection = await pool.connection(); + const otherConnection = await pool.connection(); + + try { + // Won't work for now as it's lazy loaded + // assert.notDeepStrictEqual(connection, otherConnection); + + const client = await connection.open(); + const otherClient = await otherConnection.open(); + assert.notDeepStrictEqual(client, otherClient); + } finally { + await connection.close(); + await otherConnection.close(); + await pool.close(); + } + }); + + void it('for singleton setting returns the singleton connection', async () => { + const pool = dumbo({ + connectionString, + singleton: true, + }); + const connection = await pool.connection(); + const otherConnection = await pool.connection(); + + try { + // Won't work for now as it's lazy loaded + // assert.strictEqual(connection, otherConnection); + + const client = await connection.open(); + const otherClient = await otherConnection.open(); + assert.strictEqual(client, otherClient); + } finally { + await connection.close(); + await otherConnection.close(); + await pool.close(); + } + }); + }); + + for (const { testName, connectionString } of testCases) { + void describe(`dumbo with ${testName} database`, () => { + void it('connects using default pool', async () => { + const pool = dumbo({ + connectionString, + }); + const connection = await pool.connection(); + + try { + await connection.execute.query(rawSql('SELECT 1')); + } catch (error) { + console.log(error); + assert.fail(error as Error); + } finally { + await connection.close(); + await pool.close(); + } + }); + + void it('connects using client', async () => { + const pool = dumbo({ + connectionString, + pooled: false, + }); + const connection = await pool.connection(); + + try { + await connection.execute.query(rawSql('SELECT 1')); + } finally { + await connection.close(); + await pool.close(); + } + }); + + void it('connects using ambient client', async () => { + const existingClient = sqlite3Client({ fileName }); + await existingClient.connect(); + + const pool = dumbo({ + connectionString, + client: existingClient, + }); + const connection = await pool.connection(); + + try { + await connection.execute.query(rawSql('SELECT 1')); + } finally { + await connection.close(); + await pool.close(); + await existingClient.close(); + } + }); + + void it('connects using connected ambient connected connection', async () => { + const ambientPool = dumbo({ + connectionString, + fileName, + }); + const ambientConnection = await ambientPool.connection(); + await ambientConnection.open(); + + const pool = dumbo({ + connectionString, + connection: ambientConnection, + }); + + try { + await pool.execute.query(rawSql('SELECT 1')); + } finally { + await pool.close(); + await ambientConnection.close(); + await ambientPool.close(); + } + }); + + void it('connects using connected ambient not-connected connection', async () => { + const ambientPool = dumbo({ + connectionString, + }); + const ambientConnection = await ambientPool.connection(); + + const pool = dumbo({ + connectionString, + connection: ambientConnection, + }); + + try { + await pool.execute.query(rawSql('SELECT 1')); + } finally { + await pool.close(); + await ambientConnection.close(); + await ambientPool.close(); + } + }); + + void it('connects using ambient connected connection with transaction', async () => { + const ambientPool = dumbo({ + connectionString, + }); + const ambientConnection = await ambientPool.connection(); + await ambientConnection.open(); + + try { + await ambientConnection.withTransaction(async () => { + const pool = dumbo({ + connectionString, + connection: ambientConnection, + }); + try { + await pool.execute.query(rawSql('SELECT 1')); + + return { success: true, result: undefined }; + } finally { + await pool.close(); + } + }); + } finally { + await ambientConnection.close(); + await ambientPool.close(); + } + }); + + void it('connects using ambient not-connected connection with transaction', async () => { + const ambientPool = dumbo({ + connectionString, + }); + const ambientConnection = await ambientPool.connection(); + + try { + await ambientConnection.withTransaction(async () => { + const pool = dumbo({ + connectionString, + connection: ambientConnection, + }); + try { + await pool.execute.query(rawSql('SELECT 1')); + + return { success: true, result: undefined }; + } finally { + await pool.close(); + } + }); + } finally { + await ambientConnection.close(); + await ambientPool.close(); + } + }); + + void it('connects using ambient connection in withConnection scope', async () => { + const ambientPool = dumbo({ + connectionString, + }); + try { + await ambientPool.withConnection(async (ambientConnection) => { + const pool = dumbo({ + connectionString, + connection: ambientConnection, + }); + try { + await pool.execute.query(rawSql('SELECT 1')); + + return { success: true, result: undefined }; + } finally { + await pool.close(); + } + }); + } finally { + await ambientPool.close(); + } + }); + + void it('connects using ambient connection in withConnection and withTransaction scope', async () => { + const ambientPool = dumbo({ + connectionString, + }); + try { + await ambientPool.withConnection((ambientConnection) => + ambientConnection.withTransaction(async () => { + const pool = dumbo({ + connectionString, + connection: ambientConnection, + }); + try { + await pool.execute.query(rawSql('SELECT 1')); + } finally { + await pool.close(); + } + }), + ); + } finally { + await ambientPool.close(); + } + }); + }); + } +}); diff --git a/src/packages/dumbo/src/storage/sqlite/sqlite3/connections/connection.int.spec.ts b/src/packages/dumbo/src/storage/sqlite/sqlite3/connections/connection.int.spec.ts index b3873e4..7301055 100644 --- a/src/packages/dumbo/src/storage/sqlite/sqlite3/connections/connection.int.spec.ts +++ b/src/packages/dumbo/src/storage/sqlite/sqlite3/connections/connection.int.spec.ts @@ -24,7 +24,11 @@ void describe('Node SQLite pool', () => { if (!fs.existsSync(fileName)) { return; } - fs.unlinkSync(fileName); + try { + fs.unlinkSync(fileName); + } catch (error) { + console.log('Error deleting file:', error); + } }); void describe(`in-memory database`, () => { @@ -127,6 +131,7 @@ void describe('Node SQLite pool', () => { void it('connects using ambient client', async () => { const existingClient = sqlite3Client({ fileName }); + await existingClient.connect(); const pool = sqlitePool({ connector: 'SQLite:sqlite3', diff --git a/src/packages/dumbo/src/storage/sqlite/sqlite3/connections/connection.ts b/src/packages/dumbo/src/storage/sqlite/sqlite3/connections/connection.ts index 654eeb5..bb36be3 100644 --- a/src/packages/dumbo/src/storage/sqlite/sqlite3/connections/connection.ts +++ b/src/packages/dumbo/src/storage/sqlite/sqlite3/connections/connection.ts @@ -20,11 +20,48 @@ export type ConnectionCheckResult = }; export const sqlite3Client = (options: SQLiteClientOptions): SQLiteClient => { - const db = new sqlite3.Database(options.fileName ?? InMemorySQLiteDatabase); + let db: sqlite3.Database; + + let isClosed = false; + + const connect: () => Promise = () => + db + ? Promise.resolve() // If db is already initialized, resolve immediately + : new Promise((resolve, reject) => { + try { + db = new sqlite3.Database( + options.fileName ?? + options.connectionString ?? + InMemorySQLiteDatabase, + sqlite3.OPEN_URI | sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE, + (err) => { + if (err) { + reject(err); + return; + } + }, + ); + db.run('PRAGMA journal_mode = WAL;', (err) => { + if (err) { + reject(err); + return; + } + + resolve(); + }); + } catch (error) { + reject(error as Error); + } + }); return { + connect, close: (): Promise => { - db.close(); + if (isClosed) { + return Promise.resolve(); + } + isClosed = true; + if (db) db.close(); return Promise.resolve(); }, command: (sql: string, params?: Parameters[]) => @@ -40,14 +77,18 @@ export const sqlite3Client = (options: SQLiteClientOptions): SQLiteClient => { }), query: (sql: string, params?: Parameters[]): Promise => new Promise((resolve, reject) => { - db.all(sql, params ?? [], (err: Error | null, result: T[]) => { - if (err) { - reject(err); - return; - } + try { + db.all(sql, params ?? [], (err: Error | null, result: T[]) => { + if (err) { + reject(err); + return; + } - resolve(result); - }); + resolve(result); + }); + } catch (error) { + reject(error as Error); + } }), querySingle: (sql: string, params?: Parameters[]): Promise => new Promise((resolve, reject) => {