Merge pull request #875 from drizzle-team/neon-serverless

Neon serverless

changelogs/drizzle-orm/0.27.1.md

@@ -0,0 +1,13 @@
+- 🎉 Added support for [Neon HTTP driver](https://neon.tech/docs/serverless/serverless-driver)
+
+```typescript
+import { neon, neonConfig } from '@neondatabase/serverless';
+import { drizzle } from 'drizzle-orm/neon-http';
+
+neonConfig.fetchConnectionCache = true;
+
+const sql = neon(process.env.DRIZZLE_DATABASE_URL!);
+const db = drizzle(sql);
+
+db.select(...)
+```

drizzle-orm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "drizzle-orm",
-  "version": "0.27.0",
+  "version": "0.27.1",
   "description": "Drizzle ORM package for SQL databases",
   "type": "module",
   "scripts": {
@@ -126,7 +126,7 @@
     "@aws-sdk/client-rds-data": "^3.344.0",
     "@cloudflare/workers-types": "^4.20230518.0",
     "@libsql/client": "^0.1.6",
-    "@neondatabase/serverless": "^0.4.9",
+    "@neondatabase/serverless": "^0.4.24",
     "@opentelemetry/api": "^1.4.1",
     "@originjs/vite-plugin-commonjs": "^1.0.3",
     "@planetscale/database": "^1.7.0",

drizzle-orm/rollup.common.ts

@@ -19,6 +19,8 @@ export const entries = [
 	'mysql2/migrator',
 	'neon-serverless/index',
 	'neon-serverless/migrator',
+	'neon-http/index',
+	'neon-http/migrator',
 	'node-postgres/index',
 	'node-postgres/migrator',
 	'pg-core/index',

drizzle-orm/src/neon-http/driver.ts

@@ -0,0 +1,77 @@
+import { types } from '@neondatabase/serverless';
+import { entityKind } from '~/entity';
+import type { Logger } from '~/logger';
+import { DefaultLogger } from '~/logger';
+import { PgDatabase } from '~/pg-core/db';
+import { PgDialect } from '~/pg-core/dialect';
+import {
+	createTableRelationsHelpers,
+	extractTablesRelationalConfig,
+	type RelationalSchemaConfig,
+	type TablesRelationalConfig,
+} from '~/relations';
+import { type DrizzleConfig } from '~/utils';
+import { type NeonHttpClient, type NeonHttpQueryResultHKT, NeonHttpSession } from './session';
+
+export interface NeonDriverOptions {
+	logger?: Logger;
+}
+
+export class NeonHttpDriver {
+	static readonly [entityKind]: string = 'NeonDriver';
+
+	constructor(
+		private client: NeonHttpClient,
+		private dialect: PgDialect,
+		private options: NeonDriverOptions = {},
+	) {
+		this.initMappers();
+	}
+
+	createSession(
+		schema: RelationalSchemaConfig<TablesRelationalConfig> | undefined,
+	): NeonHttpSession<Record<string, unknown>, TablesRelationalConfig> {
+		return new NeonHttpSession(this.client, this.dialect, schema, { logger: this.options.logger });
+	}
+
+	initMappers() {
+		types.setTypeParser(types.builtins.TIMESTAMPTZ, (val) => val);
+		types.setTypeParser(types.builtins.TIMESTAMP, (val) => val);
+		types.setTypeParser(types.builtins.DATE, (val) => val);
+	}
+}
+
+export type NeonHttpDatabase<
+	TSchema extends Record<string, unknown> = Record<string, never>,
+> = PgDatabase<NeonHttpQueryResultHKT, TSchema>;
+
+export function drizzle<TSchema extends Record<string, unknown> = Record<string, never>>(
+	client: NeonHttpClient,
+	config: DrizzleConfig<TSchema> = {},
+): NeonHttpDatabase<TSchema> {
+	const dialect = new PgDialect();
+	let logger;
+	if (config.logger === true) {
+		logger = new DefaultLogger();
+	} else if (config.logger !== false) {
+		logger = config.logger;
+	}
+
+	let schema: RelationalSchemaConfig<TablesRelationalConfig> | undefined;
+	if (config.schema) {
+		const tablesConfig = extractTablesRelationalConfig(
+			config.schema,
+			createTableRelationsHelpers,
+		);
+		schema = {
+			fullSchema: config.schema,
+			schema: tablesConfig.tables,
+			tableNamesMap: tablesConfig.tableNamesMap,
+		};
+	}
+
+	const driver = new NeonHttpDriver(client, dialect, { logger });
+	const session = driver.createSession(schema);
+
+	return new PgDatabase(dialect, session, schema) as NeonHttpDatabase<TSchema>;
+}

drizzle-orm/src/neon-http/index.ts

@@ -0,0 +1,2 @@
+export * from './driver';
+export * from './session';

drizzle-orm/src/neon-http/migrator.ts

@@ -0,0 +1,53 @@
+import type { MigrationConfig } from '~/migrator';
+import { readMigrationFiles } from '~/migrator';
+import { type SQL, sql } from '~/sql';
+import { type NeonHttpDatabase } from './driver';
+
+/**
+ * This function reads migrationFolder and execute each unapplied migration and mark it as executed in database
+ *
+ * NOTE: The Neon HTTP driver does not support transactions. This means that if any part of a migration fails,
+ * no rollback will be executed. Currently, you will need to handle unsuccessful migration yourself.
+ * @param db - drizzle db instance
+ * @param config - path to migration folder generated by drizzle-kit
+ */
+export async function migrate<TSchema extends Record<string, unknown>>(
+	db: NeonHttpDatabase<TSchema>,
+	config: string | MigrationConfig,
+) {
+	const migrations = readMigrationFiles(config);
+	const migrationTableCreate = sql`
+		CREATE TABLE IF NOT EXISTS "drizzle"."__drizzle_migrations" (
+			id SERIAL PRIMARY KEY,
+			hash text NOT NULL,
+			created_at bigint
+		)
+	`;
+	await db.session.execute(sql`CREATE SCHEMA IF NOT EXISTS "drizzle"`);
+	await db.session.execute(migrationTableCreate);
+
+	const dbMigrations = await db.session.all<{ id: number; hash: string; created_at: string }>(
+		sql`select id, hash, created_at from "drizzle"."__drizzle_migrations" order by created_at desc limit 1`,
+	);
+
+	const lastDbMigration = dbMigrations[0];
+	const rowsToInsert: SQL[] = [];
+	for await (const migration of migrations) {
+		if (
+			!lastDbMigration
+			|| Number(lastDbMigration.created_at) < migration.folderMillis
+		) {
+			for (const stmt of migration.sql) {
+				await db.session.execute(sql.raw(stmt));
+			}
+
+			rowsToInsert.push(
+				sql`insert into "drizzle"."__drizzle_migrations" ("hash", "created_at") values(${migration.hash}, ${migration.folderMillis})`
+			)
+		}
+	}
+
+	for await (const rowToInsert of rowsToInsert) {
+		await db.session.execute(rowToInsert);
+	}
+}

drizzle-orm/src/neon-http/session.ts

@@ -0,0 +1,164 @@
+import { type FullQueryResults, type QueryRows } from '@neondatabase/serverless';
+import { entityKind } from '~/entity';
+import type { Logger } from '~/logger';
+import { NoopLogger } from '~/logger';
+import { PgTransaction } from '~/pg-core';
+import type { PgDialect } from '~/pg-core/dialect';
+import type { SelectedFieldsOrdered } from '~/pg-core/query-builders/select.types';
+import type { PgTransactionConfig, PreparedQueryConfig, QueryResultHKT } from '~/pg-core/session';
+import { PgSession, PreparedQuery } from '~/pg-core/session';
+import { type RelationalSchemaConfig, type TablesRelationalConfig } from '~/relations';
+import { fillPlaceholders, type Query } from '~/sql';
+import { type Assume, mapResultRow } from '~/utils';
+
+export type NeonHttpClient = {
+	<A extends boolean = false, F extends boolean = true>(
+		strings: string,
+		params?: any[],
+		mode?: { arrayMode?: A; fullResults?: F },
+	): Promise<
+		F extends true ? FullQueryResults<A> : QueryRows<A>
+	>;
+};
+
+export class NeonHttpPreparedQuery<T extends PreparedQueryConfig> extends PreparedQuery<T> {
+	static readonly [entityKind]: string = 'NeonHttpPreparedQuery';
+
+	private rawQuery: { arrayMode?: false, fullResults?: true };
+	private query: { arrayMode?: true, fullResults?: true };
+
+	constructor(
+		private client: NeonHttpClient,
+		private queryString: string,
+		private params: unknown[],
+		private logger: Logger,
+		private fields: SelectedFieldsOrdered | undefined,
+		private name: string | undefined,
+		private customResultMapper?: (rows: unknown[][]) => T['execute'],
+	) {
+		super();
+		this.rawQuery = {
+			arrayMode: false,
+			fullResults: true,
+		};
+		this.query = { arrayMode: true, fullResults: true };
+	}
+
+	async execute(placeholderValues: Record<string, unknown> | undefined = {}): Promise<T['execute']> {
+		const params = fillPlaceholders(this.params, placeholderValues);
+
+		this.logger.logQuery(this.queryString, params);
+
+		const { fields, client, queryString, query, rawQuery, joinsNotNullableMap, customResultMapper } = this;
+		if (!fields && !customResultMapper) {
+			return client(queryString, params, rawQuery);
+		}
+
+		const result = await client(queryString, params, query);
+
+		return customResultMapper
+			? customResultMapper(result.rows as unknown[][])
+			: result.rows.map((row) => mapResultRow<T['execute']>(fields!, row as unknown[], joinsNotNullableMap));
+	}
+
+	all(placeholderValues: Record<string, unknown> | undefined = {}): Promise<T['all']> {
+		const params = fillPlaceholders(this.params, placeholderValues);
+		this.logger.logQuery(this.queryString, params);
+		return this.client(this.queryString, params, this.rawQuery).then((result) => result.rows);
+	}
+
+	values(placeholderValues: Record<string, unknown> | undefined = {}): Promise<T['values']> {
+		const params = fillPlaceholders(this.params, placeholderValues);
+		this.logger.logQuery(this.queryString, params);
+		return this.client(this.queryString, params).then((result) => result.rows);
+	}
+}
+
+export interface NeonHttpSessionOptions {
+	logger?: Logger;
+}
+
+export class NeonHttpSession<
+	TFullSchema extends Record<string, unknown>,
+	TSchema extends TablesRelationalConfig,
+> extends PgSession<NeonHttpQueryResultHKT, TFullSchema, TSchema> {
+	static readonly [entityKind]: string = 'NeonHttpSession';
+
+	private logger: Logger;
+
+	constructor(
+		private client: NeonHttpClient,
+		dialect: PgDialect,
+		private schema: RelationalSchemaConfig<TSchema> | undefined,
+		private options: NeonHttpSessionOptions = {},
+	) {
+		super(dialect);
+		this.logger = options.logger ?? new NoopLogger();
+	}
+
+	prepareQuery<T extends PreparedQueryConfig = PreparedQueryConfig>(
+		query: Query,
+		fields: SelectedFieldsOrdered | undefined,
+		name: string | undefined,
+		customResultMapper?: (rows: unknown[][]) => T['execute'],
+	): PreparedQuery<T> {
+		return new NeonHttpPreparedQuery(
+			this.client,
+			query.sql,
+			query.params,
+			this.logger,
+			fields,
+			name,
+			customResultMapper,
+		);
+	}
+
+	// change return type to QueryRows<true>
+	async query(query: string, params: unknown[]): Promise<FullQueryResults<true>> {
+		this.logger.logQuery(query, params);
+		const result = await this.client(query, params, { arrayMode: true });
+		return result;
+	}
+
+	// change return type to QueryRows<false>
+	async queryObjects(
+		query: string,
+		params: unknown[],
+	): Promise<FullQueryResults<false>> {
+		return this.client(query, params);
+	}
+
+	override async transaction<T>(
+		_transaction: (tx: NeonTransaction<TFullSchema, TSchema>) => Promise<T>,
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		_config: PgTransactionConfig = {},
+	): Promise<T> {
+		throw new Error('No transactions support in neon-http driver');
+	}
+}
+
+export class NeonTransaction<
+	TFullSchema extends Record<string, unknown>,
+	TSchema extends TablesRelationalConfig,
+> extends PgTransaction<NeonHttpQueryResultHKT, TFullSchema, TSchema> {
+	static readonly [entityKind]: string = 'NeonHttpTransaction';
+
+	override async transaction<T>(_transaction: (tx: NeonTransaction<TFullSchema, TSchema>) => Promise<T>): Promise<T> {
+		throw new Error('No transactions support in neon-http driver');
+		// const savepointName = `sp${this.nestedIndex + 1}`;
+		// const tx = new NeonTransaction(this.dialect, this.session, this.schema, this.nestedIndex + 1);
+		// await tx.execute(sql.raw(`savepoint ${savepointName}`));
+		// try {
+		// 	const result = await transaction(tx);
+		// 	await tx.execute(sql.raw(`release savepoint ${savepointName}`));
+		// 	return result;
+		// } catch (e) {
+		// 	await tx.execute(sql.raw(`rollback to savepoint ${savepointName}`));
+		// 	throw e;
+		// }
+	}
+}
+
+export interface NeonHttpQueryResultHKT extends QueryResultHKT {
+	type: FullQueryResults<Assume<this['row'], boolean>>;
+}

integration-tests/drizzle2/pg/0000_puzzling_flatman.sql

@@ -34,10 +34,4 @@ CREATE TABLE IF NOT EXISTS "all_columns" (
 	"datedef" date DEFAULT now(),
 	"interval" interval,
 	"intervaldef" interval DEFAULT '10 days'
-);
---> statement-breakpoint
-CREATE TABLE IF NOT EXISTS "users12" (
-	"id" serial PRIMARY KEY NOT NULL,
-	"name" text NOT NULL,
-	"email" text NOT NULL
-);
+);

integration-tests/drizzle2/pg/0001_test.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS "users12" (
+	"id" serial PRIMARY KEY NOT NULL,
+	"name" text NOT NULL,
+	"email" text NOT NULL
+);

integration-tests/drizzle2/pg/meta/_journal.json

@@ -8,6 +8,13 @@
       "when": 1680271923328,
       "tag": "0000_puzzling_flatman",
       "breakpoints": true
+    },
+    {
+      "idx": 1,
+      "version": "5",
+      "when": 1680271923329,
+      "tag": "0001_test",
+      "breakpoints": true
     }
   ]
 }

integration-tests/package.json

@@ -14,6 +14,7 @@
       "!tests/imports.test.mjs",
       "!tests/imports.test.cjs",
       "!tests/awsdatapi.alltypes.test.ts",
+      "!tests/neon-http.test.ts",
       "!tests/awsdatapi.test.ts",
       "!tests/planetscale-serverless/**/*.ts",
       "!tests/bun/**/*",
@@ -32,6 +33,7 @@
   "license": "Apache-2.0",
   "private": true,
   "devDependencies": {
+    "@neondatabase/serverless": "0.4.24",
     "@originjs/vite-plugin-commonjs": "^1.0.3",
     "@types/axios": "^0.14.0",
     "@types/better-sqlite3": "^7.6.4",