diff --git a/AGENTS.md b/AGENTS.md index 4e0bf1e6..6c135f56 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1298,6 +1298,108 @@ mix ecto.migrate # Run migrations mix ecto.rollback # Rollback last migration ``` +#### RANDOM ROWID Support (libSQL Extension) + +For security and privacy, use RANDOM ROWID to generate pseudorandom row IDs instead of sequential integers: + +```elixir +# Create table with random row IDs (prevents ID enumeration attacks) +defmodule MyApp.Repo.Migrations.CreateSessions do + use Ecto.Migration + + def change do + create table(:sessions, options: [random_rowid: true]) do + add :token, :string, null: false + add :user_id, references(:users, on_delete: :delete_all) + add :expires_at, :utc_datetime + + timestamps() + end + + create unique_index(:sessions, [:token]) + end +end +``` + +**Benefits:** +- **Security**: Prevents ID enumeration attacks (guessing valid IDs) +- **Privacy**: Doesn't leak business metrics through sequential IDs +- **Unpredictability**: Row IDs are pseudorandom, not sequential + +**Usage:** +```elixir +# Basic usage +create table(:sessions, options: [random_rowid: true]) do + add :token, :string +end + +# With composite primary key +create table(:audit_log, options: [random_rowid: true]) do + add :user_id, :integer, primary_key: true + add :action_id, :integer, primary_key: true + add :timestamp, :integer +end + +# With IF NOT EXISTS +create_if_not_exists table(:sessions, options: [random_rowid: true]) do + add :token, :string +end +``` + +**Restrictions:** +- Mutually exclusive with WITHOUT ROWID (per libSQL specification) +- Mutually exclusive with AUTOINCREMENT (per libSQL specification) +- LibSQL extension - not available in standard SQLite + +**SQL Output:** +```sql +CREATE TABLE sessions (...) RANDOM ROWID +``` + +#### ALTER COLUMN Support (libSQL Extension) + +LibSQL supports modifying column attributes with ALTER COLUMN (not available in standard SQLite): + +```elixir +defmodule MyApp.Repo.Migrations.ModifyUserColumns do + use Ecto.Migration + + def change do + alter table(:users) do + # Change column type + modify :age, :string, default: "0" + + # Add NOT NULL constraint + modify :email, :string, null: false + + # Add DEFAULT value + modify :status, :string, default: "active" + + # Add foreign key reference + modify :team_id, references(:teams, on_delete: :nilify_all) + end + end +end +``` + +**Supported Modifications:** +- Type affinity changes (`:integer` → `:string`, etc.) +- NOT NULL constraints +- DEFAULT values +- CHECK constraints +- REFERENCES (foreign keys) + +**Important Notes:** +- Changes only apply to **new or updated rows** +- Existing data is **not revalidated** or modified +- This is a **libSQL extension** - not available in standard SQLite + +**SQL Output:** +```sql +ALTER TABLE users ALTER COLUMN age TO age TEXT DEFAULT '0' +ALTER TABLE users ALTER COLUMN email TO email TEXT NOT NULL +``` + ### Basic Queries #### Insert @@ -1719,33 +1821,61 @@ Ecto types map to SQLite types as follows: ### Ecto Migration Notes -Most Ecto migrations work perfectly. SQLite limitations: +Most Ecto migrations work perfectly. LibSQL provides extensions beyond standard SQLite: ```elixir -# ✅ SUPPORTED -create table(:users) -alter table(:users) do: add :field, :type -drop table(:users) -create index(:users, [:email]) -rename table(:old), to: table(:new) -rename table(:users), :old_field, to: :new_field +# ✅ FULLY SUPPORTED +create table(:users) # CREATE TABLE +create table(:sessions, options: [random_rowid: true]) # RANDOM ROWID (libSQL extension) +alter table(:users) do: add :field, :type # ADD COLUMN +alter table(:users) do: modify :field, :new_type # ALTER COLUMN (libSQL extension) +alter table(:users) do: remove :field # DROP COLUMN (libSQL/SQLite 3.35.0+) +drop table(:users) # DROP TABLE +create index(:users, [:email]) # CREATE INDEX +rename table(:old), to: table(:new) # RENAME TABLE +rename table(:users), :old_field, to: :new_field # RENAME COLUMN + +# ⚠️ LIBSQL EXTENSIONS (not in standard SQLite) +alter table(:users) do: modify :age, :string # ALTER COLUMN - libSQL only +create table(:sessions, options: [random_rowid: true]) # RANDOM ROWID - libSQL only +``` + +**Important Notes:** + +1. **ALTER COLUMN** is a libSQL extension (not available in standard SQLite) + - Supported operations: type changes, NOT NULL, DEFAULT, CHECK, REFERENCES + - Changes only apply to new/updated rows; existing data is not revalidated -# ❌ NOT SUPPORTED -alter table(:users) do: modify :field, :new_type # Can't change column type -alter table(:users) do: remove :field # Can't drop column (SQLite < 3.35.0) +2. **DROP COLUMN** requires SQLite 3.35.0+ or libSQL + - Cannot drop PRIMARY KEY columns, UNIQUE columns, or referenced columns -# Workaround: Recreate table +3. **RANDOM ROWID** is a libSQL extension for security/privacy + - Prevents ID enumeration attacks + - Mutually exclusive with WITHOUT ROWID and AUTOINCREMENT + +**Standard SQLite Workaround (if not using libSQL's ALTER COLUMN):** + +If you need to modify columns on standard SQLite (without libSQL's extensions), recreate the table: + +```elixir defmodule MyApp.Repo.Migrations.ChangeUserAge do use Ecto.Migration def up do create table(:users_new) do - # Define new schema + add :id, :integer, primary_key: true + add :name, :string + add :email, :string + add :age, :string # Changed from :integer + timestamps() end - execute "INSERT INTO users_new SELECT * FROM users" + execute "INSERT INTO users_new (id, name, email, age, inserted_at, updated_at) SELECT id, name, email, CAST(age AS TEXT), inserted_at, updated_at FROM users" drop table(:users) rename table(:users_new), to: table(:users) + + # Recreate indexes + create unique_index(:users, [:email]) end end ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ffe74cc..cf0c2ac2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,25 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Investigated but Not Supported - -- **Hooks Investigation**: Researched implementation of SQLite hooks (update hooks and authorizer hooks) for CDC and row-level security - - **Update Hooks (CDC)**: Cannot be implemented due to Rustler threading limitations - - SQLite's update hook runs on managed BEAM threads - - Rustler's `OwnedEnv::send_and_clear()` can ONLY be called from unmanaged threads - - Would cause panic: "send_and_clear: current thread is managed" - - **Authorizer Hooks (RLS)**: Cannot be implemented due to synchronous callback requirements - - Requires immediate synchronous response (Allow/Deny/Ignore) - - No safe way to block waiting for Elixir response from scheduler thread - - Would risk deadlocks with scheduler thread blocking - - **Result**: Both `add_update_hook/2`, `remove_update_hook/1`, and `add_authorizer/2` return `{:error, :unsupported}` - - **Alternatives provided**: Comprehensive documentation of alternative approaches: - - For CDC: Application-level events, database triggers, polling, Phoenix.Tracker - - For RLS: Application-level auth, database views, query rewriting, connection-level privileges - - See Rustler issue: https://github.com/rusterlium/rustler/issues/293 - ### Added +- **RANDOM ROWID Support (libSQL Extension)** + - Added support for libSQL's RANDOM ROWID table option to generate pseudorandom rowid values instead of consecutive integers + - **Security/Privacy Benefits**: Prevents ID enumeration attacks and leaking business metrics through sequential IDs + - **Usage**: Pass `options: [random_rowid: true]` to `create table()` in migrations + - **Example**: + ```elixir + create table(:sessions, options: [random_rowid: true]) do + add :token, :string + add :user_id, :integer + timestamps() + end + ``` + - **Compatibility**: Works with all table configurations (single PK, composite PK, IF NOT EXISTS) + - **Restrictions**: Mutually exclusive with WITHOUT ROWID and AUTOINCREMENT (per libSQL specification) + - **Validation**: Early validation of mutually exclusive options with clear error messages (connection.ex:386-407) + - Raises `ArgumentError` if RANDOM ROWID is combined with WITHOUT ROWID + - Raises `ArgumentError` if RANDOM ROWID is combined with AUTOINCREMENT on any column + - Prevents libSQL runtime errors by catching conflicts during migration compilation + - SQL output: `CREATE TABLE sessions (...) RANDOM ROWID` + - Added 7 comprehensive tests covering RANDOM ROWID with various configurations and validation scenarios + - Documentation: See [libSQL extensions guide](https://github.com/tursodatabase/libsql/blob/main/libsql-sqlite3/doc/libsql_extensions.md#random-rowid) + - **SQLite Extension Loading Support (`enable_extensions/2`, `load_ext/3`)** - Load SQLite extensions dynamically from shared library files - **Security-first design**: Extension loading disabled by default, must be explicitly enabled @@ -91,6 +96,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated documentation in README.md with examples for all encryption scenarios - See [Turso Encryption Documentation](https://docs.turso.tech/cloud/encryption) for key generation and requirements +### Clarifications + +- **ALTER TABLE ALTER COLUMN Support (Already Implemented)** + - **Fully supported** since v0.6.0 - libSQL's ALTER COLUMN extension for modifying column attributes + - **Capabilities**: Modify type affinity, NOT NULL, CHECK, DEFAULT, and REFERENCES constraints + - **Usage**: Use `:modify` in migrations as with other Ecto adapters + - **Example**: + ```elixir + alter table(:users) do + modify :age, :string, default: "0" # Change type and default + modify :email, :string, null: false # Add NOT NULL constraint + end + ``` + - **Important**: Changes only apply to new/updated rows; existing data is not revalidated + - **Implementation**: `lib/ecto/adapters/libsql/connection.ex:213-219` handles `:modify` changes + - SQL output: `ALTER TABLE users ALTER COLUMN age TO age TEXT DEFAULT '0'` + - This is a **libSQL extension** beyond standard SQLite (SQLite does not support ALTER COLUMN) + +### Investigated but Not Supported + +- **Hooks Investigation**: Researched implementation of SQLite hooks (update hooks and authorizer hooks) for CDC and row-level security + - **Update Hooks (CDC)**: Cannot be implemented due to Rustler threading limitations + - SQLite's update hook runs on managed BEAM threads + - Rustler's `OwnedEnv::send_and_clear()` can ONLY be called from unmanaged threads + - Would cause panic: "send_and_clear: current thread is managed" + - **Authorizer Hooks (RLS)**: Cannot be implemented due to synchronous callback requirements + - Requires immediate synchronous response (Allow/Deny/Ignore) + - No safe way to block waiting for Elixir response from scheduler thread + - Would risk deadlocks with scheduler thread blocking + - **Result**: Both `add_update_hook/2`, `remove_update_hook/1`, and `add_authorizer/2` return `{:error, :unsupported}` + - **Alternatives provided**: Comprehensive documentation of alternative approaches: + - For CDC: Application-level events, database triggers, polling, Phoenix.Tracker + - For RLS: Application-level auth, database views, query rewriting, connection-level privileges + - See Rustler issue: https://github.com/rusterlium/rustler/issues/293 + ## [0.8.1] - 2025-12-18 ### Fixed diff --git a/lib/ecto/adapters/libsql/connection.ex b/lib/ecto/adapters/libsql/connection.ex index 9880da20..3ca75121 100644 --- a/lib/ecto/adapters/libsql/connection.ex +++ b/lib/ecto/adapters/libsql/connection.ex @@ -184,10 +184,10 @@ defmodule Ecto.Adapters.LibSql.Connection do column_definitions = Enum.map_join(columns, ", ", &column_definition(&1, composite_pk)) - table_options = table_options(table, columns) + {table_constraints, table_suffix} = table_options(table, columns) [ - "CREATE TABLE#{if_not_exists} #{table_name} (#{column_definitions}#{table_options})" + "CREATE TABLE#{if_not_exists} #{table_name} (#{column_definitions}#{table_constraints})#{table_suffix}" ] end @@ -384,23 +384,51 @@ defmodule Ecto.Adapters.LibSql.Connection do defp column_default({:fragment, expr}), do: " DEFAULT #{expr}" defp table_options(table, columns) do + # Validate mutually exclusive options (per libSQL specification) + if table.options && Keyword.get(table.options, :random_rowid, false) do + # RANDOM ROWID is mutually exclusive with WITHOUT ROWID + if Keyword.get(table.options, :without_rowid, false) do + raise ArgumentError, + "RANDOM ROWID and WITHOUT ROWID are mutually exclusive options (per libSQL specification)" + end + + # RANDOM ROWID is mutually exclusive with AUTOINCREMENT on any column + autoincrement_column = + Enum.find(columns, fn {:add, _name, _type, opts} -> + Keyword.get(opts, :autoincrement, false) + end) + + if autoincrement_column do + {:add, col_name, _type, _opts} = autoincrement_column + + raise ArgumentError, + "RANDOM ROWID and AUTOINCREMENT (on column #{inspect(col_name)}) are mutually exclusive options (per libSQL specification)" + end + end + pk = Enum.filter(columns, fn {:add, _name, _type, opts} -> Keyword.get(opts, :primary_key, false) end) - cond do - length(pk) > 1 -> + # Composite primary key constraint (goes inside CREATE TABLE parentheses) + table_constraints = + if length(pk) > 1 do pk_names = Enum.map_join(pk, ", ", fn {:add, name, _type, _opts} -> quote_name(name) end) ", PRIMARY KEY (#{pk_names})" - - table.options -> - # Handle custom table options + else "" + end - true -> + # Table suffix options (go after closing parenthesis) + table_suffix = + if table.options && Keyword.get(table.options, :random_rowid, false) do + " RANDOM ROWID" + else "" - end + end + + {table_constraints, table_suffix} end ## Query Helpers diff --git a/test/ecto_connection_test.exs b/test/ecto_connection_test.exs index 5e0a59f6..4b575889 100644 --- a/test/ecto_connection_test.exs +++ b/test/ecto_connection_test.exs @@ -46,6 +46,113 @@ defmodule Ecto.Adapters.LibSql.ConnectionTest do assert sql =~ ~s[PRIMARY KEY ("user_id", "role_id")] end + test "creates table with RANDOM ROWID (libSQL extension)" do + table = %Table{name: :sessions, prefix: nil, options: [random_rowid: true]} + + columns = [ + {:add, :id, :id, [primary_key: true]}, + {:add, :token, :string, []} + ] + + [sql] = Connection.execute_ddl({:create, table, columns}) + + assert sql =~ ~s[CREATE TABLE "sessions"] + assert sql =~ ~s[RANDOM ROWID] + # Verify the RANDOM ROWID comes after the closing parenthesis + assert sql =~ ~r/\).*RANDOM ROWID/ + end + + test "creates table with RANDOM ROWID and composite primary key" do + table = %Table{name: :audit_log, prefix: nil, options: [random_rowid: true]} + + columns = [ + {:add, :user_id, :integer, [primary_key: true]}, + {:add, :action_id, :integer, [primary_key: true]}, + {:add, :timestamp, :integer, []} + ] + + [sql] = Connection.execute_ddl({:create, table, columns}) + + assert sql =~ ~s[PRIMARY KEY ("user_id", "action_id")] + assert sql =~ ~s[RANDOM ROWID] + # Ensure both composite PK and RANDOM ROWID are present + assert sql =~ ~r/PRIMARY KEY.*\).*RANDOM ROWID/ + end + + test "creates table without RANDOM ROWID when option is false" do + table = %Table{name: :users, prefix: nil, options: [random_rowid: false]} + + columns = [ + {:add, :id, :id, [primary_key: true]}, + {:add, :name, :string, []} + ] + + [sql] = Connection.execute_ddl({:create, table, columns}) + + refute sql =~ ~s[RANDOM ROWID] + end + + test "creates table without RANDOM ROWID when no options specified" do + table = %Table{name: :users, prefix: nil} + + columns = [ + {:add, :id, :id, [primary_key: true]}, + {:add, :name, :string, []} + ] + + [sql] = Connection.execute_ddl({:create, table, columns}) + + refute sql =~ ~s[RANDOM ROWID] + end + + test "creates table with RANDOM ROWID and IF NOT EXISTS" do + table = %Table{name: :sessions, prefix: nil, options: [random_rowid: true]} + + columns = [ + {:add, :id, :id, [primary_key: true]}, + {:add, :token, :string, []} + ] + + [sql] = Connection.execute_ddl({:create_if_not_exists, table, columns}) + + assert sql =~ ~s[CREATE TABLE IF NOT EXISTS "sessions"] + assert sql =~ ~s[RANDOM ROWID] + end + + test "raises error when RANDOM ROWID is combined with WITHOUT ROWID" do + table = %Table{ + name: :users, + prefix: nil, + options: [random_rowid: true, without_rowid: true] + } + + columns = [ + {:add, :id, :id, [primary_key: true]}, + {:add, :name, :string, []} + ] + + assert_raise ArgumentError, + "RANDOM ROWID and WITHOUT ROWID are mutually exclusive options (per libSQL specification)", + fn -> + Connection.execute_ddl({:create, table, columns}) + end + end + + test "raises error when RANDOM ROWID is combined with AUTOINCREMENT" do + table = %Table{name: :users, prefix: nil, options: [random_rowid: true]} + + columns = [ + {:add, :id, :id, [primary_key: true, autoincrement: true]}, + {:add, :name, :string, []} + ] + + assert_raise ArgumentError, + "RANDOM ROWID and AUTOINCREMENT (on column :id) are mutually exclusive options (per libSQL specification)", + fn -> + Connection.execute_ddl({:create, table, columns}) + end + end + test "creates table with NOT NULL constraint" do table = %Table{name: :users, prefix: nil}