Updating native lib to LMDB 1.0 line with new API support

README.md

@@ -189,6 +189,54 @@ using var db = tx.OpenDatabase(configuration: config);
 
 Reverse variants are available for most comparers (e.g., `ReverseSignedIntegerComparer`).
 
+## LMDB 1.0
+
+Starting with LightningDB 0.23.0, the bundled native library is from the LMDB 1.0 line.
+
+> **⚠️ Data migration required:** the LMDB 1.0 on-disk format is incompatible with 0.9.x.
+> Existing environments created by earlier LightningDB versions cannot be opened; migrate
+> them once by dumping with the 0.9 `mdb_dump` tool and loading with the 1.0 `mdb_load`
+> tool. Named-database (DBI) names are also stored differently (NUL-terminated) in 1.0.
+
+New capabilities exposed by LightningDB:
+
+### Encryption and checksums (opt-in)
+
+Every page can be encrypted and/or checksummed. LMDB itself ships no cipher — the
+application supplies one; LightningDB includes hardware-accelerated AES-256-GCM and
+SHA-256 implementations out of the box (custom `LightningCipher`/`LightningChecksum`
+implementations are also supported, and required on netstandard2.0 or browser-wasm):
+
+```csharp
+var config = new EnvironmentConfiguration
+{
+    Encryption = new EncryptionConfiguration(new AesGcmCipher(), key), // key: e.g. 32 random bytes
+    Checksum = new Sha256Checksum(), // independent of encryption; either can be used alone
+};
+using var env = new LightningEnvironment("path_to_your_database", config);
+env.Open();
+```
+
+The same cipher and key must be configured every time the environment is opened.
+
+### Two-phase commit
+
+`tx.Prepare()` performs the first phase of a two-phase commit; follow with `Commit()` or
+`Abort()`. If a remote participant fails after a local commit, the last committed
+transaction can be undone with `env.RollbackLastTransaction(txId)`.
+
+### Incremental backup
+
+`env.CopyTo(path)` remains the full-backup primitive; `env.IncrementalCopyTo(file, sinceTxnId)`
+dumps only pages newer than a previous backup's `env.Info.LastTransactionId`, and
+`env.LoadIncrementalFromStream(stream)` applies the dump to a restored copy.
+
+### Other additions
+
+- `EnvironmentConfiguration.PageSize` — set the database page size (power of 2, 512–65536) at creation time.
+- `EnvironmentOpenFlags.PreviousSnapshot` — open using the previous snapshot, recovering from a bad last transaction.
+- `TransactionBeginFlags.NoSync`/`NoMetaSync` are now honored per-transaction.
+
 ## Additional Resources
 
 For more detailed examples and advanced usage, refer to the unit tests in the [Lightning.NET](https://github.com/CoreyKaylor/Lightning.NET) repository. 

lmdb/compile-lmdb-macos.sh

@@ -4,26 +4,34 @@ if [ ! -d "lmdb" ]; then
   git clone https://git.openldap.org/openldap/openldap.git lmdb
 fi
 cd ./lmdb/libraries/liblmdb || exit
-git checkout LMDB_0.9.35
+# Pinned mdb.RE/1.0 (LMDB 1.0 release-engineering line): the LMDB_1.0.0 tag alone
+# lacked Windows compile fixes and post-release encryption/checksum/backup work
+# (ITS#10515, ITS#10518-10523, ITS#10538-10539, ITS#10542). RE/1.0 still reports
+# 1.0.0 but includes those fixes. Re-pin to LMDB_1.0.1 (or later) once tagged.
+git fetch origin mdb.RE/1.0
+git checkout -f 82fa96b63c0b03d39faf083170debef6626cbdce
+# ITS#10539 fixed most mingw win32 issues; one MAP() call still passes DWORD*
+# where NtMapViewOfSection wants SIZE_T* under mingw-w64. Drop once fixed upstream.
+git apply ../../../mingw-map-len-type.patch
 
 declare -A build_outputs
 declare -A supported_targets=(
-  [ios-arm64/native/liblmdb.dylib]="make CC='xcrun --sdk iphoneos --toolchain iphoneos clang -arch arm64' LDFLAGS='-s' XCFLAGS='-DNDEBUG'"
-  [iossimulator-arm64/native/liblmdb.dylib]="make CC='xcrun --sdk iphonesimulator --toolchain iphoneos clang -arch arm64' LDFLAGS='-s' XCFLAGS='-DNDEBUG'"
-  [iossimulator-x64/native/liblmdb.dylib]="make CC='xcrun --sdk iphonesimulator --toolchain iphoneos clang -arch x86_64' LDFLAGS='-s' XCFLAGS='-DNDEBUG'"
-  [osx-arm64/native/lmdb.dylib]="make LDFLAGS='-s' XCFLAGS='-DNDEBUG'"
-  [osx/native/lmdb.dylib]="make CC='clang -mmacosx-version-min=10.15 -arch x86_64' LDFLAGS='-s' XCFLAGS='-DNDEBUG'"
+  [ios-arm64/native/liblmdb.dylib]="make CC='xcrun --sdk iphoneos --toolchain iphoneos clang -arch arm64' LDFLAGS='-s' XCFLAGS='-DNDEBUG' VERSION_OPT='-Wl,-current_version,1.0'"
+  [iossimulator-arm64/native/liblmdb.dylib]="make CC='xcrun --sdk iphonesimulator --toolchain iphoneos clang -arch arm64' LDFLAGS='-s' XCFLAGS='-DNDEBUG' VERSION_OPT='-Wl,-current_version,1.0'"
+  [iossimulator-x64/native/liblmdb.dylib]="make CC='xcrun --sdk iphonesimulator --toolchain iphoneos clang -arch x86_64' LDFLAGS='-s' XCFLAGS='-DNDEBUG' VERSION_OPT='-Wl,-current_version,1.0'"
+  [osx-arm64/native/lmdb.dylib]="make LDFLAGS='-s' XCFLAGS='-DNDEBUG' VERSION_OPT='-Wl,-current_version,1.0'"
+  [osx/native/lmdb.dylib]="make CC='clang -mmacosx-version-min=10.15 -arch x86_64' LDFLAGS='-s' XCFLAGS='-DNDEBUG' VERSION_OPT='-Wl,-current_version,1.0'"
   [linux-arm/native/liblmdb.so]="docker run --mount type=bind,source=$(pwd),target=/lmdb --rm --platform=linux/arm/7 -w /lmdb gcc:latest make LDFLAGS='-s' XCFLAGS='-DNDEBUG'"
   [linux-arm64/native/liblmdb.so]="docker run --mount type=bind,source=$(pwd),target=/lmdb --rm --platform=linux/arm64 -w /lmdb gcc:latest make LDFLAGS='-s' XCFLAGS='-DNDEBUG'"
   [linux-x64/native/liblmdb.so]="docker run --mount type=bind,source=$(pwd),target=/lmdb --rm --platform=linux/amd64 -w /lmdb gcc:latest make LDFLAGS='-s' XCFLAGS='-DNDEBUG'"
-  [win-x64/native/lmdb.dll]="make CC='x86_64-w64-mingw32-gcc' AR='x86_64-w64-mingw32-gcc-ar' LDFLAGS='-s' XCFLAGS='-DNDEBUG'"
-  [win-x86/native/lmdb.dll]="make CC='i686-w64-mingw32-gcc' AR='i686-w64-mingw32-gcc-ar' LDFLAGS='-s' XCFLAGS='-DNDEBUG'"
-  [win-arm64/native/lmdb.dll]="docker run --mount type=bind,source='$(pwd)',target=/lmdb --rm -w /lmdb dockcross/windows-arm64 bash -c 'make CC=aarch64-w64-mingw32-gcc AR=aarch64-w64-mingw32-ar LDFLAGS=-s XCFLAGS=-DNDEBUG'"
+  [win-x64/native/lmdb.dll]="make CC='x86_64-w64-mingw32-gcc' AR='x86_64-w64-mingw32-gcc-ar' LDFLAGS='-s' XCFLAGS='-DNDEBUG' LDL= VERSION_OPT="
+  [win-x86/native/lmdb.dll]="make CC='i686-w64-mingw32-gcc' AR='i686-w64-mingw32-gcc-ar' LDFLAGS='-s' XCFLAGS='-DNDEBUG' LDL= VERSION_OPT="
+  [win-arm64/native/lmdb.dll]="docker run --mount type=bind,source='$(pwd)',target=/lmdb --rm -w /lmdb dockcross/windows-arm64 bash -c 'make CC=aarch64-w64-mingw32-gcc AR=aarch64-w64-mingw32-ar LDFLAGS=-s XCFLAGS=-DNDEBUG LDL= VERSION_OPT='"
   [android-arm64/native/liblmdb.so]="make CC=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/aarch64-linux-android21-clang AR=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-ar LDFLAGS='-s' XCFLAGS='-UMDB_USE_ROBUST -DMDB_USE_POSIX_MUTEX -DANDROID -DNDEBUG'"
   [android-arm/native/liblmdb.so]="make CC=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/armv7a-linux-androideabi21-clang AR=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-ar LDFLAGS='-s' XCFLAGS='-UMDB_USE_ROBUST -DMDB_USE_POSIX_MUTEX -DANDROID -DNDEBUG'"
   [android-x86/native/liblmdb.so]="make CC=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/i686-linux-android21-clang AR=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-ar LDFLAGS='-s' XCFLAGS='-UMDB_USE_ROBUST -DMDB_USE_POSIX_MUTEX -DANDROID -DNDEBUG'"
   [android-x64/native/liblmdb.so]="make CC=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/x86_64-linux-android21-clang AR=$NDK/toolchains/llvm/prebuilt/darwin-x86_64/bin/llvm-ar LDFLAGS='-s' XCFLAGS='-UMDB_USE_ROBUST -DMDB_USE_POSIX_MUTEX -DANDROID -DNDEBUG'"
-  [browser-wasm/native/liblmdb.wasm]="emmake make LDFLAGS='-s' XCFLAGS='-DNDEBUG'"
+  [browser-wasm/native/liblmdb.wasm]="emcc -O2 -pthread -fPIC -DNDEBUG -sSIDE_MODULE=1 -o liblmdb.so mdb.c midl.c module.c"
 )
 
 function compile_lib() {

lmdb/mingw-map-len-type.patch

@@ -0,0 +1,25 @@
+diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c
+index 3331511683..6c072b3895 100644
+--- a/libraries/liblmdb/mdb.c
++++ b/libraries/liblmdb/mdb.c
+@@ -11637,8 +11637,10 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
+ #if MDB_RPAGE_CACHE
+ #ifdef _WIN32
+ 	LARGE_INTEGER	off = {0};
++	SIZE_T	mlen;
+ #else
+ 	off_t	off = 0;
++	size_t	mlen;
+ #endif
+ #endif
+
+@@ -11708,7 +11710,8 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
+ 		w2 = (wsize > MAX_WRITE) ? MAX_WRITE : wsize;
+ #if MDB_RPAGE_CACHE
+ 		if (MDB_REMAPPING(env->me_flags)) {
+-			MAP(rc, env, ptr, w2, off);
++			mlen = w2;
++			MAP(rc, env, ptr, mlen, off);
+ 		}
+ #endif
+ 		DO_WRITE(rc, fd, ptr, w2, len);

src/LightningDB.Tests/DatabaseTests.cs

@@ -74,7 +74,8 @@ public void named_database_name_exists_in_master()
             using (var cursor = tx.CreateCursor(db))
             {
                 cursor.Next();
-                UTF8.GetString(cursor.GetCurrent().key.CopyToNewArray())
+                //LMDB 1.0 stores DBI names with their NUL terminator
+                UTF8.GetString(cursor.GetCurrent().key.CopyToNewArray()).TrimEnd('\0')
                     .ShouldBe("customdb");
             }
         }

src/LightningDB.Tests/EncryptionTests.cs

@@ -0,0 +1,146 @@
+using System;
+using System.Text;
+using Shouldly;
+
+namespace LightningDB.Tests;
+
+public class EncryptionTests : TestBase
+{
+    private static byte[] Key() => "0123456789abcdef0123456789abcdef"u8.ToArray();
+
+    private LightningEnvironment CreateEncryptedEnvironment(string path, byte[]? key = null) =>
+        CreateEnvironment(path, new EnvironmentConfiguration
+        {
+            Encryption = new EncryptionConfiguration(new AesGcmCipher(), key ?? Key())
+        });
+
+    public void can_write_and_read_in_encrypted_environment()
+    {
+        using var env = CreateEncryptedEnvironment(TempPath());
+        env.Open();
+
+        using (var tx = env.BeginTransaction())
+        using (var db = tx.OpenDatabase())
+        {
+            tx.Put(db, "key"u8.ToArray(), "value"u8.ToArray());
+            tx.Commit().ThrowOnError();
+        }
+
+        using (var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly))
+        using (var db = tx.OpenDatabase())
+        {
+            var (resultCode, _, value) = tx.Get(db, "key"u8.ToArray());
+            resultCode.ShouldBe(MDBResultCode.Success);
+            Encoding.UTF8.GetString(value.CopyToNewArray()).ShouldBe("value");
+        }
+    }
+
+    public void encrypted_environment_survives_reopen_with_same_key()
+    {
+        var path = TempPath();
+        using (var env = CreateEncryptedEnvironment(path))
+        {
+            env.Open();
+            using var tx = env.BeginTransaction();
+            using var db = tx.OpenDatabase();
+            tx.Put(db, "persisted"u8.ToArray(), "still here"u8.ToArray());
+            tx.Commit().ThrowOnError();
+        }
+
+        using (var env = CreateEncryptedEnvironment(path))
+        {
+            env.Open();
+            using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly);
+            using var db = tx.OpenDatabase();
+            var (resultCode, _, value) = tx.Get(db, "persisted"u8.ToArray());
+            resultCode.ShouldBe(MDBResultCode.Success);
+            Encoding.UTF8.GetString(value.CopyToNewArray()).ShouldBe("still here");
+        }
+    }
+
+    public void opening_encrypted_environment_with_wrong_key_fails()
+    {
+        var path = TempPath();
+        using (var env = CreateEncryptedEnvironment(path))
+        {
+            env.Open();
+            using var tx = env.BeginTransaction();
+            using var db = tx.OpenDatabase();
+            tx.Put(db, "secret"u8.ToArray(), "data"u8.ToArray());
+            tx.Commit().ThrowOnError();
+        }
+
+        var wrongKey = "ffffffffffffffffffffffffffffffff"u8.ToArray();
+        using (var env = CreateEncryptedEnvironment(path, wrongKey))
+        {
+            //data pages decrypt lazily, so the failure surfaces on first read
+            //rather than at open
+            try
+            {
+                env.Open();
+                using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly);
+                using var db = tx.OpenDatabase();
+                var (resultCode, _, _) = tx.Get(db, "secret"u8.ToArray());
+                resultCode.ShouldNotBe(MDBResultCode.Success);
+            }
+            catch (LightningException)
+            {
+            }
+        }
+    }
+
+    public void opening_encrypted_environment_without_cipher_fails()
+    {
+        var path = TempPath();
+        using (var env = CreateEncryptedEnvironment(path))
+        {
+            env.Open();
+            using var tx = env.BeginTransaction();
+            using var db = tx.OpenDatabase();
+            tx.Put(db, "secret"u8.ToArray(), "data"u8.ToArray());
+            tx.Commit().ThrowOnError();
+        }
+
+        using (var env = CreateEnvironment(path))
+        {
+            Should.Throw<Exception>(() =>
+            {
+                env.Open();
+                using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly);
+                using var db = tx.OpenDatabase();
+                tx.Get(db, "secret"u8.ToArray());
+            });
+        }
+    }
+
+    public void encrypted_environment_reports_encrypted_flag()
+    {
+        using var env = CreateEncryptedEnvironment(TempPath());
+        env.Open();
+        env.Flags.HasFlag(EnvironmentOpenFlags.Encrypted).ShouldBeTrue();
+    }
+
+    public void checksum_environment_round_trip()
+    {
+        var path = TempPath();
+        var config = new EnvironmentConfiguration { Checksum = new Sha256Checksum() };
+        using (var env = CreateEnvironment(path, config))
+        {
+            env.Open();
+            using var tx = env.BeginTransaction();
+            using var db = tx.OpenDatabase();
+            tx.Put(db, "summed"u8.ToArray(), "verified"u8.ToArray());
+            tx.Commit().ThrowOnError();
+        }
+
+        using (var env = CreateEnvironment(path, new EnvironmentConfiguration { Checksum = new Sha256Checksum() }))
+        {
+            env.Open();
+            using var tx = env.BeginTransaction(TransactionBeginFlags.ReadOnly);
+            using var db = tx.OpenDatabase();
+            var (resultCode, _, value) = tx.Get(db, "summed"u8.ToArray());
+            resultCode.ShouldBe(MDBResultCode.Success);
+            Encoding.UTF8.GetString(value.CopyToNewArray()).ShouldBe("verified");
+        }
+    }
+}