From 72e17fd898eb177e4c782a12e97d2638dec0af42 Mon Sep 17 00:00:00 2001 From: uclaros Date: Sat, 8 Aug 2026 18:06:01 +0300 Subject: [PATCH 1/5] Don't set SQLITE_JOURNAL=DELETE QGIS will now enable accessing geopackages in WAL mode --- app/main.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/main.cpp b/app/main.cpp index 3057bd5da..1545a998e 100644 --- a/app/main.cpp +++ b/app/main.cpp @@ -729,11 +729,6 @@ int main( int argc, char *argv[] ) QQuickStyle::setStyle( "Basic" ); initDeclarative(); - // QGIS environment variables to set - // OGR_SQLITE_JOURNAL is set to DELETE to avoid working with WAL files - // and properly close connection after writting changes to gpkg. - qputenv( "OGR_SQLITE_JOURNAL", "DELETE" ); - // Register to QQmlEngine engine.rootContext()->setContextProperty( "__notificationModel", ¬ificationModel ); From 46f1aeff4037bdfc2911ae3d15aa368d46d98ceb Mon Sep 17 00:00:00 2001 From: uclaros Date: Sat, 8 Aug 2026 18:07:03 +0300 Subject: [PATCH 2/5] Reorder helper functions in testmerginapi.cpp --- app/test/testmerginapi.cpp | 500 +++++++++++++++++++------------------ 1 file changed, 251 insertions(+), 249 deletions(-) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index 2b5e0c91d..d46b5335b 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -115,6 +115,257 @@ void TestMerginApi::cleanupTestCase() // do not remove project data locally - keep them around so it's possible to inspect their final state if needed } +//////// HELPER FUNCTIONS //////// + +MerginProjectsList TestMerginApi::getProjectList( QString tag ) +{ + QSignalSpy spy( mApi, &MerginApi::listProjectsFinished ); + mApi->listProjects( QString(), tag ); + spy.wait( TestUtils::SHORT_REPLY ); + + return projectListFromSpy( spy ); +} + +MerginProjectsList TestMerginApi::projectListFromSpy( QSignalSpy &spy ) +{ + MerginProjectsList projects; + + if ( !spy.isEmpty() ) + { + QList response = spy.takeFirst(); + + // get projects emited from MerginAPI, it is first argument in listProjectsFinished signal + if ( response.length() > 0 ) + projects = qvariant_cast( response.at( 0 ) ); + } + return projects; +} + +int TestMerginApi::serverVersionFromSpy( QSignalSpy &spy ) +{ + int serverVersion = -1; + + if ( !spy.isEmpty() ) + { + QList response = spy.takeFirst(); + + // get version number emited from MerginApi::syncProjectFinished, it is third argument + if ( response.length() >= 4 ) + serverVersion = response.at( 3 ).toInt(); + } + return serverVersion; +} + +void TestMerginApi::createRemoteProject( MerginApi *api, const QString &projectNamespace, const QString &projectName, const QString &sourcePath, bool force ) +{ + if ( force ) + { + deleteRemoteProjectNow( api, projectNamespace, projectName ); + } + + // create a project + QSignalSpy spy( api, &MerginApi::projectCreated ); + api->createProject( projectNamespace, projectName, true ); + QVERIFY( spy.wait( TestUtils::SHORT_REPLY ) ); + QCOMPARE( spy.count(), 1 ); + QCOMPARE( spy.takeFirst().at( 1 ).toBool(), true ); + + // Copy data + QString projectDir = api->projectsPath() + "/" + projectName + "/"; + InputUtils::cpDir( sourcePath, projectDir ); + + // make MerginApi aware of the project and its directory + api->localProjectsManager().addMerginProject( projectDir, projectNamespace, projectName ); + + // Upload data + QSignalSpy spy3( api, &MerginApi::syncProjectFinished ); + api->pushProject( projectNamespace, projectName ); + QVERIFY( spy3.wait( TestUtils::LONG_REPLY ) ); + QCOMPARE( spy3.count(), 1 ); + QList arguments = spy3.takeFirst(); + int version = arguments.at( 2 ).toInt(); + QCOMPARE( version, 1 ); + + // Remove the whole project + QDir( projectDir ).removeRecursively(); + QFileInfo info( projectDir ); + QDir dir( projectDir ); + QCOMPARE( info.size(), 0 ); + QVERIFY( dir.isEmpty() ); + + api->localProjectsManager().removeLocalProject( MerginApi::getFullProjectName( projectNamespace, projectName ) ); + + QCOMPARE( QFileInfo( projectDir ).size(), 0 ); + QVERIFY( QDir( projectDir ).isEmpty() ); +} + +QString TestMerginApi::projectIdFromProjectFullName( MerginApi *api, const QString &projectNamespace, const QString &projectName ) +{ + QString ret; + if ( !api->validateAuth() || api->mApiVersionStatus != MerginApiStatus::OK || api->mServerType == MerginServerType::OLD ) + { + return ret; + } + + QString projectFullName = api->getFullProjectName( projectNamespace, projectName ); + + QNetworkReply *r = api->getProjectInfo( projectFullName ); + Q_ASSERT( r ); + QSignalSpy spy( r, &QNetworkReply::finished ); + spy.wait( TestUtils::SHORT_REPLY ); + + if ( r->error() == QNetworkReply::NoError ) + { + QByteArray data = r->readAll(); + MerginProjectMetadata serverProject = MerginProjectMetadata::fromJson( data ); + ret = serverProject.projectId; + } + else + { + qDebug() << "Project " << projectFullName << " probably does not exists on remote server"; + } + + r->deleteLater(); + return ret; +} + +void TestMerginApi::deleteRemoteProjectNow( MerginApi *api, const QString &projectNamespace, const QString &projectName ) +{ + if ( !api->validateAuth() || api->mApiVersionStatus != MerginApiStatus::OK || api->mServerType == MerginServerType::OLD ) + { + return; + } + + QString projectId = projectIdFromProjectFullName( api, projectNamespace, projectName ); + if ( projectId.isEmpty() ) + { + // probably no such project exist on server + return; + } + + QNetworkRequest request = api->getDefaultRequest(); + QUrl url( api->mApiRoot + QStringLiteral( "/v2/projects/%1" ).arg( projectId ) ); + request.setUrl( url ); + qDebug() << "Trying to delete project " << projectName << ", id: " << projectId << " (" << url << ")"; + QNetworkReply *r = api->mManager->deleteResource( request ); + QSignalSpy spy( r, &QNetworkReply::finished ); + spy.wait( TestUtils::SHORT_REPLY ); + + QCOMPARE( r->error(), QNetworkReply::NoError ); + r->deleteLater(); +} + + +void TestMerginApi::deleteLocalProject( MerginApi *api, const QString &projectNamespace, const QString &projectName ) +{ + LocalProject project = api->getLocalProject( MerginApi::getFullProjectName( projectNamespace, projectName ) ); + QVERIFY( project.isValid() ); + QVERIFY( project.projectDir.startsWith( api->projectsPath() ) ); // just to make sure we don't delete something wrong (-: + + api->localProjectsManager().removeLocalProject( project.id() ); +} + +void TestMerginApi::deleteLocalDir( MerginApi *api, const QString &dirPath ) +{ + QDir dir( api->projectsPath() + "/" + dirPath ); + QVERIFY( dir.removeRecursively() ); +} + +void TestMerginApi::downloadRemoteProject( MerginApi *api, const QString &projectNamespace, const QString &projectName ) +{ + int serverVersion; + downloadRemoteProject( api, projectNamespace, projectName, serverVersion ); +} + +void TestMerginApi::downloadRemoteProject( MerginApi *api, const QString &projectNamespace, const QString &projectName, int &serverVersion ) +{ + QSignalSpy spy( api, &MerginApi::syncProjectFinished ); + api->pullProject( projectNamespace, projectName ); + QCOMPARE( api->transactions().count(), 1 ); + QVERIFY( spy.wait( TestUtils::LONG_REPLY * 5 ) ); + serverVersion = serverVersionFromSpy( spy ); +} + +void TestMerginApi::uploadRemoteProject( MerginApi *api, const QString &projectNamespace, const QString &projectName ) +{ + int serverVersion; + uploadRemoteProject( api, projectNamespace, projectName, serverVersion ); +} + +void TestMerginApi::uploadRemoteProject( MerginApi *api, const QString &projectNamespace, const QString &projectName, int &serverVersion ) +{ + api->pushProject( projectNamespace, projectName ); + QSignalSpy spy( api, &MerginApi::syncProjectFinished ); + QVERIFY( spy.wait( TestUtils::LONG_REPLY * 30 ) ); + QCOMPARE( spy.count(), 1 ); + serverVersion = serverVersionFromSpy( spy ); +} + +void TestMerginApi::writeFileContent( const QString &filename, const QByteArray &data ) +{ + QFile f( filename ); + bool ok = f.open( QIODeviceBase::WriteOnly ); + Q_ASSERT( ok ); + f.write( data ); + f.flush(); + f.close(); +} + +QByteArray TestMerginApi::readFileContent( const QString &filename ) +{ + QFile f( filename ); + if ( !f.exists() ) + { + qDebug() << "Filename " << filename << " does not exist"; + Q_ASSERT( false ); + } + bool ok = f.open( QIODeviceBase::ReadOnly ); + Q_ASSERT( ok ); + QByteArray data = f.readAll(); + f.close(); + return data; +} + +void TestMerginApi::createLocalProject( const QString projectDir ) +{ + QDir().mkdir( projectDir ); + bool r0 = QFile::copy( mTestDataPath + "/diff_project/base.gpkg", projectDir + "/base.gpkg" ); + + QVERIFY( r0 ); +} + +bool TestMerginApi::createJsonFile( const QString &path, const QVariantMap ¶ms ) +{ + QJsonObject json = QJsonObject::fromVariantMap( params ); + QJsonDocument doc( json ); + QByteArray data = doc.toJson(); + + writeFileContent( path, data ); + + QFile config( path ); + return config.exists(); +} + +void TestMerginApi::refreshProjectsModel( const ProjectsModel::ProjectModelTypes modelType ) +{ + if ( modelType == ProjectsModel::LocalProjectsModel ) + { + QSignalSpy spy( mApi, &MerginApi::listProjectsByNameFinished ); + mLocalProjectsModel->listProjects(); + QVERIFY( spy.wait( TestUtils::SHORT_REPLY ) ); + QCOMPARE( spy.count(), 1 ); + } + else if ( modelType == ProjectsModel::WorkspaceProjectsModel ) + { + QSignalSpy spy( mApi, &MerginApi::listProjectsFinished ); + mWorkspaceProjectsModel->listProjects(); + QVERIFY( spy.wait( TestUtils::SHORT_REPLY ) ); + QCOMPARE( spy.count(), 1 ); + } +} + +//////// ACTUAL TEST CASES //////// + void TestMerginApi::testListProject() { QString projectName = "testListProject"; @@ -2592,255 +2843,6 @@ void TestMerginApi::testExcludeFromSync() QVERIFY( mApi->excludeFromSync( selectiveSyncDir + "/image.jpg", config ) ); } -//////// HELPER FUNCTIONS //////// - -MerginProjectsList TestMerginApi::getProjectList( QString tag ) -{ - QSignalSpy spy( mApi, &MerginApi::listProjectsFinished ); - mApi->listProjects( QString(), tag ); - spy.wait( TestUtils::SHORT_REPLY ); - - return projectListFromSpy( spy ); -} - -MerginProjectsList TestMerginApi::projectListFromSpy( QSignalSpy &spy ) -{ - MerginProjectsList projects; - - if ( !spy.isEmpty() ) - { - QList response = spy.takeFirst(); - - // get projects emited from MerginAPI, it is first argument in listProjectsFinished signal - if ( response.length() > 0 ) - projects = qvariant_cast( response.at( 0 ) ); - } - return projects; -} - -int TestMerginApi::serverVersionFromSpy( QSignalSpy &spy ) -{ - int serverVersion = -1; - - if ( !spy.isEmpty() ) - { - QList response = spy.takeFirst(); - - // get version number emited from MerginApi::syncProjectFinished, it is third argument - if ( response.length() >= 4 ) - serverVersion = response.at( 3 ).toInt(); - } - return serverVersion; -} - -void TestMerginApi::createRemoteProject( MerginApi *api, const QString &projectNamespace, const QString &projectName, const QString &sourcePath, bool force ) -{ - if ( force ) - { - deleteRemoteProjectNow( api, projectNamespace, projectName ); - } - - // create a project - QSignalSpy spy( api, &MerginApi::projectCreated ); - api->createProject( projectNamespace, projectName, true ); - QVERIFY( spy.wait( TestUtils::SHORT_REPLY ) ); - QCOMPARE( spy.count(), 1 ); - QCOMPARE( spy.takeFirst().at( 1 ).toBool(), true ); - - // Copy data - QString projectDir = api->projectsPath() + "/" + projectName + "/"; - InputUtils::cpDir( sourcePath, projectDir ); - - // make MerginApi aware of the project and its directory - api->localProjectsManager().addMerginProject( projectDir, projectNamespace, projectName ); - - // Upload data - QSignalSpy spy3( api, &MerginApi::syncProjectFinished ); - api->pushProject( projectNamespace, projectName ); - QVERIFY( spy3.wait( TestUtils::LONG_REPLY ) ); - QCOMPARE( spy3.count(), 1 ); - QList arguments = spy3.takeFirst(); - int version = arguments.at( 2 ).toInt(); - QCOMPARE( version, 1 ); - - // Remove the whole project - QDir( projectDir ).removeRecursively(); - QFileInfo info( projectDir ); - QDir dir( projectDir ); - QCOMPARE( info.size(), 0 ); - QVERIFY( dir.isEmpty() ); - - api->localProjectsManager().removeLocalProject( MerginApi::getFullProjectName( projectNamespace, projectName ) ); - - QCOMPARE( QFileInfo( projectDir ).size(), 0 ); - QVERIFY( QDir( projectDir ).isEmpty() ); -} - -QString TestMerginApi::projectIdFromProjectFullName( MerginApi *api, const QString &projectNamespace, const QString &projectName ) -{ - QString ret; - if ( !api->validateAuth() || api->mApiVersionStatus != MerginApiStatus::OK || api->mServerType == MerginServerType::OLD ) - { - return ret; - } - - QString projectFullName = api->getFullProjectName( projectNamespace, projectName ); - - QNetworkReply *r = api->getProjectInfo( projectFullName ); - Q_ASSERT( r ); - QSignalSpy spy( r, &QNetworkReply::finished ); - spy.wait( TestUtils::SHORT_REPLY ); - - if ( r->error() == QNetworkReply::NoError ) - { - QByteArray data = r->readAll(); - MerginProjectMetadata serverProject = MerginProjectMetadata::fromJson( data ); - ret = serverProject.projectId; - } - else - { - qDebug() << "Project " << projectFullName << " probably does not exists on remote server"; - } - - r->deleteLater(); - return ret; -} - -void TestMerginApi::deleteRemoteProjectNow( MerginApi *api, const QString &projectNamespace, const QString &projectName ) -{ - if ( !api->validateAuth() || api->mApiVersionStatus != MerginApiStatus::OK || api->mServerType == MerginServerType::OLD ) - { - return; - } - - QString projectId = projectIdFromProjectFullName( api, projectNamespace, projectName ); - if ( projectId.isEmpty() ) - { - // probably no such project exist on server - return; - } - - QNetworkRequest request = api->getDefaultRequest(); - QUrl url( api->mApiRoot + QStringLiteral( "/v2/projects/%1" ).arg( projectId ) ); - request.setUrl( url ); - qDebug() << "Trying to delete project " << projectName << ", id: " << projectId << " (" << url << ")"; - QNetworkReply *r = api->mManager->deleteResource( request ); - QSignalSpy spy( r, &QNetworkReply::finished ); - spy.wait( TestUtils::SHORT_REPLY ); - - QCOMPARE( r->error(), QNetworkReply::NoError ); - r->deleteLater(); -} - - -void TestMerginApi::deleteLocalProject( MerginApi *api, const QString &projectNamespace, const QString &projectName ) -{ - LocalProject project = api->getLocalProject( MerginApi::getFullProjectName( projectNamespace, projectName ) ); - QVERIFY( project.isValid() ); - QVERIFY( project.projectDir.startsWith( api->projectsPath() ) ); // just to make sure we don't delete something wrong (-: - - api->localProjectsManager().removeLocalProject( project.id() ); -} - -void TestMerginApi::deleteLocalDir( MerginApi *api, const QString &dirPath ) -{ - QDir dir( api->projectsPath() + "/" + dirPath ); - QVERIFY( dir.removeRecursively() ); -} - -void TestMerginApi::downloadRemoteProject( MerginApi *api, const QString &projectNamespace, const QString &projectName ) -{ - int serverVersion; - downloadRemoteProject( api, projectNamespace, projectName, serverVersion ); -} - -void TestMerginApi::downloadRemoteProject( MerginApi *api, const QString &projectNamespace, const QString &projectName, int &serverVersion ) -{ - QSignalSpy spy( api, &MerginApi::syncProjectFinished ); - api->pullProject( projectNamespace, projectName ); - QCOMPARE( api->transactions().count(), 1 ); - QVERIFY( spy.wait( TestUtils::LONG_REPLY * 5 ) ); - serverVersion = serverVersionFromSpy( spy ); -} - -void TestMerginApi::uploadRemoteProject( MerginApi *api, const QString &projectNamespace, const QString &projectName ) -{ - int serverVersion; - uploadRemoteProject( api, projectNamespace, projectName, serverVersion ); -} - -void TestMerginApi::uploadRemoteProject( MerginApi *api, const QString &projectNamespace, const QString &projectName, int &serverVersion ) -{ - api->pushProject( projectNamespace, projectName ); - QSignalSpy spy( api, &MerginApi::syncProjectFinished ); - QVERIFY( spy.wait( TestUtils::LONG_REPLY * 30 ) ); - QCOMPARE( spy.count(), 1 ); - serverVersion = serverVersionFromSpy( spy ); -} - -void TestMerginApi::writeFileContent( const QString &filename, const QByteArray &data ) -{ - QFile f( filename ); - bool ok = f.open( QIODeviceBase::WriteOnly ); - Q_ASSERT( ok ); - f.write( data ); - f.flush(); - f.close(); -} - -QByteArray TestMerginApi::readFileContent( const QString &filename ) -{ - QFile f( filename ); - if ( !f.exists() ) - { - qDebug() << "Filename " << filename << " does not exist"; - Q_ASSERT( false ); - } - bool ok = f.open( QIODeviceBase::ReadOnly ); - Q_ASSERT( ok ); - QByteArray data = f.readAll(); - f.close(); - return data; -} - -void TestMerginApi::createLocalProject( const QString projectDir ) -{ - QDir().mkdir( projectDir ); - bool r0 = QFile::copy( mTestDataPath + "/diff_project/base.gpkg", projectDir + "/base.gpkg" ); - - QVERIFY( r0 ); -} - -bool TestMerginApi::createJsonFile( const QString &path, const QVariantMap ¶ms ) -{ - QJsonObject json = QJsonObject::fromVariantMap( params ); - QJsonDocument doc( json ); - QByteArray data = doc.toJson(); - - writeFileContent( path, data ); - - QFile config( path ); - return config.exists(); -} - -void TestMerginApi::refreshProjectsModel( const ProjectsModel::ProjectModelTypes modelType ) -{ - if ( modelType == ProjectsModel::LocalProjectsModel ) - { - QSignalSpy spy( mApi, &MerginApi::listProjectsByNameFinished ); - mLocalProjectsModel->listProjects(); - QVERIFY( spy.wait( TestUtils::SHORT_REPLY ) ); - QCOMPARE( spy.count(), 1 ); - } - else if ( modelType == ProjectsModel::WorkspaceProjectsModel ) - { - QSignalSpy spy( mApi, &MerginApi::listProjectsFinished ); - mWorkspaceProjectsModel->listProjects(); - QVERIFY( spy.wait( TestUtils::SHORT_REPLY ) ); - QCOMPARE( spy.count(), 1 ); - } -} - void TestMerginApi::testServerType() { QSignalSpy spy( mApi, &MerginApi::serverTypeChanged ); From fb625de8745212b98ce0b10b9b9828653d886fe5 Mon Sep 17 00:00:00 2001 From: uclaros Date: Sat, 8 Aug 2026 18:10:02 +0300 Subject: [PATCH 3/5] Fix conflict naming, would fail if email did not match user name (eg emails with +) --- app/test/testmerginapi.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index d46b5335b..1edb40e12 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -1177,7 +1177,7 @@ void TestMerginApi::testConflictRemoteUpdateLocalUpdate() // verify the result: the server version should be in test1.txt // and the local version should go to "test1 (conflicted copy, v).txt" - QString conflictFilename = projectDir + "/test1 (conflicted copy, " + mUsername + " v1).txt"; + QString conflictFilename = projectDir + "/test1 (conflicted copy, " + mApi->userInfo()->username() + " v1).txt"; QCOMPARE( readFileContent( filename ), QByteArray( "remote content" ) ); QCOMPARE( readFileContent( conflictFilename ), QByteArray( "local content" ) ); @@ -1201,7 +1201,7 @@ void TestMerginApi::testConflictRemoteUpdateLocalUpdate() // verify the result: the server version should be in test1.txt // and the local version should go to "test1 (conflicted copy, v).txt" // Note: test1.txt conflict file should still be the same - QString conflictFilename2 = projectDir + "/test1 (conflicted copy, " + mUsername + " v3).txt"; + QString conflictFilename2 = projectDir + "/test1 (conflicted copy, " + mApi->userInfo()->username() + " v3).txt"; QCOMPARE( readFileContent( filename ), QByteArray( "remote content 2" ) ); QCOMPARE( readFileContent( conflictFilename ), QByteArray( "local content" ) ); QCOMPARE( readFileContent( conflictFilename2 ), QByteArray( "local content 2" ) ); @@ -1243,7 +1243,7 @@ void TestMerginApi::testConflictRemoteAddLocalAdd() // verify the result: the server version should be in test1.txt // and the local version should go to conflicted copy file - QString conflictFilename = projectDir + "/test-new-file (conflicted copy, " + mUsername + " v1).txt"; + QString conflictFilename = projectDir + "/test-new-file (conflicted copy, " + mApi->userInfo()->username() + " v1).txt"; QCOMPARE( readFileContent( filename ), QByteArray( "new remote content" ) ); QCOMPARE( readFileContent( conflictFilename ), QByteArray( "new local content" ) ); } @@ -1296,11 +1296,11 @@ void TestMerginApi::testEditConflictScenario() QDir projDir( projectDir ); // check the edit conflict file presence - QVERIFY( InputUtils::fileExists( projectDir + "/" + QString( "data (edit conflict, %1 v2).json" ).arg( mUsername ) ) ); + QVERIFY( InputUtils::fileExists( projectDir + "/" + QString( "data (edit conflict, %1 v2).json" ).arg( mApi->userInfo()->username() ) ) ); // when client B downloads changes, he should also have that edit conflict file downloadRemoteProject( mApiExtra, mWorkspaceName, projectName ); - QVERIFY( InputUtils::fileExists( projectDir + "/" + QString( "data (edit conflict, %1 v2).json" ).arg( mUsername ) ) ); + QVERIFY( InputUtils::fileExists( projectDir + "/" + QString( "data (edit conflict, %1 v2).json" ).arg( mApi->userInfo()->username() ) ) ); } void TestMerginApi::testUploadWithUpdate() @@ -1611,7 +1611,7 @@ void TestMerginApi::testDiffUpdateWithRebaseFailed() // // check the result // - QString conflictFilename = "base (conflicted copy, " + mUsername + " v1).gpkg"; + QString conflictFilename = "base (conflicted copy, " + mApi->userInfo()->username() + " v1).gpkg"; QVERIFY( QFile::exists( projectDir + "/base.gpkg" ) ); QVERIFY( QFile::exists( projectDir + "/" + conflictFilename ) ); From dae828aa53ebaa3da4f1cc6bf0b4878555c6c694 Mon Sep 17 00:00:00 2001 From: uclaros Date: Sat, 8 Aug 2026 18:10:40 +0300 Subject: [PATCH 4/5] Add WAL related test cases --- app/test/testmerginapi.cpp | 212 +++++++++++++++++++++++++++++++++++++ app/test/testmerginapi.h | 4 + 2 files changed, 216 insertions(+) diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index 1edb40e12..2ed0996e4 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -3410,3 +3410,215 @@ void TestMerginApi::testServerVersionIsAtLeast() QVERIFY( mApi->serverVersionIsAtLeast( 9999, 9999, 9999 ) ); // equal QVERIFY( !mApi->serverVersionIsAtLeast( 10000, 0, 0 ) ); // higher major } + +void TestMerginApi::testSyncWhileReadingFromGpkg() +{ + // QGIS uses WAL for local geopackage access. + // This test will fail if for some reason OGR_SQLITE_JOURNAL=DELETE is set + // either by us or QGIS at some point. Database will be locked by the reader and + // a conflicted copy will be created by geodiff. + // + // If a gpkg is being read from (eg still rendering in the background) then geodiff + // should properly handle it and not create a conflicted copy. + + QString projectName = "testSyncWhileReading"; + QString projectDir = mApi->projectsPath() + "/" + projectName; + QString projectDirExtra = mApiExtra->projectsPath() + "/" + projectName; + + createRemoteProject( mApiExtra, mWorkspaceName, projectName, mTestDataPath + "/" + "diff_project" + "/" ); + + downloadRemoteProject( mApi, mWorkspaceName, projectName ); + + QStringList conflictFilter = QStringList() << QStringLiteral( "*conflicted copy*" ); + QVERIFY( QDir( projectDir ).entryList( conflictFilter, QDir::Files ).isEmpty() ); + + // somebody else adds a row and pushes version 2 + downloadRemoteProject( mApiExtra, mWorkspaceName, projectName ); + bool r0 = QFile::remove( projectDirExtra + "/base.gpkg" ); + bool r1 = QFile::copy( mTestDataPath + "/added_row.gpkg", projectDirExtra + "/base.gpkg" ); + QVERIFY( r0 && r1 ); + uploadRemoteProject( mApiExtra, mWorkspaceName, projectName ); + + // start reading the local gpkg and stop half way through, like a render that is + // still in progress. This leaves the SQLite read transaction open and the database is locked if + // sqlite journal_mode = DELETE + QgsVectorLayer *vl = new QgsVectorLayer( projectDir + "/base.gpkg|layername=simple", "base", "ogr" ); + QVERIFY( vl->isValid() ); + QCOMPARE( vl->featureCount(), static_cast( 3 ) ); + + QgsFeatureIterator it = vl->getFeatures(); + QgsFeature f; + QVERIFY( it.nextFeature( f ) ); + + // pull version 2 while the gpkg is still being read from + downloadRemoteProject( mApi, mWorkspaceName, projectName ); + + delete vl; + + // the server change should have been applied to our file, without creating a conflicted copy + QCOMPARE( QDir( projectDir ).entryList( conflictFilter, QDir::Files ), QStringList() ); + + QgsVectorLayer *vl2 = new QgsVectorLayer( projectDir + "/base.gpkg|layername=simple", "base", "ogr" ); + QVERIFY( vl2->isValid() ); + QCOMPARE( vl2->featureCount(), static_cast( 4 ) ); + delete vl2; +} + +void TestMerginApi::testSyncWhileWalFilesExist() +{ + QString projectName = "testSyncWhileWalFilesExist"; + QString projectDir = mApi->projectsPath() + "/" + projectName; + QString projectDirExtra = mApiExtra->projectsPath() + "/" + projectName; + + createRemoteProject( mApi, mWorkspaceName, projectName, mTestDataPath + "/" + "diff_project" + "/" ); + + downloadRemoteProject( mApi, mWorkspaceName, projectName ); + + QStringList conflictFilter = QStringList() << QStringLiteral( "*conflicted copy*" ); + QVERIFY( QDir( projectDir ).entryList( conflictFilter, QDir::Files ).isEmpty() ); + + // somebody else adds a row and pushes version 2 + downloadRemoteProject( mApiExtra, mWorkspaceName, projectName ); + bool r0 = QFile::remove( projectDirExtra + "/base.gpkg" ); + bool r1 = QFile::copy( mTestDataPath + "/added_row.gpkg", projectDirExtra + "/base.gpkg" ); + QVERIFY( r0 && r1 ); + uploadRemoteProject( mApiExtra, mWorkspaceName, projectName ); + + // Add a feature, commit it to the provider but don't stop editing so the changes are in the -wal file + QgsVectorLayer *vl = new QgsVectorLayer( projectDir + "/base.gpkg|layername=simple", "base", "ogr" ); + QVERIFY( vl->isValid() ); + QCOMPARE( vl->featureCount(), static_cast( 3 ) ); + + vl->startEditing(); + QgsFeature f( vl->fields() ); + vl->addFeature( f ); + // save but don't stop editing so -wal file is preserved. The added feature now lives in the -wal file + vl->commitChanges( false ); + QVERIFY( QFile::exists( projectDir + "/base.gpkg-wal" ) ); + + // pull version 2 while the layer is in editing mode + downloadRemoteProject( mApi, mWorkspaceName, projectName ); + + delete vl; + + // the server change should have been applied to our file, without creating a conflicted copy + QCOMPARE( QDir( projectDir ).entryList( conflictFilter, QDir::Files ), QStringList() ); + + QgsVectorLayer *vl2 = new QgsVectorLayer( projectDir + "/base.gpkg|layername=simple", "base", "ogr" ); + QVERIFY( vl2->isValid() ); + + // the edits stored in the WAL exist in the final file ( 3 + 1 + 1 ) + QCOMPARE( vl2->featureCount(), static_cast( 5 ) ); + delete vl2; +} + +void TestMerginApi::testSyncWhileWalFilesExistWithConflict() +{ + QString projectName = "testSyncWhileWalFilesExistWithConflict"; + QString projectDir = mApi->projectsPath() + "/" + projectName; + QString projectDirExtra = mApiExtra->projectsPath() + "/" + projectName; + + createRemoteProject( mApi, mWorkspaceName, projectName, mTestDataPath + "/" + "diff_project" + "/" ); + + downloadRemoteProject( mApi, mWorkspaceName, projectName ); + downloadRemoteProject( mApiExtra, mWorkspaceName, projectName ); + + QStringList conflictFilter = QStringList() << QStringLiteral( "*conflict*" ); + QVERIFY( QDir( projectDir ).entryList( conflictFilter, QDir::Files ).isEmpty() ); + + // two clients add one feature and update an existing one (both the same) + { + QgsVectorLayer *vl = new QgsVectorLayer( projectDirExtra + "/base.gpkg|layername=simple", "base", "ogr" ); + QVERIFY( vl->isValid() ); + QCOMPARE( vl->featureCount(), static_cast( 3 ) ); + vl->startEditing(); + QgsFeature f( vl->fields() ); + vl->addFeature( f ); + f = vl->getFeature( 1 ); + f.setAttribute( "rating", 42 ); + vl->updateFeature( f ); + vl->commitChanges(); + delete vl; + } + + uploadRemoteProject( mApiExtra, mWorkspaceName, projectName ); + + // now second client does the same but different values to create conflict + QgsVectorLayer *vl = new QgsVectorLayer( projectDir + "/base.gpkg|layername=simple", "base", "ogr" ); + QVERIFY( vl->isValid() ); + QCOMPARE( vl->featureCount(), static_cast( 3 ) ); + vl->startEditing(); + QgsFeature f( vl->fields() ); + vl->addFeature( f ); + f = vl->getFeature( 1 ); + f.setAttribute( "rating", 69 ); + vl->updateFeature( f ); + // save but don't stop editing so -wal file is preserved. The added feature now lives in the -wal file + vl->commitChanges( false ); + QVERIFY( QFile::exists( projectDir + "/base.gpkg-wal" ) ); + + // pull version 2 while the gpkg is still open with changes in the -wal file + downloadRemoteProject( mApi, mWorkspaceName, projectName ); + + delete vl; + + // the server change should persist, while the local change should be in a conflict file + QCOMPARE( QDir( projectDir ).entryList( conflictFilter, QDir::Files ).count(), static_cast( 1 ) ); + + QgsVectorLayer *vl2 = new QgsVectorLayer( projectDir + "/base.gpkg|layername=simple", "base", "ogr" ); + QVERIFY( vl2->isValid() ); + + // the edits stored in the WAL must also exist in the final file ( 3 + 1 + 1 ) + QCOMPARE( vl2->featureCount(), static_cast( 5 ) ); + delete vl2; +} + +void TestMerginApi::testSyncWhileWalFilesExistWithConflictedCopy() +{ + QString projectName = "testSyncWhileWalFilesExistWithConflictedCopy"; + QString projectDir = mApi->projectsPath() + "/" + projectName; + QString projectDirExtra = mApiExtra->projectsPath() + "/" + projectName; + + createRemoteProject( mApi, mWorkspaceName, projectName, mTestDataPath + "/" + "diff_project" + "/" ); + + downloadRemoteProject( mApi, mWorkspaceName, projectName ); + downloadRemoteProject( mApiExtra, mWorkspaceName, projectName ); + + QStringList conflictFilter = QStringList() << QStringLiteral( "*conflict*" ); + QVERIFY( QDir( projectDir ).entryList( conflictFilter, QDir::Files ).isEmpty() ); + + // One client modifies schema, so that other client will generate a conflicted copy + bool r0 = QFile::remove( projectDirExtra + "/base.gpkg" ); + bool r1 = QFile::copy( mTestDataPath + "/added_column.gpkg", projectDirExtra + "/base.gpkg" ); + QVERIFY( r0 && r1 ); + + uploadRemoteProject( mApiExtra, mWorkspaceName, projectName ); + + // now second client adds a feature, + QgsVectorLayer *vl = new QgsVectorLayer( projectDir + "/base.gpkg|layername=simple", "base", "ogr" ); + QVERIFY( vl->isValid() ); + QCOMPARE( vl->featureCount(), static_cast( 3 ) ); + vl->startEditing(); + QgsFeature f( vl->fields() ); + vl->addFeature( f ); + // save but don't stop editing so -wal file is preserved. The added feature now lives in the -wal file + vl->commitChanges( false ); + QVERIFY( QFile::exists( projectDir + "/base.gpkg-wal" ) ); + + // pull version 2 while the gpkg is still open with changes in the -wal file + downloadRemoteProject( mApi, mWorkspaceName, projectName ); + + delete vl; + + // the server change should persist and the local change should be in a conflicted copy + QString conflictFilename = "base (conflicted copy, " + mApi->userInfo()->username() + " v1).gpkg"; + QVERIFY( QFile::exists( projectDir + "/base.gpkg" ) ); + QVERIFY( QFile::exists( projectDir + "/" + conflictFilename ) ); + + QgsVectorLayer *vl2 = new QgsVectorLayer( projectDir + "/" + conflictFilename + "|layername=simple", "base", "ogr" ); + QVERIFY( vl2->isValid() ); + + // the edits stored in the WAL file must exist in the conflicted copy file file + QCOMPARE( vl2->featureCount(), static_cast( 4 ) ); + delete vl2; +} diff --git a/app/test/testmerginapi.h b/app/test/testmerginapi.h index 44c47fa2c..3a11c5466 100644 --- a/app/test/testmerginapi.h +++ b/app/test/testmerginapi.h @@ -151,6 +151,10 @@ class TestMerginApi: public QObject void testMerginConfigFromFile(); void testHasLocalChangesWithSelectiveSyncEnabled(); void testHasLocalProjectChanges(); + void testSyncWhileReadingFromGpkg(); + void testSyncWhileWalFilesExist(); + void testSyncWhileWalFilesExistWithConflict(); + void testSyncWhileWalFilesExistWithConflictedCopy(); void testOfflineCache(); void testRegisterAndDelete(); void testCreateWorkspace(); From 7f38c5cd6c94678031785798c6de7e1aa9a49aee Mon Sep 17 00:00:00 2001 From: uclaros Date: Sat, 8 Aug 2026 18:13:17 +0300 Subject: [PATCH 5/5] Add WAL aware copy and rename for when copying gpkg with WAL files present. Copy uses sqlite backup to make sure that WAL contents are also copied along. Rename tries to also rename -wal and -shm files --- core/geodiffutils.cpp | 24 ++++++++++++++++++++++++ core/geodiffutils.h | 6 ++++++ core/merginapi.cpp | 8 ++++---- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/core/geodiffutils.cpp b/core/geodiffutils.cpp index d1ee2cb4f..a78e867f8 100644 --- a/core/geodiffutils.cpp +++ b/core/geodiffutils.cpp @@ -187,3 +187,27 @@ void GeodiffUtils::log( GEODIFF_LoggerLevel level, const char *msg ) } CoreUtils::log( prefix, msg ); } + +bool GeodiffUtils::copyWalAware( const QString &src, const QString &dst ) +{ + if ( src.endsWith( QLatin1String( ".gpkg" ), Qt::CaseInsensitive ) ) + { + const int res = GEODIFF_makeCopySqlite( GeodiffContext::instance().handle(), src.toLocal8Bit().constData(), dst.toLocal8Bit().constData() ); + return res == GEODIFF_SUCCESS; + } + else + { + return QFile::copy( src, dst ); + } +} + +bool GeodiffUtils::renameWalAware( const QString& src, const QString& dst ) +{ + const bool res = QFile::rename( src, dst ); + if ( res && src.endsWith( QLatin1String( ".gpkg" ), Qt::CaseInsensitive ) ) + { + ( void ) QFile::rename( QStringLiteral( "%1-wal" ).arg( src ), QStringLiteral( "%1-wal" ).arg( dst ) ); + ( void ) QFile::rename( QStringLiteral( "%1-shm" ).arg( src ), QStringLiteral( "%1-shm" ).arg( dst ) ); + } + return res; +} diff --git a/core/geodiffutils.h b/core/geodiffutils.h index 7b954b8bd..3695f9d2a 100644 --- a/core/geodiffutils.h +++ b/core/geodiffutils.h @@ -68,6 +68,12 @@ class GeodiffUtils //! Rebases local modified version from base to modified_their version static bool rebase( const QString &base, const QString &modified_their, const QString &modified, const QString &conflictfile ); + + //! Copies file. If file is gpkg, a sqlite backup is performed so WAL is taken into account + static bool copyWalAware( const QString &src, const QString &dst ); + + //! Renames file. If file is gpkg, any -wal and -shm files are renamed too + static bool renameWalAware( const QString &src, const QString &dst ); }; #endif // GEODIFFUTILS_H diff --git a/core/merginapi.cpp b/core/merginapi.cpp index ab91e0611..29b0c0468 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -2129,7 +2129,7 @@ void MerginApi::finalizeProjectPullCopy( const QString &projectFullName, const Q { CoreUtils::log( "pull " + projectFullName, "failed to remove old basefile for: " + filePath ); } - if ( !QFile::copy( dest, basefile ) ) + if ( !GeodiffUtils::copyWalAware( dest, basefile ) ) { CoreUtils::log( "pull " + projectFullName, "failed to copy new basefile for: " + filePath ); } @@ -2208,7 +2208,7 @@ bool MerginApi::finalizeProjectPullApplyDiff( const QString &projectFullName, co hasConflicts = true; LocalProject info = mLocalProjects.projectFromMerginName( projectFullName ); QString newDest = CoreUtils::findUniquePath( CoreUtils::generateConflictedCopyFileName( dest, mUserInfo->username(), info.localVersion ) ); - if ( !QFile::rename( dest, newDest ) ) + if ( !GeodiffUtils::renameWalAware( dest, newDest ) ) { CoreUtils::log( "pull " + projectFullName, "failed rename of conflicting file after failed geodiff rebase: " + filePath ); } @@ -2263,7 +2263,7 @@ void MerginApi::finalizeProjectPull( const QString &projectFullName ) QString origPath = projectDir + "/" + finalizationItem.filePath; LocalProject info = mLocalProjects.projectFromMerginName( projectFullName ); QString newPath = CoreUtils::findUniquePath( CoreUtils::generateConflictedCopyFileName( origPath, mUserInfo->username(), info.localVersion ) ); - if ( !QFile::rename( origPath, newPath ) ) + if ( !GeodiffUtils::renameWalAware( origPath, newPath ) ) { CoreUtils::log( "pull " + projectFullName, "failed rename of conflicting file: " + finalizationItem.filePath ); } @@ -3094,7 +3094,7 @@ void MerginApi::pushFinishReplyFinished() createPathIfNotExists( basefile ); QString sourcePath = transaction.projectDir + "/" + filePath; - if ( !QFile::copy( sourcePath, basefile ) ) + if ( !GeodiffUtils::copyWalAware( sourcePath, basefile ) ) { CoreUtils::log( "push " + projectFullName, "failed to copy new basefile for: " + filePath ); }