From 16e85e73e33b30c57dcefa3ab2e395f18121f32d Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Thu, 17 Apr 2025 17:25:51 +0200 Subject: [PATCH 01/11] feat(quota): add quota-available-bytes and quota-used-bytes to PROPFIND properties request. DiscoveryPhase: add quota used and available properties to RemoteInfo and LocalInfo structs. Signed-off-by: Camila Ayres --- src/libsync/discovery.cpp | 4 ++++ src/libsync/discoveryphase.cpp | 22 ++++++++++++++++++++++ src/libsync/discoveryphase.h | 8 ++++++++ 3 files changed, 34 insertions(+) diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 5a7495d21d011..839d2ca92bb73 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -1280,6 +1280,8 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( item->_size = localEntry.size; item->_modtime = localEntry.modtime; item->_type = localEntry.isDirectory ? ItemTypeDirectory : ItemTypeFile; + item->_quotaUsed = localEntry.isDirectory ? localEntry.quotaUsed : 0; + item->_quotaAvailable = localEntry.isDirectory ? localEntry.quotaAailable : 0; _childModified = true; } else if (dbEntry._modtime > 0 && (localEntry.modtime <= 0 || localEntry.modtime >= 0xFFFFFFFF) && dbEntry._fileSize == localEntry.size) { item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; @@ -1288,6 +1290,8 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( item->_modtime = dbEntry._modtime; item->_previousModtime = dbEntry._modtime; item->_type = localEntry.isDirectory ? ItemTypeDirectory : ItemTypeFile; + item->_quotaUsed = localEntry.isDirectory ? localEntry.quotaUsed : 0; + item->_quotaAvailable = localEntry.isDirectory ? localEntry.quotaAailable : 0; qCDebug(lcDisco) << "CSYNC_INSTRUCTION_SYNC: File" << item->_file << "if (dbEntry._modtime > 0 && localEntry.modtime <= 0)" << "dbEntry._modtime:" << dbEntry._modtime << "localEntry.modtime:" << localEntry.modtime; diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp index 8c0a108188f34..89bc1d693a16f 100644 --- a/src/libsync/discoveryphase.cpp +++ b/src/libsync/discoveryphase.cpp @@ -409,6 +409,8 @@ void DiscoverySingleDirectoryJob::start() << "getlastmodified" << "getcontentlength" << "getetag" + << "quota-available-bytes" + << "quota-used-bytes" << "http://owncloud.org/ns:size" << "http://owncloud.org/ns:id" << "http://owncloud.org/ns:fileid" @@ -581,6 +583,14 @@ static void propertyMapToRemoteInfo(const QMap &map, RemotePer if (result.isDirectory && map.contains("size")) { result.sizeOfFolder = map.value("size").toInt(); } + + if (result.isDirectory && map.contains("quota-used-bytes")) { + result.quotaUsed = map.value("quota-used-bytes").toLongLong(); + } + + if (result.isDirectory && map.contains("quota-available-bytes")) { + result.quotaAvailable = map.value("quota-available-bytes").toLongLong(); + } } void DiscoverySingleDirectoryJob::directoryListingIteratedSlot(const QString &file, const QMap &map) @@ -615,11 +625,23 @@ void DiscoverySingleDirectoryJob::directoryListingIteratedSlot(const QString &fi if (map.contains("size")) { _size = map.value("size").toInt(); } + if (map.contains("quota-used-bytes")) { + _quotaUsed = map.value("quota-used-bytes").toLongLong(); + } + if (map.contains("quota-available-bytes")) { + _quotaAvailable = map.value("quota-available-bytes").toLongLong(); + } } else { RemoteInfo result; int slash = file.lastIndexOf('/'); result.name = file.mid(slash + 1); result.size = -1; + if (map.contains("quota-used-bytes")) { + result.quotaUsed = map.value("quota-used-bytes").toInt(); + } + if (map.contains("quota-available-bytes")) { + result.quotaAvailable = map.value("quota-available-bytes").toInt(); + } propertyMapToRemoteInfo(map, _account->serverHasMountRootProperty() ? RemotePermissions::MountedPermissionAlgorithm::UseMountRootProperty : RemotePermissions::MountedPermissionAlgorithm::WildGuessMountedSubProperty, result); diff --git a/src/libsync/discoveryphase.h b/src/libsync/discoveryphase.h index a29937d1f0b78..af97539083da2 100644 --- a/src/libsync/discoveryphase.h +++ b/src/libsync/discoveryphase.h @@ -83,6 +83,9 @@ struct RemoteInfo bool isLivePhoto = false; QString livePhotoFile; + + int64_t quotaUsed = 0; + int64_t quotaAvailable = 0; }; struct LocalInfo @@ -101,6 +104,8 @@ struct LocalInfo bool isMetadataMissing = false; bool isPermissionsInvalid = false; [[nodiscard]] bool isValid() const { return !name.isNull(); } + int64_t quotaUsed = 0; + int64_t quotaAailable = 0; }; /** @@ -206,6 +211,9 @@ private slots: // store top level E2EE folder paths as they are used later when discovering nested folders QSet _topLevelE2eeFolderPaths; + int64_t _quotaUsed = 0; + int64_t _quotaAvailable = 0; + public: QByteArray _dataFingerprint; }; From 8abdba62b276fcdcce125f185b338da2a86f6553 Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Tue, 22 Apr 2025 17:58:19 +0200 Subject: [PATCH 02/11] feat(quota): throw error when item size does not match folder quota. Use public struct to store bytes used and available for each folder. Signed-off-by: Camila Ayres --- src/libsync/discovery.cpp | 36 +++++++++++++++++++++++++++------- src/libsync/discovery.h | 5 +++++ src/libsync/discoveryphase.cpp | 17 ++++++++-------- src/libsync/discoveryphase.h | 19 +++++++++++------- src/libsync/syncfileitem.h | 7 +++++++ 5 files changed, 61 insertions(+), 23 deletions(-) diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 839d2ca92bb73..1d1d696e1b424 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -579,7 +579,6 @@ void ProcessDirectoryJob::processFile(PathTuple path, << " | (db/local/remote)" << " | valid: " << dbEntry.isValid() << "/" << hasLocal << "/" << hasServer << " | mtime: " << dbEntry._modtime << "/" << localEntry.modtime << "/" << serverEntry.modtime - << " | size: " << dbEntry._fileSize << "/" << localEntry.size << "/" << serverEntry.size << " | etag: " << dbEntry._etag << "//" << serverEntry.etag << " | checksum: " << dbEntry._checksumHeader << "//" << serverEntry.checksumHeader << " | perm: " << dbEntry._remotePerm << "//" << serverEntry.remotePerm @@ -591,7 +590,10 @@ void ProcessDirectoryJob::processFile(PathTuple path, << " | file lock: " << localFileIsLocked << "//" << serverFileIsLocked << " | file lock type: " << localFileLockType << "//" << serverFileLockType << " | live photo: " << dbEntry._isLivePhoto << "//" << serverEntry.isLivePhoto - << " | metadata missing: /" << localEntry.isMetadataMissing << '/'; + << " | metadata missing: /" << localEntry.isMetadataMissing << '/' + << " | size: " << dbEntry._fileSize << "/" << localEntry.size << "/" << serverEntry.size + << " | quota used: //" << serverEntry.folderQuota.bytesUsed + << " | quota available: //" << serverEntry.folderQuota.bytesAvailable; qCInfo(lcDisco).nospace() << processingLog; @@ -766,6 +768,9 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(const SyncFileItemPtr &it item->_isLivePhoto = serverEntry.isLivePhoto; item->_livePhotoFile = serverEntry.livePhotoFile; + item->_folderQuota.bytesUsed = serverEntry.folderQuota.bytesUsed; + item->_folderQuota.bytesAvailable = serverEntry.folderQuota.bytesAvailable; + // Check for missing server data { QStringList missingData; @@ -1126,6 +1131,20 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( item->_status = SyncFileItem::Status::NormalError; } + if (!item->isDirectory() && item->_direction == SyncFileItem::Up && item->_size > 0 + && (item->_instruction == CSYNC_INSTRUCTION_SYNC || item->_instruction == CSYNC_INSTRUCTION_NEW)) { + if (const auto parentFolderQuotaAvailable = _dirItem ? _dirItem->_folderQuota.bytesAvailable + : _folderQuota.bytesAvailable; + item->_size > parentFolderQuotaAvailable) { + item->_instruction = CSYNC_INSTRUCTION_ERROR; + item->_errorString = tr("Upload of %1 exceeds the quota %2 for the folder %3.").arg(Utility::octetsToString(item->_size), + Utility::octetsToString(parentFolderQuotaAvailable), + _currentFolder._server); + item->_status = SyncFileItem::Status::NormalError; + _discoveryData->_anotherSyncNeeded = true; + } + } + if (item->_type != CSyncEnums::ItemTypeVirtualFile) { const auto foundEditorsKeepingFileBusy = queryEditorsKeepingFileBusy(item, path); if (!foundEditorsKeepingFileBusy.isEmpty()) { @@ -1280,8 +1299,6 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( item->_size = localEntry.size; item->_modtime = localEntry.modtime; item->_type = localEntry.isDirectory ? ItemTypeDirectory : ItemTypeFile; - item->_quotaUsed = localEntry.isDirectory ? localEntry.quotaUsed : 0; - item->_quotaAvailable = localEntry.isDirectory ? localEntry.quotaAailable : 0; _childModified = true; } else if (dbEntry._modtime > 0 && (localEntry.modtime <= 0 || localEntry.modtime >= 0xFFFFFFFF) && dbEntry._fileSize == localEntry.size) { item->_instruction = CSYNC_INSTRUCTION_UPDATE_METADATA; @@ -1290,8 +1307,6 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( item->_modtime = dbEntry._modtime; item->_previousModtime = dbEntry._modtime; item->_type = localEntry.isDirectory ? ItemTypeDirectory : ItemTypeFile; - item->_quotaUsed = localEntry.isDirectory ? localEntry.quotaUsed : 0; - item->_quotaAvailable = localEntry.isDirectory ? localEntry.quotaAailable : 0; qCDebug(lcDisco) << "CSYNC_INSTRUCTION_SYNC: File" << item->_file << "if (dbEntry._modtime > 0 && localEntry.modtime <= 0)" << "dbEntry._modtime:" << dbEntry._modtime << "localEntry.modtime:" << localEntry.modtime; @@ -2157,6 +2172,7 @@ DiscoverySingleDirectoryJob *ProcessDirectoryJob::startAsyncServerQuery() } connect(serverJob, &DiscoverySingleDirectoryJob::etag, this, &ProcessDirectoryJob::etag); + connect(serverJob, &DiscoverySingleDirectoryJob::setfolderQuota, this, &ProcessDirectoryJob::setFolderQuota); _discoveryData->_currentlyActiveJobs++; _pendingAsyncJobs++; connect(serverJob, &DiscoverySingleDirectoryJob::finished, this, [this, serverJob](const auto &results) { @@ -2183,6 +2199,7 @@ DiscoverySingleDirectoryJob *ProcessDirectoryJob::startAsyncServerQuery() _serverQueryDone = true; if (!serverJob->_dataFingerprint.isEmpty() && _discoveryData->_dataFingerprint.isEmpty()) _discoveryData->_dataFingerprint = serverJob->_dataFingerprint; + if (_localQueryDone) this->process(); } else { @@ -2213,6 +2230,12 @@ DiscoverySingleDirectoryJob *ProcessDirectoryJob::startAsyncServerQuery() return serverJob; } +void ProcessDirectoryJob::setFolderQuota(FolderQuota folderQuota) +{ + _folderQuota.bytesUsed = folderQuota.bytesUsed; + _folderQuota.bytesAvailable = folderQuota.bytesAvailable; +} + void ProcessDirectoryJob::startAsyncLocalQuery() { QString localPath = _discoveryData->_localDir + _currentFolder._local; @@ -2341,5 +2364,4 @@ bool ProcessDirectoryJob::maybeRenameForWindowsCompatibility(const QString &abso } return result; } - } diff --git a/src/libsync/discovery.h b/src/libsync/discovery.h index 715b416de5eaa..761cc121ec515 100644 --- a/src/libsync/discovery.h +++ b/src/libsync/discovery.h @@ -296,9 +296,14 @@ class ProcessDirectoryJob : public QObject PinState _pinState = PinState::Unspecified; // The directory's pin-state, see computePinState() bool _isInsideEncryptedTree = false; // this directory is encrypted or is within the tree of directories with root directory encrypted + FolderQuota _folderQuota; + signals: void finished(); // The root etag of this directory was fetched void etag(const QByteArray &, const QDateTime &time); + +private slots: + void setFolderQuota(FolderQuota folderQuota); }; } diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp index 89bc1d693a16f..94e89ae691b49 100644 --- a/src/libsync/discoveryphase.cpp +++ b/src/libsync/discoveryphase.cpp @@ -585,11 +585,11 @@ static void propertyMapToRemoteInfo(const QMap &map, RemotePer } if (result.isDirectory && map.contains("quota-used-bytes")) { - result.quotaUsed = map.value("quota-used-bytes").toLongLong(); + result.folderQuota.bytesUsed = map.value("quota-used-bytes").toLongLong(); } if (result.isDirectory && map.contains("quota-available-bytes")) { - result.quotaAvailable = map.value("quota-available-bytes").toLongLong(); + result.folderQuota.bytesAvailable = map.value("quota-available-bytes").toLongLong(); } } @@ -625,11 +625,10 @@ void DiscoverySingleDirectoryJob::directoryListingIteratedSlot(const QString &fi if (map.contains("size")) { _size = map.value("size").toInt(); } - if (map.contains("quota-used-bytes")) { - _quotaUsed = map.value("quota-used-bytes").toLongLong(); - } - if (map.contains("quota-available-bytes")) { - _quotaAvailable = map.value("quota-available-bytes").toLongLong(); + + // all folders will contain both + if (map.contains("quota-used-bytes") && map.contains("quota-available-bytes")) { + emit setfolderQuota(FolderQuota{map.value("quota-used-bytes").toLongLong(), map.value("quota-available-bytes").toLongLong()}); } } else { RemoteInfo result; @@ -637,10 +636,10 @@ void DiscoverySingleDirectoryJob::directoryListingIteratedSlot(const QString &fi result.name = file.mid(slash + 1); result.size = -1; if (map.contains("quota-used-bytes")) { - result.quotaUsed = map.value("quota-used-bytes").toInt(); + result.folderQuota.bytesUsed = map.value("quota-used-bytes").toInt(); } if (map.contains("quota-available-bytes")) { - result.quotaAvailable = map.value("quota-available-bytes").toInt(); + result.folderQuota.bytesAvailable = map.value("quota-available-bytes").toInt(); } propertyMapToRemoteInfo(map, _account->serverHasMountRootProperty() ? RemotePermissions::MountedPermissionAlgorithm::UseMountRootProperty : RemotePermissions::MountedPermissionAlgorithm::WildGuessMountedSubProperty, diff --git a/src/libsync/discoveryphase.h b/src/libsync/discoveryphase.h index af97539083da2..2a90f7f6e36b2 100644 --- a/src/libsync/discoveryphase.h +++ b/src/libsync/discoveryphase.h @@ -45,6 +45,15 @@ class ProcessDirectoryJob; enum class ErrorCategory; +/** + * Represent the quota for each folder retrieved from the server + */ +struct FolderQuota +{ + int64_t bytesUsed = 0; + int64_t bytesAvailable = 0; +}; + /** * Represent all the meta-data about a file in the server */ @@ -84,8 +93,7 @@ struct RemoteInfo bool isLivePhoto = false; QString livePhotoFile; - int64_t quotaUsed = 0; - int64_t quotaAvailable = 0; + FolderQuota folderQuota; }; struct LocalInfo @@ -104,8 +112,6 @@ struct LocalInfo bool isMetadataMissing = false; bool isPermissionsInvalid = false; [[nodiscard]] bool isValid() const { return !name.isNull(); } - int64_t quotaUsed = 0; - int64_t quotaAailable = 0; }; /** @@ -169,6 +175,7 @@ class DiscoverySingleDirectoryJob : public QObject void firstDirectoryPermissions(OCC::RemotePermissions); void etag(const QByteArray &, const QDateTime &time); void finished(const OCC::HttpResult> &result); + void setfolderQuota(FolderQuota folderQuota); private slots: void directoryListingIteratedSlot(const QString &, const QMap &); @@ -211,11 +218,9 @@ private slots: // store top level E2EE folder paths as they are used later when discovering nested folders QSet _topLevelE2eeFolderPaths; - int64_t _quotaUsed = 0; - int64_t _quotaAvailable = 0; - public: QByteArray _dataFingerprint; + FolderQuota _folderQuota; }; class DiscoveryPhase : public QObject diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h index 8ff5eeedcf482..aacacfab2722e 100644 --- a/src/libsync/syncfileitem.h +++ b/src/libsync/syncfileitem.h @@ -343,6 +343,13 @@ class OWNCLOUDSYNC_EXPORT SyncFileItem /// if true, requests the file to be permanently deleted instead of moved to the trashbin /// only relevant for when `_instruction` is set to `CSYNC_INSTRUCTION_REMOVE` bool _wantsPermanentDeletion = false; + + struct FolderQuota { + int64_t bytesUsed = 0; + int64_t bytesAvailable = 0; + }; + + FolderQuota _folderQuota; }; inline bool operator<(const SyncFileItemPtr &item1, const SyncFileItemPtr &item2) From de32757b9e3005a67d971a49e0105cb0eeb48730 Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Wed, 23 Apr 2025 15:51:26 +0200 Subject: [PATCH 03/11] style(DiscoveryPhase): document possible values returned by the server for FolderQuota bytesAvailable. Signed-off-by: Camila Ayres --- src/libsync/discoveryphase.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libsync/discoveryphase.h b/src/libsync/discoveryphase.h index 2a90f7f6e36b2..a2da3ca1d1d86 100644 --- a/src/libsync/discoveryphase.h +++ b/src/libsync/discoveryphase.h @@ -47,6 +47,11 @@ enum class ErrorCategory; /** * Represent the quota for each folder retrieved from the server + * bytesUsed: space used in bytes + * bytesAvailale: free space available in bytes or + * -1: Uncomputed free space - new folder (externally created) not yet scanned by the server + * -2: Unknown free space + * -3: Unlimited free space. */ struct FolderQuota { From 35223ff52124d0892722c11aebf6e35652a4f619 Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Wed, 23 Apr 2025 15:53:36 +0200 Subject: [PATCH 04/11] fix(quota): check correctly folders with unlimited space (-3). Signed-off-by: Camila Ayres --- src/libsync/discovery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 1d1d696e1b424..022ca6d8f3e52 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -1135,7 +1135,7 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( && (item->_instruction == CSYNC_INSTRUCTION_SYNC || item->_instruction == CSYNC_INSTRUCTION_NEW)) { if (const auto parentFolderQuotaAvailable = _dirItem ? _dirItem->_folderQuota.bytesAvailable : _folderQuota.bytesAvailable; - item->_size > parentFolderQuotaAvailable) { + item->_size > parentFolderQuotaAvailable && parentFolderQuotaAvailable > -1) { item->_instruction = CSYNC_INSTRUCTION_ERROR; item->_errorString = tr("Upload of %1 exceeds the quota %2 for the folder %3.").arg(Utility::octetsToString(item->_size), Utility::octetsToString(parentFolderQuotaAvailable), From 8d58bf30e7821912fd42d2bc5b7d4cc443a8bd66 Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Wed, 23 Apr 2025 22:44:53 +0200 Subject: [PATCH 05/11] test(syncenginetestutils): add quota-available-bytes and quota-used-bytes to dav propfind request. Signed-off-by: Camila Ayres --- test/syncenginetestutils.cpp | 2 ++ test/syncenginetestutils.h | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/test/syncenginetestutils.cpp b/test/syncenginetestutils.cpp index 58fecc6e8e6cd..745cbbe5d3c8d 100644 --- a/test/syncenginetestutils.cpp +++ b/test/syncenginetestutils.cpp @@ -406,6 +406,8 @@ FakePropfindReply::FakePropfindReply(FileInfo &remoteRootFileInfo, QNetworkAcces xml.writeTextElement(davUri, QStringLiteral("getlastmodified"), stringDate); xml.writeTextElement(davUri, QStringLiteral("getcontentlength"), QString::number(fileInfo.size)); xml.writeTextElement(davUri, QStringLiteral("getetag"), QStringLiteral("\"%1\"").arg(QString::fromLatin1(fileInfo.etag))); + xml.writeTextElement(ocUri, QStringLiteral("quota-available-bytes"), std::to_string(fileInfo.quota.bytesAvailable)); + xml.writeTextElement(ocUri, QStringLiteral("quota-used-bytes"), std::to_string(fileInfo.quota.bytesUsed)); xml.writeTextElement(ocUri, QStringLiteral("permissions"), !fileInfo.permissions.isNull() ? QString(fileInfo.permissions.toString()) : fileInfo.isShared ? QStringLiteral("SRDNVCKW") : QStringLiteral("RDNVCKW")); xml.writeTextElement(ocUri, QStringLiteral("share-permissions"), QString::number(static_cast(OCC::SharePermissions(OCC::SharePermissionRead | OCC::SharePermissionUpdate | diff --git a/test/syncenginetestutils.h b/test/syncenginetestutils.h index 44ed5edda7e3d..2a70f54b2d260 100644 --- a/test/syncenginetestutils.h +++ b/test/syncenginetestutils.h @@ -192,6 +192,12 @@ class FileInfo : public FileModifier bool isEncrypted = false; bool isLivePhoto = false; + struct FileInfoQuota { + int64_t bytesUsed = 0; + int64_t bytesAvailable = 5000000000; + }; + FileInfoQuota quota; + // Sorted by name to be able to compare trees QMap children; QString parentPath; From 35b49a454a0d41cfddd4989b3046017b83b22924 Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Thu, 24 Apr 2025 12:54:34 +0200 Subject: [PATCH 06/11] fix(quota): only check for quota for items with a parent folder that exist and that are not blacklisted. Signed-off-by: Camila Ayres --- src/libsync/discovery.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 022ca6d8f3e52..18d39b634cbc3 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -1131,7 +1131,8 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( item->_status = SyncFileItem::Status::NormalError; } - if (!item->isDirectory() && item->_direction == SyncFileItem::Up && item->_size > 0 + if (_queryServer != QueryMode::InBlackList && _queryServer != QueryMode::ParentDontExist + && !item->isDirectory() && item->_direction == SyncFileItem::Up && item->_size > 0 && (item->_instruction == CSYNC_INSTRUCTION_SYNC || item->_instruction == CSYNC_INSTRUCTION_NEW)) { if (const auto parentFolderQuotaAvailable = _dirItem ? _dirItem->_folderQuota.bytesAvailable : _folderQuota.bytesAvailable; From 4935cc55a6ff169d25945caeb373cd2cda46419e Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Thu, 24 Apr 2025 13:15:06 +0200 Subject: [PATCH 07/11] fix(quota): check for the quota of the original parent folder, not of the renamed one (0 bytes available). Signed-off-by: Camila Ayres --- src/libsync/discovery.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 18d39b634cbc3..be1c6f8941f78 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -1134,9 +1134,11 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( if (_queryServer != QueryMode::InBlackList && _queryServer != QueryMode::ParentDontExist && !item->isDirectory() && item->_direction == SyncFileItem::Up && item->_size > 0 && (item->_instruction == CSYNC_INSTRUCTION_SYNC || item->_instruction == CSYNC_INSTRUCTION_NEW)) { - if (const auto parentFolderQuotaAvailable = _dirItem ? _dirItem->_folderQuota.bytesAvailable - : _folderQuota.bytesAvailable; - item->_size > parentFolderQuotaAvailable && parentFolderQuotaAvailable > -1) { + int64_t parentFolderQuotaAvailable = _folderQuota.bytesAvailable; + if (_dirItem && _dirItem->_instruction != CSYNC_INSTRUCTION_RENAME) { + parentFolderQuotaAvailable = _dirItem->_folderQuota.bytesAvailable; + } + if (item->_size > parentFolderQuotaAvailable && parentFolderQuotaAvailable > -1) { item->_instruction = CSYNC_INSTRUCTION_ERROR; item->_errorString = tr("Upload of %1 exceeds the quota %2 for the folder %3.").arg(Utility::octetsToString(item->_size), Utility::octetsToString(parentFolderQuotaAvailable), From 3c8eb21a95d65c60cb8baa9e30a641f77905658c Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Thu, 24 Apr 2025 15:21:13 +0200 Subject: [PATCH 08/11] fix(quota): if item is new and parent dir has been renamed, quota is always 0. Also moved quota check to a function. Signed-off-by: Camila Ayres --- src/libsync/discovery.cpp | 53 ++++++++++++++++++++++++++++----------- src/libsync/discovery.h | 2 ++ 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index be1c6f8941f78..d3f8da001d43a 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -1077,6 +1077,37 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(const SyncFileItemPtr &it processFileAnalyzeLocalInfo(item, path, localEntry, serverEntry, dbEntry, _queryServer); } +int64_t ProcessDirectoryJob::folderQuotaAvailable(const SyncFileItemPtr &item) +{ + if (const auto isNewItem = item->_instruction == CSYNC_INSTRUCTION_NEW; + _queryServer != QueryMode::InBlackList && _queryServer != QueryMode::ParentDontExist + && !item->isDirectory() + && item->_direction == SyncFileItem::Up + && item->_size > 0 + && (item->_instruction == CSYNC_INSTRUCTION_SYNC || isNewItem)) { + + if (!_dirItem) { + return _folderQuota.bytesAvailable; + } + + const auto isDirItemRenamed = _dirItem && _dirItem->_instruction == CSYNC_INSTRUCTION_RENAME; + + // quota is unknown at this point - follow server values: -2: Unknown free space + if (isDirItemRenamed && isNewItem) { + return -2; + } + + if (isDirItemRenamed) { + return _folderQuota.bytesAvailable; + } + + return _dirItem->_folderQuota.bytesAvailable; + } + + // -3: Unlimited free space + return -3; +} + void ProcessDirectoryJob::processFileAnalyzeLocalInfo( const SyncFileItemPtr &item, PathTuple path, const LocalInfo &localEntry, const RemoteInfo &serverEntry, const SyncJournalFileRecord &dbEntry, QueryMode recurseQueryServer) @@ -1131,21 +1162,13 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( item->_status = SyncFileItem::Status::NormalError; } - if (_queryServer != QueryMode::InBlackList && _queryServer != QueryMode::ParentDontExist - && !item->isDirectory() && item->_direction == SyncFileItem::Up && item->_size > 0 - && (item->_instruction == CSYNC_INSTRUCTION_SYNC || item->_instruction == CSYNC_INSTRUCTION_NEW)) { - int64_t parentFolderQuotaAvailable = _folderQuota.bytesAvailable; - if (_dirItem && _dirItem->_instruction != CSYNC_INSTRUCTION_RENAME) { - parentFolderQuotaAvailable = _dirItem->_folderQuota.bytesAvailable; - } - if (item->_size > parentFolderQuotaAvailable && parentFolderQuotaAvailable > -1) { - item->_instruction = CSYNC_INSTRUCTION_ERROR; - item->_errorString = tr("Upload of %1 exceeds the quota %2 for the folder %3.").arg(Utility::octetsToString(item->_size), - Utility::octetsToString(parentFolderQuotaAvailable), - _currentFolder._server); - item->_status = SyncFileItem::Status::NormalError; - _discoveryData->_anotherSyncNeeded = true; - } + if (const auto folderQuota = folderQuotaAvailable(item);item->_size > folderQuota && folderQuota > -1) { + item->_instruction = CSYNC_INSTRUCTION_ERROR; + item->_errorString = tr("Upload of %1 exceeds the %2 of space left in %3 folder.").arg(Utility::octetsToString(item->_size), + Utility::octetsToString(folderQuota), + _currentFolder._server); + item->_status = SyncFileItem::Status::NormalError; + _discoveryData->_anotherSyncNeeded = true; } if (item->_type != CSyncEnums::ItemTypeVirtualFile) { diff --git a/src/libsync/discovery.h b/src/libsync/discovery.h index 761cc121ec515..2ac09ec433bae 100644 --- a/src/libsync/discovery.h +++ b/src/libsync/discovery.h @@ -298,6 +298,8 @@ class ProcessDirectoryJob : public QObject FolderQuota _folderQuota; + int64_t folderQuotaAvailable(const SyncFileItemPtr &item); + signals: void finished(); // The root etag of this directory was fetched From d0ea889f98b350a52b8b97e6cce2465cd4bb2521 Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Thu, 24 Apr 2025 16:29:16 +0200 Subject: [PATCH 09/11] fix(quota): show personal files instead of empty string when error message is about root folder. Signed-off-by: Camila Ayres --- src/libsync/discovery.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index d3f8da001d43a..97d73e1a943ba 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -1164,9 +1164,15 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( if (const auto folderQuota = folderQuotaAvailable(item);item->_size > folderQuota && folderQuota > -1) { item->_instruction = CSYNC_INSTRUCTION_ERROR; - item->_errorString = tr("Upload of %1 exceeds the %2 of space left in %3 folder.").arg(Utility::octetsToString(item->_size), - Utility::octetsToString(folderQuota), - _currentFolder._server); + if (_currentFolder._server.isEmpty()) { + item->_errorString = tr("Upload of %1 exceeds %2 of space left in personal files.").arg(Utility::octetsToString(item->_size), + Utility::octetsToString(folderQuota)); + } else { + item->_errorString = tr("Upload of %1 exceeds %2 of space left in folder %3.").arg(Utility::octetsToString(item->_size), + Utility::octetsToString(folderQuota), + _currentFolder._server); + } + item->_status = SyncFileItem::Status::NormalError; _discoveryData->_anotherSyncNeeded = true; } From f677fda0717c83e8c255f1ba501847f3ba1ede4e Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Sun, 4 May 2025 10:28:42 +0200 Subject: [PATCH 10/11] fix(quota): use _filesNeedingScheduledSync to delay sync of files exceeding quota. Signed-off-by: Camila Ayres --- src/libsync/discovery.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 97d73e1a943ba..58ebd4b43123a 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -31,6 +31,7 @@ namespace constexpr const char *editorNamesForDelayedUpload[] = {"PowerPDF"}; constexpr const char *fileExtensionsToCheckIfOpenForSigning[] = {".pdf"}; constexpr auto delayIntervalForSyncRetryForOpenedForSigningFilesSeconds = 60; +constexpr auto delayIntervalForSyncRetryForFilesExceedQuotaSeconds = 60; } namespace OCC { @@ -1175,6 +1176,7 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( item->_status = SyncFileItem::Status::NormalError; _discoveryData->_anotherSyncNeeded = true; + _discoveryData->_filesNeedingScheduledSync.insert(path._original, delayIntervalForSyncRetryForFilesExceedQuotaSeconds); } if (item->_type != CSyncEnums::ItemTypeVirtualFile) { From 80f787979ba3e82b2a6cc94ecb2697fe1007554f Mon Sep 17 00:00:00 2001 From: Camila Ayres Date: Tue, 8 Apr 2025 18:43:32 +0200 Subject: [PATCH 11/11] refactor(quota): use const; make logs less verbose. - Add {} to single line if. Signed-off-by: Camila Ayres --- src/libsync/discovery.cpp | 26 +++++++++++++++----------- src/libsync/discovery.h | 2 +- src/libsync/discoveryphase.h | 2 +- src/libsync/propagateupload.cpp | 3 ++- src/libsync/syncengine.cpp | 3 ++- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/libsync/discovery.cpp b/src/libsync/discovery.cpp index 58ebd4b43123a..8135edff975f6 100644 --- a/src/libsync/discovery.cpp +++ b/src/libsync/discovery.cpp @@ -580,6 +580,7 @@ void ProcessDirectoryJob::processFile(PathTuple path, << " | (db/local/remote)" << " | valid: " << dbEntry.isValid() << "/" << hasLocal << "/" << hasServer << " | mtime: " << dbEntry._modtime << "/" << localEntry.modtime << "/" << serverEntry.modtime + << " | size: " << dbEntry._fileSize << "/" << localEntry.size << "/" << serverEntry.size << " | etag: " << dbEntry._etag << "//" << serverEntry.etag << " | checksum: " << dbEntry._checksumHeader << "//" << serverEntry.checksumHeader << " | perm: " << dbEntry._remotePerm << "//" << serverEntry.remotePerm @@ -591,10 +592,7 @@ void ProcessDirectoryJob::processFile(PathTuple path, << " | file lock: " << localFileIsLocked << "//" << serverFileIsLocked << " | file lock type: " << localFileLockType << "//" << serverFileLockType << " | live photo: " << dbEntry._isLivePhoto << "//" << serverEntry.isLivePhoto - << " | metadata missing: /" << localEntry.isMetadataMissing << '/' - << " | size: " << dbEntry._fileSize << "/" << localEntry.size << "/" << serverEntry.size - << " | quota used: //" << serverEntry.folderQuota.bytesUsed - << " | quota available: //" << serverEntry.folderQuota.bytesAvailable; + << " | metadata missing: /" << localEntry.isMetadataMissing << '/'; qCInfo(lcDisco).nospace() << processingLog; @@ -1080,6 +1078,7 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(const SyncFileItemPtr &it int64_t ProcessDirectoryJob::folderQuotaAvailable(const SyncFileItemPtr &item) { + const auto unlimitedFreeSpace = -3; if (const auto isNewItem = item->_instruction == CSYNC_INSTRUCTION_NEW; _queryServer != QueryMode::InBlackList && _queryServer != QueryMode::ParentDontExist && !item->isDirectory() @@ -1087,26 +1086,28 @@ int64_t ProcessDirectoryJob::folderQuotaAvailable(const SyncFileItemPtr &item) && item->_size > 0 && (item->_instruction == CSYNC_INSTRUCTION_SYNC || isNewItem)) { + const auto unknownFreeSpace = -2; + const auto bytesAvailable = _folderQuota.bytesAvailable; + if (!_dirItem) { - return _folderQuota.bytesAvailable; + return bytesAvailable; } const auto isDirItemRenamed = _dirItem && _dirItem->_instruction == CSYNC_INSTRUCTION_RENAME; - // quota is unknown at this point - follow server values: -2: Unknown free space + // quota is unknown at this point if (isDirItemRenamed && isNewItem) { - return -2; + return unknownFreeSpace; } if (isDirItemRenamed) { - return _folderQuota.bytesAvailable; + return bytesAvailable; } return _dirItem->_folderQuota.bytesAvailable; } - // -3: Unlimited free space - return -3; + return unlimitedFreeSpace; } void ProcessDirectoryJob::processFileAnalyzeLocalInfo( @@ -1164,6 +1165,9 @@ void ProcessDirectoryJob::processFileAnalyzeLocalInfo( } if (const auto folderQuota = folderQuotaAvailable(item);item->_size > folderQuota && folderQuota > -1) { + qCDebug(lcDisco) << "Folder" << item->_file + << "- quota used: " << serverEntry.folderQuota.bytesUsed + << "- quota available: " << serverEntry.folderQuota.bytesAvailable; item->_instruction = CSYNC_INSTRUCTION_ERROR; if (_currentFolder._server.isEmpty()) { item->_errorString = tr("Upload of %1 exceeds %2 of space left in personal files.").arg(Utility::octetsToString(item->_size), @@ -2264,7 +2268,7 @@ DiscoverySingleDirectoryJob *ProcessDirectoryJob::startAsyncServerQuery() return serverJob; } -void ProcessDirectoryJob::setFolderQuota(FolderQuota folderQuota) +void ProcessDirectoryJob::setFolderQuota(const FolderQuota &folderQuota) { _folderQuota.bytesUsed = folderQuota.bytesUsed; _folderQuota.bytesAvailable = folderQuota.bytesAvailable; diff --git a/src/libsync/discovery.h b/src/libsync/discovery.h index 2ac09ec433bae..cf441373c9e02 100644 --- a/src/libsync/discovery.h +++ b/src/libsync/discovery.h @@ -306,6 +306,6 @@ class ProcessDirectoryJob : public QObject void etag(const QByteArray &, const QDateTime &time); private slots: - void setFolderQuota(FolderQuota folderQuota); + void setFolderQuota(const FolderQuota &folderQuota); }; } diff --git a/src/libsync/discoveryphase.h b/src/libsync/discoveryphase.h index a2da3ca1d1d86..521967a8dd003 100644 --- a/src/libsync/discoveryphase.h +++ b/src/libsync/discoveryphase.h @@ -180,7 +180,7 @@ class DiscoverySingleDirectoryJob : public QObject void firstDirectoryPermissions(OCC::RemotePermissions); void etag(const QByteArray &, const QDateTime &time); void finished(const OCC::HttpResult> &result); - void setfolderQuota(FolderQuota folderQuota); + void setfolderQuota(const FolderQuota &folderQuota); private slots: void directoryListingIteratedSlot(const QString &, const QMap &); diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp index 9ca1a3b713bc2..7e878c476c232 100644 --- a/src/libsync/propagateupload.cpp +++ b/src/libsync/propagateupload.cpp @@ -753,8 +753,9 @@ void PropagateUploadFileCommon::slotJobDestroyed(QObject *job) // This function is used whenever there is an error occurring and jobs might be in progress void PropagateUploadFileCommon::abortWithError(SyncFileItem::Status status, const QString &error) { - if (_aborting) + if (_aborting) { return; + } abort(AbortType::Synchronous); done(status, error); } diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 47c98444390a7..23dd4b9f26b80 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -1442,8 +1442,9 @@ void SyncEngine::slotInsufficientLocalStorage() void SyncEngine::slotInsufficientRemoteStorage() { auto msg = tr("There is insufficient space available on the server for some uploads."); - if (_uniqueErrors.contains(msg)) + if (_uniqueErrors.contains(msg)) { return; + } _uniqueErrors.insert(msg); emit syncError(msg, ErrorCategory::InsufficientRemoteStorage);