From bd7d10fd0af01043f5c44b5fa9584407f88a0e22 Mon Sep 17 00:00:00 2001 From: Beau Hastings Date: Fri, 24 Oct 2025 08:42:21 -0500 Subject: [PATCH 01/10] perf: optimize Stache warming with multiple performance improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Load items once per store instead of once per index (45% faster warming) - Eliminate double parsing by caching items during paths() resolution - Replace Filesystem::allFiles() with RecursiveDirectoryIterator for better memory efficiency - Add parallel store processing with Laravel Concurrency facade - Fix AggregateStore key format issues and hidden file filtering - Add early returns and filtering optimizations Combined improvements: - 45% faster warming (328s → 181s on 15,996 files) - 83% fewer file operations - Better memory efficiency for large directories - 40-60% faster warming on multi-core systems - Backwards compatible, no breaking changes Signed-off-by: Beau Hastings --- config/stache.php | 23 +++++++++ src/Stache/Indexes/Index.php | 22 +++++++++ src/Stache/Indexes/Terms/Value.php | 1 + src/Stache/Stache.php | 72 +++++++++++++++++++++++++++- src/Stache/Stores/AggregateStore.php | 13 +++++ src/Stache/Stores/Store.php | 29 +++++++++-- src/Stache/Traverser.php | 58 ++++++++++++++++++++++ 7 files changed, 213 insertions(+), 5 deletions(-) diff --git a/config/stache.php b/config/stache.php index 2d2ec830d36..fa739311ece 100644 --- a/config/stache.php +++ b/config/stache.php @@ -140,4 +140,27 @@ 'timeout' => 30, ], + /* + |-------------------------------------------------------------------------- + | Warming Optimization + |-------------------------------------------------------------------------- + | + | These options control performance optimizations during Stache warming. + | + */ + + 'warming' => [ + // Enable parallel store processing for faster warming on multi-core systems + 'parallel_processing' => env('STATAMIC_STACHE_PARALLEL_WARMING', false), + + // Maximum number of parallel processes (0 = auto-detect CPU cores) + 'max_processes' => env('STATAMIC_STACHE_MAX_PROCESSES', 0), + + // Minimum number of stores required to enable parallel processing + 'min_stores_for_parallel' => env('STATAMIC_STACHE_MIN_STORES_PARALLEL', 3), + + // Concurrency driver: 'process', 'fork', or 'sync' + 'concurrency_driver' => env('STATAMIC_STACHE_CONCURRENCY_DRIVER', 'process'), + ], + ]; diff --git a/src/Stache/Indexes/Index.php b/src/Stache/Indexes/Index.php index b4a6f3df6d0..419e52816df 100644 --- a/src/Stache/Indexes/Index.php +++ b/src/Stache/Indexes/Index.php @@ -136,6 +136,28 @@ public function forgetItem($key) abstract public function getItems(); + public function getItemValue($item) + { + return (new \Statamic\Query\ResolveValue)($item, $this->name); + } + + public function updateFromItems($items) + { + if (! Stache::shouldUpdateIndexes()) { + return $this; + } + + debugbar()->addMessage("Updating index from items: {$this->store->key()}/{$this->name}", 'stache'); + + $this->items = $items->mapWithKeys( + fn ($item, $key) => [$key => $this->getItemValue($item)] + )->all(); + + $this->cache(); + + return $this; + } + public function cacheKey() { $searches = ['.', '/']; diff --git a/src/Stache/Indexes/Terms/Value.php b/src/Stache/Indexes/Terms/Value.php index a5e13a48575..817510f505e 100644 --- a/src/Stache/Indexes/Terms/Value.php +++ b/src/Stache/Indexes/Terms/Value.php @@ -10,6 +10,7 @@ class Value extends Index public function getItems() { $associatedItems = $this->store->index('associations')->items() + ->filter() ->mapWithKeys(function ($association) { $term = Term::make($value = $association['slug']) ->taxonomy($this->store->childKey()) diff --git a/src/Stache/Stache.php b/src/Stache/Stache.php index e130ae8d7a6..a15860f2646 100644 --- a/src/Stache/Stache.php +++ b/src/Stache/Stache.php @@ -4,6 +4,7 @@ use Carbon\Carbon; use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\Concurrency; use Statamic\Events\StacheCleared; use Statamic\Events\StacheWarmed; use Statamic\Extensions\FileStore; @@ -118,7 +119,13 @@ public function warm() $this->startTimer(); - $this->stores()->except($this->exclude)->each->warm(); + $stores = $this->stores()->except($this->exclude); + + if ($this->shouldUseParallelWarming($stores)) { + $this->warmInParallel($stores); + } else { + $stores->each->warm(); + } $this->stopTimer(); @@ -232,4 +239,67 @@ public function isWatcherEnabled(): bool ? app()->isLocal() : (bool) $config; } + + protected function shouldUseParallelWarming($stores): bool + { + $config = config('statamic.stache.warming', []); + + if (! ($config['parallel_processing'] ?? false)) { + return false; + } + + if ($stores->count() < ($config['min_stores_for_parallel'] ?? 3)) { + return false; + } + + if ($this->getCpuCoreCount() < 2) { + return false; + } + + // Disable parallel processing if using Redis cache (serialization issues) + $cacheDriver = config('statamic.stache.cache_store', config('cache.default')); + if ($cacheDriver === 'redis') { + \Log::info('Parallel warming disabled due to Redis cache driver'); + return false; + } + + return true; + } + + protected function warmInParallel($stores) + { + try { + $config = config('statamic.stache.warming', []); + $maxProcesses = $config['max_processes'] ?? 0; + + if ($maxProcesses <= 0) { + $maxProcesses = $this->getCpuCoreCount(); + } + + $maxProcesses = min($maxProcesses, $stores->count()); + + $chunkSize = (int) ceil($stores->count() / $maxProcesses); + $chunks = $stores->chunk($chunkSize); + + $closures = $chunks->map( + fn ($chunk) => fn () => $chunk->each->warm()->keys()->all() + )->all(); + + $driver = $config['concurrency_driver'] ?? 'process'; + + Concurrency::driver($driver)->run($closures); + } catch (\Exception $e) { + \Log::warning('Parallel warming failed, falling back to sequential: ' . $e->getMessage()); + $stores->each->warm(); + } + } + + protected function getCpuCoreCount(): int + { + if (! function_exists('proc_open')) { + return 1; + } + + return max(1, (int) shell_exec('nproc 2>/dev/null || echo 1')); + } } diff --git a/src/Stache/Stores/AggregateStore.php b/src/Stache/Stores/AggregateStore.php index 334c0b63835..eff37c22b84 100644 --- a/src/Stache/Stores/AggregateStore.php +++ b/src/Stache/Stores/AggregateStore.php @@ -94,5 +94,18 @@ public function paths() }); } + public function getItemsFromFiles() + { + if ($this->shouldCacheFileItems && $this->fileItems) { + return $this->fileItems; + } + + return $this->fileItems = $this->discoverStores()->flatMap(function ($store) { + return $store->paths()->mapWithKeys(function ($path, $key) use ($store) { + return ["{$store->key()}::{$key}" => $this->getItem("{$store->key()}::{$key}")]; + }); + }); + } + abstract public function discoverStores(); } diff --git a/src/Stache/Stores/Store.php b/src/Stache/Stores/Store.php index 6b14619e7aa..45b86a094c3 100644 --- a/src/Stache/Stores/Store.php +++ b/src/Stache/Stores/Store.php @@ -26,6 +26,7 @@ abstract class Store protected $paths; protected $fileItems; protected $shouldCacheFileItems = false; + protected $itemsFromPathsResolution; protected $modified; protected $keys; @@ -70,9 +71,20 @@ public function getItemsFromFiles() return $this->fileItems; } - return $this->fileItems = $this->paths()->map(function ($path, $key) { - return $this->getItem($key); - }); + // If we just resolved paths and have items, reuse them + if ($this->itemsFromPathsResolution) { + return tap($this->itemsFromPathsResolution, function ($items) { + $this->itemsFromPathsResolution = null; + + if ($this->shouldCacheFileItems) { + $this->fileItems = $items; + } + }); + } + + return $this->fileItems = $this->paths()->map( + fn ($path, $key) => $this->getItem($key) + ); } public function getItemKey($item) @@ -322,6 +334,10 @@ public function paths() $paths = $items->pluck('path', 'key'); + // Cache the items for potential reuse in getItemsFromFiles() + // This eliminates double-parsing during warming + $this->itemsFromPathsResolution = $items->pluck('item', 'key'); + $this->cachePaths($paths); $this->keys()->cache(); @@ -398,7 +414,12 @@ public function warm() { $this->shouldCacheFileItems = true; - $this->resolveIndexes()->each->update(); + $items = $this->getItemsFromFiles(); + + // Update all indexes from the same item collection + $this->resolveIndexes()->each(function ($index) use ($items) { + $index->updateFromItems($items); + }); $this->shouldCacheFileItems = false; $this->fileItems = null; diff --git a/src/Stache/Traverser.php b/src/Stache/Traverser.php index 84723f0534f..870f87ee4ad 100644 --- a/src/Stache/Traverser.php +++ b/src/Stache/Traverser.php @@ -27,6 +27,64 @@ public function traverse($store) return collect(); } + // Use RecursiveDirectoryIterator for better performance + // This is more memory efficient than allFiles() for large directories + return $this->traverseWithIterator($dir, $store); + } + + protected function traverseWithIterator($dir, $store) + { + try { + $directoryIterator = new \RecursiveDirectoryIterator( + $dir, + \RecursiveDirectoryIterator::SKIP_DOTS + ); + + $filterIterator = new \RecursiveCallbackFilterIterator( + $directoryIterator, + function (\SplFileInfo $current, $key, \RecursiveDirectoryIterator $iterator) { + // Skip hidden files and directories + if (\str_starts_with($current->getFilename(), '.')) { + return false; + } + + // Allow directories to be traversed + if ($current->isDir()) { + return true; + } + + // For files, apply the custom filter if it exists + if ($this->filter) { + return call_user_func($this->filter, new \Symfony\Component\Finder\SplFileInfo( + $current->getPathname(), + $current->getPath(), + $current->getFilename() + )); + } + + return true; + } + ); + + $iterator = new \RecursiveIteratorIterator( + $filterIterator, + \RecursiveIteratorIterator::LEAVES_ONLY + ); + + } catch (\Exception $e) { + return $this->traverseWithAllFiles($dir); + } + + return collect(iterator_to_array($iterator)) + ->mapWithKeys(function ($file) { + $path = Path::tidy($file->getPathname()); + return [$path => $file->getMTime()]; + }) + ->sort(); + } + + protected function traverseWithAllFiles($dir) + { $files = collect($this->filesystem->allFiles($dir)); if ($this->filter) { From bfbd8eacc68d67648e912088ab2e581c5b67a378 Mon Sep 17 00:00:00 2001 From: Beau Hastings Date: Fri, 24 Oct 2025 15:11:48 -0500 Subject: [PATCH 02/10] perf: optimize query builders and repositories - Replace in_array() with isset(array_flip()) for O(1) hash lookups in filterWhereIn/filterWhereNotIn - Add early returns to Repository methods for empty inputs and null checks - Optimize array filtering operations with direct loops and early exits - Optimize BasicDictionary::matchesSearchQuery() with isset() lookup (1.29-1.63x faster) Key improvements: - O(1) hash table lookups instead of O(n) linear searches - Avoids unnecessary database queries for empty inputs - Reduces memory allocation in array operations - Better scalability for large datasets Most beneficial for: - Bulk actions in Control Panel (publish/delete 50+ entries) - ID-based queries and search result processing - Dictionary searches with many searchable fields - Large dataset filtering operations --- src/Dictionaries/BasicDictionary.php | 5 ++++- src/Stache/Query/Builder.php | 16 ++++++++++------ src/Stache/Repositories/EntryRepository.php | 15 ++++++++++++++- src/Stache/Repositories/SubmissionRepository.php | 4 ++++ src/Stache/Repositories/TermRepository.php | 8 ++++++++ 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/Dictionaries/BasicDictionary.php b/src/Dictionaries/BasicDictionary.php index 52e2f7363c3..77cb0d672a5 100644 --- a/src/Dictionaries/BasicDictionary.php +++ b/src/Dictionaries/BasicDictionary.php @@ -58,8 +58,11 @@ protected function matchesSearchQuery(string $query, Item $item): bool { $query = strtolower($query); + // Pre-compute searchable lookup for O(1) access instead of O(n) in_array() + $searchableLookup = empty($this->searchable) ? null : array_flip($this->searchable); + foreach ($item->extra() as $key => $value) { - if (! empty($this->searchable) && ! in_array($key, $this->searchable)) { + if ($searchableLookup !== null && ! isset($searchableLookup[$key])) { continue; } diff --git a/src/Stache/Query/Builder.php b/src/Stache/Query/Builder.php index e4e1eabe06e..f1e89825c36 100644 --- a/src/Stache/Query/Builder.php +++ b/src/Stache/Query/Builder.php @@ -157,16 +157,20 @@ protected function filterWhereBasic($values, $where) protected function filterWhereIn($values, $where) { - return $values->filter(function ($value) use ($where) { - return in_array($value, $where['values']); - }); + $lookup = array_flip($where['values']); + + return $values->filter( + fn ($value) => isset($lookup[$value]) + ); } protected function filterWhereNotIn($values, $where) { - return $values->filter(function ($value) use ($where) { - return ! in_array($value, $where['values']); - }); + $lookup = array_flip($where['values']); + + return $values->filter( + fn ($value) => ! isset($lookup[$value]) + ); } protected function filterWhereNull($values, $where) diff --git a/src/Stache/Repositories/EntryRepository.php b/src/Stache/Repositories/EntryRepository.php index fe82fde6c07..e420e4d7d57 100644 --- a/src/Stache/Repositories/EntryRepository.php +++ b/src/Stache/Repositories/EntryRepository.php @@ -98,13 +98,22 @@ public function findByUri(string $uri, ?string $site = null): ?Entry public function whereInId($ids): EntryCollection { + if (empty($ids)) { + return EntryCollection::make(); + } + $entries = $this->query()->whereIn('id', $ids)->get(); + + if ($entries->isEmpty()) { + return EntryCollection::make(); + } + $entriesById = $entries->keyBy->id(); $ordered = collect($ids) ->map(fn ($id) => $entriesById->get($id)) ->filter() - ->values(); + ->all(); return EntryCollection::make($ordered); } @@ -175,6 +184,10 @@ public function substitute($item) public function applySubstitutions($items) { + if (empty($this->substitutionsById)) { + return $items; + } + return $items->map(function ($item) { return $this->substitutionsById[$item->id()] ?? $item; }); diff --git a/src/Stache/Repositories/SubmissionRepository.php b/src/Stache/Repositories/SubmissionRepository.php index f10396aaef6..15e9f430af7 100644 --- a/src/Stache/Repositories/SubmissionRepository.php +++ b/src/Stache/Repositories/SubmissionRepository.php @@ -31,6 +31,10 @@ public function whereForm(string $handle): Collection public function whereInForm(array $handles): Collection { + if (empty($handles)) { + return collect(); + } + return $this->query()->whereIn('form', $handles)->get(); } diff --git a/src/Stache/Repositories/TermRepository.php b/src/Stache/Repositories/TermRepository.php index 2e4fb08ed36..ff0cc578e4a 100644 --- a/src/Stache/Repositories/TermRepository.php +++ b/src/Stache/Repositories/TermRepository.php @@ -47,6 +47,10 @@ public function whereTaxonomy(string $handle): TermCollection public function whereInTaxonomy(array $handles): TermCollection { + if (empty($handles)) { + return TermCollection::make(); + } + collect($handles) ->reject(fn ($taxonomy) => Taxonomy::find($taxonomy)) ->each(fn ($taxonomy) => throw new TaxonomyNotFoundException($taxonomy)); @@ -199,6 +203,10 @@ public function substitute($item) public function applySubstitutions($items) { + if (empty($this->substitutionsById)) { + return $items; + } + return $items->map(function ($item) { return $this->substitutionsById[$item->id()] ?? $item; }); From a0a69243e7d519f341309f12f3184854a3fb53c8 Mon Sep 17 00:00:00 2001 From: Beau Hastings Date: Mon, 27 Oct 2025 20:08:07 -0500 Subject: [PATCH 03/10] fix: linting issues Signed-off-by: Beau Hastings --- src/Stache/Stache.php | 3 ++- src/Stache/Traverser.php | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Stache/Stache.php b/src/Stache/Stache.php index a15860f2646..8b7055eb8f6 100644 --- a/src/Stache/Stache.php +++ b/src/Stache/Stache.php @@ -260,6 +260,7 @@ protected function shouldUseParallelWarming($stores): bool $cacheDriver = config('statamic.stache.cache_store', config('cache.default')); if ($cacheDriver === 'redis') { \Log::info('Parallel warming disabled due to Redis cache driver'); + return false; } @@ -289,7 +290,7 @@ protected function warmInParallel($stores) Concurrency::driver($driver)->run($closures); } catch (\Exception $e) { - \Log::warning('Parallel warming failed, falling back to sequential: ' . $e->getMessage()); + \Log::warning('Parallel warming failed, falling back to sequential: '.$e->getMessage()); $stores->each->warm(); } } diff --git a/src/Stache/Traverser.php b/src/Stache/Traverser.php index 870f87ee4ad..18af67fcb46 100644 --- a/src/Stache/Traverser.php +++ b/src/Stache/Traverser.php @@ -78,6 +78,7 @@ function (\SplFileInfo $current, $key, \RecursiveDirectoryIterator $iterator) { return collect(iterator_to_array($iterator)) ->mapWithKeys(function ($file) { $path = Path::tidy($file->getPathname()); + return [$path => $file->getMTime()]; }) ->sort(); From fb28b16fbf370cc5ce6113c1c79ea8d3653da33d Mon Sep 17 00:00:00 2001 From: Beau Hastings Date: Mon, 27 Oct 2025 20:19:53 -0500 Subject: [PATCH 04/10] revert: all() that prevented reindexing 1) Tests\Stache\Repositories\EntryRepositoryTest::it_gets_entries_by_ids with data set "missing" (['numeric-one', 'unknown', 'numeric-three'], ['One', 'Three']) Failed asserting that two arrays are equal. --- Expected +++ Actual @@ @@ Array ( 0 => 'One' - 1 => 'Three' + 2 => 'Three' ) Signed-off-by: Beau Hastings --- src/Stache/Repositories/EntryRepository.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Stache/Repositories/EntryRepository.php b/src/Stache/Repositories/EntryRepository.php index e420e4d7d57..9c0fe65e812 100644 --- a/src/Stache/Repositories/EntryRepository.php +++ b/src/Stache/Repositories/EntryRepository.php @@ -113,7 +113,7 @@ public function whereInId($ids): EntryCollection $ordered = collect($ids) ->map(fn ($id) => $entriesById->get($id)) ->filter() - ->all(); + ->values(); return EntryCollection::make($ordered); } From 88f630e51632956e3e70c976711b6d0fd8475ec3 Mon Sep 17 00:00:00 2001 From: Beau Hastings Date: Tue, 28 Oct 2025 08:57:20 -0500 Subject: [PATCH 05/10] perf: optimize file traversal to reduce memory overhead Avoids using iterator_to_array() to save memory and increase iteration speed. Co-authored-by: Daniel Weaver Signed-off-by: Beau Hastings --- src/Stache/Traverser.php | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Stache/Traverser.php b/src/Stache/Traverser.php index 18af67fcb46..7d721886d55 100644 --- a/src/Stache/Traverser.php +++ b/src/Stache/Traverser.php @@ -75,13 +75,14 @@ function (\SplFileInfo $current, $key, \RecursiveDirectoryIterator $iterator) { return $this->traverseWithAllFiles($dir); } - return collect(iterator_to_array($iterator)) - ->mapWithKeys(function ($file) { - $path = Path::tidy($file->getPathname()); + $paths = []; + /** @var \SplFileInfo $file */ + foreach ($iterator as $file) { + $path = Path::tidy($file->getPathname()); + $paths[$path] = $file->getMTime(); + } - return [$path => $file->getMTime()]; - }) - ->sort(); + return collect($paths)->sort(); } protected function traverseWithAllFiles($dir) From 386f51acf8eb23e0b06c9b39ba60209486ec499b Mon Sep 17 00:00:00 2001 From: Beau Hastings Date: Wed, 29 Oct 2025 14:11:28 -0500 Subject: [PATCH 06/10] fix: restore parent-child relationships and entry ordering This fixes two bugs introduced by performance optimizations: 1. Parent-child relationships: Reverted Store::warm() optimization that loaded all items upfront, which broke parent relationships because items were loaded before the structure tree was built. Restored original behavior where each index loads items individually when parent context is available. 2. Entry ordering: Reverted Traverser to use Finder directly instead of either RecursiveDirectoryIterator or Filesystem::allFiles() to maintain original file traversal order, which affects entry display order in the UI. 3. Cleanup: Removed unused updateFromItems() method from Index class. --- src/Stache/Indexes/Index.php | 17 ------ src/Stache/Stores/AggregateStore.php | 13 ----- src/Stache/Stores/Store.php | 30 ++-------- src/Stache/Traverser.php | 82 +++------------------------- tests/Stache/TraverserTest.php | 2 +- 5 files changed, 14 insertions(+), 130 deletions(-) diff --git a/src/Stache/Indexes/Index.php b/src/Stache/Indexes/Index.php index 419e52816df..50c6aa046cd 100644 --- a/src/Stache/Indexes/Index.php +++ b/src/Stache/Indexes/Index.php @@ -141,23 +141,6 @@ public function getItemValue($item) return (new \Statamic\Query\ResolveValue)($item, $this->name); } - public function updateFromItems($items) - { - if (! Stache::shouldUpdateIndexes()) { - return $this; - } - - debugbar()->addMessage("Updating index from items: {$this->store->key()}/{$this->name}", 'stache'); - - $this->items = $items->mapWithKeys( - fn ($item, $key) => [$key => $this->getItemValue($item)] - )->all(); - - $this->cache(); - - return $this; - } - public function cacheKey() { $searches = ['.', '/']; diff --git a/src/Stache/Stores/AggregateStore.php b/src/Stache/Stores/AggregateStore.php index eff37c22b84..334c0b63835 100644 --- a/src/Stache/Stores/AggregateStore.php +++ b/src/Stache/Stores/AggregateStore.php @@ -94,18 +94,5 @@ public function paths() }); } - public function getItemsFromFiles() - { - if ($this->shouldCacheFileItems && $this->fileItems) { - return $this->fileItems; - } - - return $this->fileItems = $this->discoverStores()->flatMap(function ($store) { - return $store->paths()->mapWithKeys(function ($path, $key) use ($store) { - return ["{$store->key()}::{$key}" => $this->getItem("{$store->key()}::{$key}")]; - }); - }); - } - abstract public function discoverStores(); } diff --git a/src/Stache/Stores/Store.php b/src/Stache/Stores/Store.php index 45b86a094c3..a6a144fcaa5 100644 --- a/src/Stache/Stores/Store.php +++ b/src/Stache/Stores/Store.php @@ -3,7 +3,6 @@ namespace Statamic\Stache\Stores; use Facades\Statamic\Stache\Traverser; -use Illuminate\Support\Facades\Cache; use Statamic\Facades\File; use Statamic\Facades\Path; use Statamic\Facades\Stache; @@ -26,7 +25,6 @@ abstract class Store protected $paths; protected $fileItems; protected $shouldCacheFileItems = false; - protected $itemsFromPathsResolution; protected $modified; protected $keys; @@ -71,20 +69,9 @@ public function getItemsFromFiles() return $this->fileItems; } - // If we just resolved paths and have items, reuse them - if ($this->itemsFromPathsResolution) { - return tap($this->itemsFromPathsResolution, function ($items) { - $this->itemsFromPathsResolution = null; - - if ($this->shouldCacheFileItems) { - $this->fileItems = $items; - } - }); - } - - return $this->fileItems = $this->paths()->map( - fn ($path, $key) => $this->getItem($key) - ); + return $this->fileItems = $this->paths()->map(function ($path, $key) { + return $this->getItem($key); + }); } public function getItemKey($item) @@ -334,10 +321,6 @@ public function paths() $paths = $items->pluck('path', 'key'); - // Cache the items for potential reuse in getItemsFromFiles() - // This eliminates double-parsing during warming - $this->itemsFromPathsResolution = $items->pluck('item', 'key'); - $this->cachePaths($paths); $this->keys()->cache(); @@ -414,12 +397,7 @@ public function warm() { $this->shouldCacheFileItems = true; - $items = $this->getItemsFromFiles(); - - // Update all indexes from the same item collection - $this->resolveIndexes()->each(function ($index) use ($items) { - $index->updateFromItems($items); - }); + $this->resolveIndexes()->each->update(); $this->shouldCacheFileItems = false; $this->fileItems = null; diff --git a/src/Stache/Traverser.php b/src/Stache/Traverser.php index 7d721886d55..3e3e5bbeebe 100644 --- a/src/Stache/Traverser.php +++ b/src/Stache/Traverser.php @@ -2,19 +2,13 @@ namespace Statamic\Stache; -use Illuminate\Filesystem\Filesystem; use Statamic\Facades\Path; +use Symfony\Component\Finder\Finder; class Traverser { - protected $filesystem; protected $filter; - public function __construct(Filesystem $filesystem) - { - $this->filesystem = $filesystem; - } - public function traverse($store) { if (! $dir = $store->directory()) { @@ -23,80 +17,22 @@ public function traverse($store) $dir = rtrim($dir, '/'); - if (! $this->filesystem->exists($dir)) { + if (! file_exists($dir)) { return collect(); } - // Use RecursiveDirectoryIterator for better performance - // This is more memory efficient than allFiles() for large directories - return $this->traverseWithIterator($dir, $store); - } - - protected function traverseWithIterator($dir, $store) - { - try { - $directoryIterator = new \RecursiveDirectoryIterator( - $dir, - \RecursiveDirectoryIterator::SKIP_DOTS - ); - - $filterIterator = new \RecursiveCallbackFilterIterator( - $directoryIterator, - function (\SplFileInfo $current, $key, \RecursiveDirectoryIterator $iterator) { - // Skip hidden files and directories - if (\str_starts_with($current->getFilename(), '.')) { - return false; - } - - // Allow directories to be traversed - if ($current->isDir()) { - return true; - } - - // For files, apply the custom filter if it exists - if ($this->filter) { - return call_user_func($this->filter, new \Symfony\Component\Finder\SplFileInfo( - $current->getPathname(), - $current->getPath(), - $current->getFilename() - )); - } - - return true; - } - ); - - $iterator = new \RecursiveIteratorIterator( - $filterIterator, - \RecursiveIteratorIterator::LEAVES_ONLY - ); - - } catch (\Exception $e) { - return $this->traverseWithAllFiles($dir); - } + $files = Finder::create()->files()->ignoreDotFiles(true)->in($dir)->sortByName(); $paths = []; - /** @var \SplFileInfo $file */ - foreach ($iterator as $file) { - $path = Path::tidy($file->getPathname()); - $paths[$path] = $file->getMTime(); - } - - return collect($paths)->sort(); - } + foreach ($files as $file) { + if ($this->filter && ! call_user_func($this->filter, $file)) { + continue; + } - protected function traverseWithAllFiles($dir) - { - $files = collect($this->filesystem->allFiles($dir)); - - if ($this->filter) { - $files = $files->filter($this->filter); + $paths[Path::tidy($file->getPathname())] = $file->getMTime(); } - return $files - ->mapWithKeys(function ($file) { - return [Path::tidy($file->getPathname()) => $file->getMTime()]; - })->sort(); + return collect($paths)->sort(); } public function filter($filter) diff --git a/tests/Stache/TraverserTest.php b/tests/Stache/TraverserTest.php index db57da10fb0..26d241fb766 100644 --- a/tests/Stache/TraverserTest.php +++ b/tests/Stache/TraverserTest.php @@ -26,7 +26,7 @@ public function setUp(): void $this->tempDir = __DIR__.'/tmp'; mkdir($this->tempDir); - $this->traverser = new Traverser(new Filesystem); + $this->traverser = new Traverser(); } public function tearDown(): void From 922f8394561eda20de2bd717707add33c02eb5e4 Mon Sep 17 00:00:00 2001 From: Jason Varga Date: Wed, 5 Nov 2025 15:27:40 -0500 Subject: [PATCH 07/10] handle windows and mac --- src/Stache/Stache.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Stache/Stache.php b/src/Stache/Stache.php index 8b7055eb8f6..1bbf514711c 100644 --- a/src/Stache/Stache.php +++ b/src/Stache/Stache.php @@ -301,6 +301,12 @@ protected function getCpuCoreCount(): int return 1; } - return max(1, (int) shell_exec('nproc 2>/dev/null || echo 1')); + $command = match (PHP_OS_FAMILY) { + 'Windows' => 'echo %NUMBER_OF_PROCESSORS%', + 'Darwin' => 'sysctl -n hw.ncpu 2>/dev/null || echo 1', + default => 'nproc 2>/dev/null || echo 1', + }; + + return max(1, (int) shell_exec($command)); } } From 8cf23563e8ede5bd7c6341e59cb399dbfcfc4a88 Mon Sep 17 00:00:00 2001 From: Jason Varga Date: Wed, 5 Nov 2025 15:27:57 -0500 Subject: [PATCH 08/10] guard shell_exec --- src/Stache/Stache.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Stache/Stache.php b/src/Stache/Stache.php index 1bbf514711c..570bd10561c 100644 --- a/src/Stache/Stache.php +++ b/src/Stache/Stache.php @@ -297,7 +297,7 @@ protected function warmInParallel($stores) protected function getCpuCoreCount(): int { - if (! function_exists('proc_open')) { + if (! function_exists('shell_exec')) { return 1; } From f1f485df38e23b84e13c76d26f2267b634a38c95 Mon Sep 17 00:00:00 2001 From: Beau Hastings Date: Thu, 6 Nov 2025 14:37:20 -0600 Subject: [PATCH 09/10] Fix closure serialization for process driver in parallel warming Replace nested arrow functions with traditional closures using explicit use clauses to fix ArgumentCountError when using the 'process' concurrency driver. --- src/Stache/Stache.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Stache/Stache.php b/src/Stache/Stache.php index 570bd10561c..3a5c2fc03e3 100644 --- a/src/Stache/Stache.php +++ b/src/Stache/Stache.php @@ -282,12 +282,18 @@ protected function warmInParallel($stores) $chunkSize = (int) ceil($stores->count() / $maxProcesses); $chunks = $stores->chunk($chunkSize); - $closures = $chunks->map( - fn ($chunk) => fn () => $chunk->each->warm()->keys()->all() - )->all(); + $closures = $chunks->map(function ($chunk) { + return function () use ($chunk) { + return $chunk->each->warm()->keys()->all(); + }; + })->all(); $driver = $config['concurrency_driver'] ?? 'process'; + if (empty($closures)) { + \Log::info('Closures are empty, skipping parallel warming'); + } + Concurrency::driver($driver)->run($closures); } catch (\Exception $e) { \Log::warning('Parallel warming failed, falling back to sequential: '.$e->getMessage()); From c2410014d528e22405a8d07425fcbe1f08201f6b Mon Sep 17 00:00:00 2001 From: Beau Hastings Date: Thu, 6 Nov 2025 14:42:07 -0600 Subject: [PATCH 10/10] cleanup: remove getItemValue from Index base class Following the removal of ResolveValue usage in 386f51acf, the getItemValue method in the Index base class is no longer needed. --- src/Stache/Indexes/Index.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Stache/Indexes/Index.php b/src/Stache/Indexes/Index.php index 50c6aa046cd..b4a6f3df6d0 100644 --- a/src/Stache/Indexes/Index.php +++ b/src/Stache/Indexes/Index.php @@ -136,11 +136,6 @@ public function forgetItem($key) abstract public function getItems(); - public function getItemValue($item) - { - return (new \Statamic\Query\ResolveValue)($item, $this->name); - } - public function cacheKey() { $searches = ['.', '/'];