From f6d7b0624e115df79c406ef3256ec6c0d28cf8a5 Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Fri, 3 Jul 2026 17:53:38 -0400 Subject: [PATCH 1/2] Build the search index one package at a time The hourly search index regeneration (which also runs at startup) decoded every package in the database into memory at once before building the index. The decoded packages now need more memory than the server has, so the server was being OOM-killed several times an hour (over 1,600 times in the last 30 days), taking the site down for a couple of minutes each time. Instead, decode packages one at a time and retain only each package's index entries, filling in reverse dependency counts once all packages have been seen. Serving the full production data set now peaks around 1.1GB resident instead of 3.6GB+. Also: - Use both vCPUs and cap the heap below available RAM (-N2 -A64m -M3G) - Update deprecated GitHub Actions (cache/upload-artifact v2 no longer run) and pin the runner to ubuntu-24.04 to match the server OS --- .github/workflows/ci.yml | 24 ++++++------ CHANGELOG.md | 16 ++++++++ deploy/pursuit.service | 5 ++- pursuit.cabal | 2 +- src/Application.hs | 9 ++--- src/Handler/Database.hs | 34 ++++++++++++----- src/SearchIndex.hs | 80 +++++++++++++++++++++++++--------------- 7 files changed, 113 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1c1242..0e19234 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,19 +14,21 @@ jobs: build_server: name: Build server # Note that this must be kept in sync with the version of Ubuntu which the - # Pursit server is running, otherwise the server binary may fail to run. - runs-on: ubuntu-latest + # Pursuit server is running, otherwise the server binary may fail to run + # (the binary links against the runner's glibc, and a binary built against + # a newer glibc than the server's will not start). + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: haskell/actions/setup@v1 + - uses: haskell-actions/setup@v2 with: enable-stack: true - stack-version: "2.5.1" + stack-version: "latest" stack-no-global: true - name: Cache dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.stack key: ${{ runner.os }}-stack-${{ hashFiles('stack.yaml.lock') }}-${{ hashFiles('pursuit.cabal') }} @@ -49,7 +51,7 @@ jobs: tar czf pursuit.tar.gz -C ${{ env.BUNDLE_DIR }}/ . - name: Persist server assets - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: github.event_name == 'release' with: name: pursuit.tar.gz @@ -58,9 +60,7 @@ jobs: release: name: Release - # Note that this must be kept in sync with the version of Ubuntu which the - # Pursit server is running, otherwise the server binary may fail to run. - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: github.event_name == 'release' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -68,12 +68,12 @@ jobs: - build_server steps: - name: Retrieve server assets - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: pursuit.tar.gz - name: Upload assets - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: files: | pursuit.tar.gz diff --git a/CHANGELOG.md b/CHANGELOG.md index 241f78a..214ed64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ the most up-to-date version of this file. ## Unreleased +## v0.9.10 + +- Build the search index one package at a time (@thomashoneyman) + + Previously the hourly search index regeneration (which also runs at startup) + decoded every package in the database into memory at once before building + the index. The decoded packages now require more memory than the server has, + which caused the server to be OOM-killed several times an hour. Packages are + now decoded and indexed one at a time, so only the index entries themselves + are retained. + +- Use both CPU cores and cap the heap below available RAM (`-N2 -A64m -M3G`) + in `pursuit.service` (@thomashoneyman) +- Update GitHub Actions to current action versions and pin the runner to + `ubuntu-24.04` to match the server (@thomashoneyman) + ## v0.9.9 - Update pursuit.service to 3.5GB max memory (@thomashoneyman) diff --git a/deploy/pursuit.service b/deploy/pursuit.service index 8d3fb42..8963ca1 100644 --- a/deploy/pursuit.service +++ b/deploy/pursuit.service @@ -4,7 +4,10 @@ Description=Web service for hosting of PureScript API documentation [Service] Type=simple User=www-data -ExecStart=/usr/local/bin/pursuit +RTS -N1 -A128m -M3.5G -RTS +# The server has 2 vCPUs and 4GB of RAM. The heap limit needs to leave room +# for nginx and the rest of the system, so that the RTS runs out of heap +# before the kernel's OOM killer steps in. +ExecStart=/usr/local/bin/pursuit +RTS -N2 -A64m -M3G -RTS Restart=always RestartSec=5s Environment="PURSUIT_APPROOT=https://pursuit.purescript.org" diff --git a/pursuit.cabal b/pursuit.cabal index 24ac6af..a33cebb 100644 --- a/pursuit.cabal +++ b/pursuit.cabal @@ -1,5 +1,5 @@ name: pursuit -version: 0.9.9 +version: 0.9.10 cabal-version: >= 1.8 build-type: Simple license: MIT diff --git a/src/Application.hs b/src/Application.hs index b75f451..46c188b 100644 --- a/src/Application.hs +++ b/src/Application.hs @@ -40,7 +40,7 @@ import Handler.Packages import Handler.Search import Handler.PackageBadges import Handler.Help -import SearchIndex (emptySearchIndex, createSearchIndex, evalSearchIndex) +import SearchIndex (emptySearchIndex, evalSearchIndex) -- This line actually creates our YesodDispatch instance. It is the second half -- of the call to mkYesodData which occurs in Foundation.hs. Please see the @@ -71,17 +71,16 @@ makeFoundation appSettings = do let hour = 60 * 60 * 1000 * 1000 -- microseconds in every hour $ do let emptySessionMap = mempty :: SessionMap - pkgs <- Unsafe.runFakeHandler + searchIndex <- Unsafe.runFakeHandler emptySessionMap appLogger foundation - getAllPackages + createSearchIndexFromDatabase traverse ( atomically . writeTVar (appSearchIndex foundation) . withStrategy evalSearchIndex - . createSearchIndex - ) pkgs + ) searchIndex -- | Convert our foundation to a WAI Application by calling @toWaiAppPlain@ and -- applyng some additional middlewares. diff --git a/src/Handler/Database.hs b/src/Handler/Database.hs index 0c9f099..bb084f2 100644 --- a/src/Handler/Database.hs +++ b/src/Handler/Database.hs @@ -3,7 +3,7 @@ -- Pursuit). module Handler.Database ( getAllPackageNames - , getAllPackages + , createSearchIndexFromDatabase , getLatestPackages , lookupPackage , availableVersionsFor @@ -26,6 +26,7 @@ import qualified Language.PureScript.Docs as D import Handler.Utils import Handler.Caching (clearCache) +import SearchIndex (SearchIndex, PackageEntries, packageEntries, buildSearchIndex) getAllPackageNames :: Handler [PackageName] getAllPackageNames = do @@ -46,16 +47,31 @@ getLatestPackages = do withVersion :: PackageName -> Handler (Maybe (PackageName, Version)) withVersion name = (map . map) (name,) (getLatestVersionFor name) --- | This is horribly inefficient, but it will do for now. Note that this --- only gets the latest version of each package in the database. -getAllPackages :: Handler [D.VerifiedPackage] -getAllPackages = do +-- | Build the search index from the latest version of each package in the +-- database. Packages are read and decoded one at a time, and only the +-- (comparatively small) index entries for each package are retained: decoding +-- every package up front requires several gigabytes of memory, which is more +-- than the server has. +createSearchIndexFromDatabase :: Handler SearchIndex +createSearchIndexFromDatabase = do pkgNames <- getAllPackageNames - pkgNamesAndVersions <- catMaybes <$> traverse withVersion pkgNames - catMaybes <$> traverse lookupPackageMay pkgNamesAndVersions + entries <- catMaybes <$> traverse entriesFor pkgNames + return (buildSearchIndex entries) where - withVersion name = (map . map) (name,) (getLatestVersionFor name) - lookupPackageMay = map hush . uncurry lookupPackage + entriesFor :: PackageName -> Handler (Maybe PackageEntries) + entriesFor name = do + mversion <- getLatestVersionFor name + case mversion of + Nothing -> return Nothing + Just version -> do + mpkg <- hush <$> lookupPackage name version + case mpkg of + Nothing -> return Nothing + Just pkg -> do + -- Force the entries fully before moving on to the next package, + -- so that the decoded package can be garbage collected. + let pkgEntries = packageEntries pkg + pkgEntries `deepseq` return (Just pkgEntries) data SomethingMissing = NoSuchPackage diff --git a/src/SearchIndex.hs b/src/SearchIndex.hs index b1b1e37..bb78b8e 100644 --- a/src/SearchIndex.hs +++ b/src/SearchIndex.hs @@ -5,7 +5,9 @@ module SearchIndex , searchResultTitle , SearchIndex , emptySearchIndex - , createSearchIndex + , PackageEntries + , packageEntries + , buildSearchIndex , evalSearchIndex , searchForName , searchForType @@ -114,38 +116,58 @@ instance NFData IndexEntry emptySearchIndex :: SearchIndex emptySearchIndex = SearchIndex Trie.empty --- | Given a list of packages, create a search index for them. -createSearchIndex :: [D.Package a] -> SearchIndex -createSearchIndex = - countReverseDependencies - >>> sortOn (Down . snd) - >>> concatMap (uncurry entriesForPackage) - >>> (primEntries ++) - >>> fromListWithDuplicates - >>> SearchIndex - --- | A strategy for evaluating a SearchIndex in parallel. -evalSearchIndex :: Strategy SearchIndex -evalSearchIndex = fmap SearchIndex . evalTraversable rdeepseq . unSearchIndex - --- | --- Given a list of packages (which should not include duplicates, or more than --- one version of any given package), return a list of packages together with --- the number of reverse dependencies each one has, in no particular order. +-- | Everything the search index needs from a single package: the package's +-- name, the names of its dependencies (for counting reverse dependencies), +-- and the package's index entries. -- -countReverseDependencies :: [D.Package a] -> [(D.Package a, Int)] -countReverseDependencies packages = - Map.elems $ foldl' go initialMap packages +-- The reverse dependency count in each entry is zero until it is filled in +-- by 'buildSearchIndex', once every package's dependencies have been seen. +data PackageEntries = PackageEntries + { peName :: PackageName + , peDeps :: [PackageName] + , peEntries :: [(ByteString, IndexEntry)] + } + deriving (Generic) + +instance NFData PackageEntries + +-- | Compute the index entries for a single package. The index is built from +-- one 'PackageEntries' per package (see 'buildSearchIndex') rather than from +-- a list of packages, so that the decoded packages themselves need not all be +-- held in memory at the same time: a fully evaluated 'PackageEntries' retains +-- only the small parts of a package that the index needs. +packageEntries :: D.Package a -> PackageEntries +packageEntries pkg = PackageEntries + { peName = D.packageName pkg + , peDeps = map fst (D.pkgResolvedDependencies pkg) + , peEntries = entriesForPackage pkg 0 + } + +-- | Given entries for each package (which should not include duplicates, or +-- more than one version of any given package), assemble them into a search +-- index, filling in the reverse dependency count of each entry. +buildSearchIndex :: [PackageEntries] -> SearchIndex +buildSearchIndex pkgs = + SearchIndex + (fromListWithDuplicates + (primEntries ++ + concatMap entriesWithRevDeps (sortOn (Down . revDepsOf) pkgs))) where - initialMap = - Map.fromList $ map (\pkg -> (D.packageName pkg, (pkg, 0))) packages + revDepCounts :: Map.Map PackageName Int + revDepCounts = + Map.fromListWith (+) [ (dep, 1) | pkg <- pkgs, dep <- peDeps pkg ] + + revDepsOf :: PackageEntries -> Int + revDepsOf pkg = Map.findWithDefault 0 (peName pkg) revDepCounts - go m pkg = - foldl' (flip increment) m - (map fst (D.pkgResolvedDependencies pkg)) + entriesWithRevDeps :: PackageEntries -> [(ByteString, IndexEntry)] + entriesWithRevDeps pkg = + map (second (\entry -> entry { entryRevDeps = Just (Down (revDepsOf pkg)) })) + (peEntries pkg) - increment = - Map.adjust (second (+1)) +-- | A strategy for evaluating a SearchIndex in parallel. +evalSearchIndex :: Strategy SearchIndex +evalSearchIndex = fmap SearchIndex . evalTraversable rdeepseq . unSearchIndex primEntries :: [(ByteString, IndexEntry)] primEntries = From d074e8e4a60ab307ffe129c6ea2a3e05d00a2a9a Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Sat, 4 Jul 2026 09:40:50 -0400 Subject: [PATCH 2/2] Address review: skip unreadable packages, force index on regen thread - A package file which cannot be read or decoded is now skipped with a logged error instead of silently aborting the entire index build (which left the search index empty until the next successful hourly run). Verified by corrupting a package file in a local run against production data: the package is skipped and the rest of the index builds normally. - Fully evaluate the new index on the regeneration thread before publishing it, so the first search request after a regen no longer pays for building the trie. - Pin stack to 3.9.3 in CI (what "latest" resolves to today) so release builds stay reproducible. Memory, measured over 21 consecutive index rebuilds against the full production data set with +RTS -s -N2 -A64m -M3G: 715MB maximum residency, 2.2GB peak RTS memory in use, no growth across cycles. --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 5 +++++ src/Application.hs | 8 +++++--- src/Handler/Database.hs | 34 ++++++++++++++++++++++------------ 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e19234..c86d666 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: - uses: haskell-actions/setup@v2 with: enable-stack: true - stack-version: "latest" + stack-version: "3.9.3" stack-no-global: true - name: Cache dependencies diff --git a/CHANGELOG.md b/CHANGELOG.md index 214ed64..85edde9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ the most up-to-date version of this file. now decoded and indexed one at a time, so only the index entries themselves are retained. + A package file which cannot be read or decoded is now skipped (previously it + silently aborted the entire index build, leaving the search index empty), + and the new index is fully evaluated on the regeneration thread before being + published, rather than by the first search request to use it. + - Use both CPU cores and cap the heap below available RAM (`-N2 -A64m -M3G`) in `pursuit.service` (@thomashoneyman) - Update GitHub Actions to current action versions and pin the runner to diff --git a/src/Application.hs b/src/Application.hs index 46c188b..562a872 100644 --- a/src/Application.hs +++ b/src/Application.hs @@ -77,9 +77,11 @@ makeFoundation appSettings = do foundation createSearchIndexFromDatabase - traverse ( atomically - . writeTVar (appSearchIndex foundation) - . withStrategy evalSearchIndex + -- Evaluate the new index fully on this thread before publishing + -- it, so that request handlers never have to pay for building it. + traverse ( \idx -> do + idx' <- evaluate (withStrategy evalSearchIndex idx) + atomically (writeTVar (appSearchIndex foundation) idx') ) searchIndex -- | Convert our foundation to a WAI Application by calling @toWaiAppPlain@ and diff --git a/src/Handler/Database.hs b/src/Handler/Database.hs index bb084f2..4c2306a 100644 --- a/src/Handler/Database.hs +++ b/src/Handler/Database.hs @@ -58,20 +58,30 @@ createSearchIndexFromDatabase = do entries <- catMaybes <$> traverse entriesFor pkgNames return (buildSearchIndex entries) where + -- A package which cannot be read or decoded is skipped, rather than + -- aborting the entire index build; in particular, decodePackageFile + -- responds with a 500 (thrown as an exception) on invalid JSON. entriesFor :: PackageName -> Handler (Maybe PackageEntries) entriesFor name = do - mversion <- getLatestVersionFor name - case mversion of - Nothing -> return Nothing - Just version -> do - mpkg <- hush <$> lookupPackage name version - case mpkg of - Nothing -> return Nothing - Just pkg -> do - -- Force the entries fully before moving on to the next package, - -- so that the decoded package can be garbage collected. - let pkgEntries = packageEntries pkg - pkgEntries `deepseq` return (Just pkgEntries) + result <- tryAny $ do + mversion <- getLatestVersionFor name + case mversion of + Nothing -> return Nothing + Just version -> do + mpkg <- hush <$> lookupPackage name version + case mpkg of + Nothing -> return Nothing + Just pkg -> do + -- Force the entries fully before moving on to the next package, + -- so that the decoded package can be garbage collected. + let pkgEntries = packageEntries pkg + pkgEntries `deepseq` return (Just pkgEntries) + case result of + Right r -> return r + Left err -> do + $logError ("Skipping " <> runPackageName name <> + " while building the search index: " <> tshow err) + return Nothing data SomethingMissing = NoSuchPackage