Build the search index one package at a time

.github/workflows/ci.yml

@@ -14,19 +14,21 @@ jobs:
   build_server:
     name: Build server
     # Note that this must be kept in sync with the version of Ubuntu which the
-    # Pursit server is running, otherwise the server binary may fail to run.
-    runs-on: ubuntu-latest
+    # Pursuit server is running, otherwise the server binary may fail to run
+    # (the binary links against the runner's glibc, and a binary built against
+    # a newer glibc than the server's will not start).
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - uses: haskell/actions/setup@v1
+      - uses: haskell-actions/setup@v2
         with:
           enable-stack: true
-          stack-version: "2.5.1"
+          stack-version: "3.9.3"
           stack-no-global: true
 
       - name: Cache dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.stack
           key: ${{ runner.os }}-stack-${{ hashFiles('stack.yaml.lock') }}-${{ hashFiles('pursuit.cabal') }}
@@ -49,7 +51,7 @@ jobs:
           tar czf pursuit.tar.gz -C ${{ env.BUNDLE_DIR }}/ .
 
       - name: Persist server assets
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         if: github.event_name == 'release'
         with:
           name: pursuit.tar.gz
@@ -58,22 +60,20 @@ jobs:
 
   release:
     name: Release
-    # Note that this must be kept in sync with the version of Ubuntu which the
-    # Pursit server is running, otherwise the server binary may fail to run.
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     if: github.event_name == 'release'
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     needs:
       - build_server
     steps:
       - name: Retrieve server assets
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: pursuit.tar.gz
 
       - name: Upload assets
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           files: |
             pursuit.tar.gz

CHANGELOG.md

@@ -5,6 +5,27 @@ the most up-to-date version of this file.
 
 ## Unreleased
 
+## v0.9.10
+
+- Build the search index one package at a time (@thomashoneyman)
+
+  Previously the hourly search index regeneration (which also runs at startup)
+  decoded every package in the database into memory at once before building
+  the index. The decoded packages now require more memory than the server has,
+  which caused the server to be OOM-killed several times an hour. Packages are
+  now decoded and indexed one at a time, so only the index entries themselves
+  are retained.
+
+  A package file which cannot be read or decoded is now skipped (previously it
+  silently aborted the entire index build, leaving the search index empty),
+  and the new index is fully evaluated on the regeneration thread before being
+  published, rather than by the first search request to use it.
+
+- Use both CPU cores and cap the heap below available RAM (`-N2 -A64m -M3G`)
+  in `pursuit.service` (@thomashoneyman)
+- Update GitHub Actions to current action versions and pin the runner to
+  `ubuntu-24.04` to match the server (@thomashoneyman)
+
 ## v0.9.9
 
 - Update pursuit.service to 3.5GB max memory (@thomashoneyman)

deploy/pursuit.service

@@ -4,7 +4,10 @@ Description=Web service for hosting of PureScript API documentation
 [Service]
 Type=simple
 User=www-data
-ExecStart=/usr/local/bin/pursuit +RTS -N1 -A128m -M3.5G -RTS
+# The server has 2 vCPUs and 4GB of RAM. The heap limit needs to leave room
+# for nginx and the rest of the system, so that the RTS runs out of heap
+# before the kernel's OOM killer steps in.
+ExecStart=/usr/local/bin/pursuit +RTS -N2 -A64m -M3G -RTS
 Restart=always
 RestartSec=5s
 Environment="PURSUIT_APPROOT=https://pursuit.purescript.org"

pursuit.cabal

@@ -1,5 +1,5 @@
 name:              pursuit
-version:           0.9.9
+version:           0.9.10
 cabal-version:     >= 1.8
 build-type:        Simple
 license:           MIT

src/Application.hs

@@ -40,7 +40,7 @@ import Handler.Packages
 import Handler.Search
 import Handler.PackageBadges
 import Handler.Help
-import SearchIndex (emptySearchIndex, createSearchIndex, evalSearchIndex)
+import SearchIndex (emptySearchIndex, evalSearchIndex)
 
 -- This line actually creates our YesodDispatch instance. It is the second half
 -- of the call to mkYesodData which occurs in Foundation.hs. Please see the
@@ -71,17 +71,18 @@ makeFoundation appSettings = do
        let hour = 60 * 60 * 1000 * 1000 -- microseconds
        in every hour $ do
            let emptySessionMap = mempty :: SessionMap
-           pkgs <- Unsafe.runFakeHandler
+           searchIndex <- Unsafe.runFakeHandler
                       emptySessionMap
                       appLogger
                       foundation
-                      getAllPackages
-
-           traverse ( atomically
-                    . writeTVar (appSearchIndex foundation)
-                    . withStrategy evalSearchIndex
-                    . createSearchIndex
-                    ) pkgs
+                      createSearchIndexFromDatabase
+
+           -- Evaluate the new index fully on this thread before publishing
+           -- it, so that request handlers never have to pay for building it.
+           traverse ( \idx -> do
+                        idx' <- evaluate (withStrategy evalSearchIndex idx)
+                        atomically (writeTVar (appSearchIndex foundation) idx')
+                    ) searchIndex
 
 -- | Convert our foundation to a WAI Application by calling @toWaiAppPlain@ and
 -- applyng some additional middlewares.

src/Handler/Database.hs

@@ -3,7 +3,7 @@
 -- Pursuit).
 module Handler.Database
   ( getAllPackageNames
-  , getAllPackages
+  , createSearchIndexFromDatabase
   , getLatestPackages
   , lookupPackage
   , availableVersionsFor
@@ -26,6 +26,7 @@ import qualified Language.PureScript.Docs as D
 
 import Handler.Utils
 import Handler.Caching (clearCache)
+import SearchIndex (SearchIndex, PackageEntries, packageEntries, buildSearchIndex)
 
 getAllPackageNames :: Handler [PackageName]
 getAllPackageNames = do
@@ -46,16 +47,41 @@ getLatestPackages = do
     withVersion :: PackageName -> Handler (Maybe (PackageName, Version))
     withVersion name = (map . map) (name,) (getLatestVersionFor name)
 
--- | This is horribly inefficient, but it will do for now. Note that this
--- only gets the latest version of each package in the database.
-getAllPackages :: Handler [D.VerifiedPackage]
-getAllPackages = do
+-- | Build the search index from the latest version of each package in the
+-- database. Packages are read and decoded one at a time, and only the
+-- (comparatively small) index entries for each package are retained: decoding
+-- every package up front requires several gigabytes of memory, which is more
+-- than the server has.
+createSearchIndexFromDatabase :: Handler SearchIndex
+createSearchIndexFromDatabase = do
   pkgNames <- getAllPackageNames
-  pkgNamesAndVersions <- catMaybes <$> traverse withVersion pkgNames
-  catMaybes <$> traverse lookupPackageMay pkgNamesAndVersions
+  entries <- catMaybes <$> traverse entriesFor pkgNames
+  return (buildSearchIndex entries)
   where
-  withVersion name = (map . map) (name,) (getLatestVersionFor name)
-  lookupPackageMay = map hush . uncurry lookupPackage
+  -- A package which cannot be read or decoded is skipped, rather than
+  -- aborting the entire index build; in particular, decodePackageFile
+  -- responds with a 500 (thrown as an exception) on invalid JSON.
+  entriesFor :: PackageName -> Handler (Maybe PackageEntries)
+  entriesFor name = do
+    result <- tryAny $ do
+      mversion <- getLatestVersionFor name
+      case mversion of
+        Nothing -> return Nothing
+        Just version -> do
+          mpkg <- hush <$> lookupPackage name version
+          case mpkg of
+            Nothing -> return Nothing
+            Just pkg -> do
+              -- Force the entries fully before moving on to the next package,
+              -- so that the decoded package can be garbage collected.
+              let pkgEntries = packageEntries pkg
+              pkgEntries `deepseq` return (Just pkgEntries)
+    case result of
+      Right r -> return r
+      Left err -> do
+        $logError ("Skipping " <> runPackageName name <>
+                   " while building the search index: " <> tshow err)
+        return Nothing
 
 data SomethingMissing
   = NoSuchPackage

src/SearchIndex.hs

@@ -5,7 +5,9 @@ module SearchIndex
   , searchResultTitle
   , SearchIndex
   , emptySearchIndex
-  , createSearchIndex
+  , PackageEntries
+  , packageEntries
+  , buildSearchIndex
   , evalSearchIndex
   , searchForName
   , searchForType
@@ -114,38 +116,58 @@ instance NFData IndexEntry
 emptySearchIndex :: SearchIndex
 emptySearchIndex = SearchIndex Trie.empty
 
--- | Given a list of packages, create a search index for them.
-createSearchIndex :: [D.Package a] -> SearchIndex
-createSearchIndex =
-  countReverseDependencies
-  >>> sortOn (Down . snd)
-  >>> concatMap (uncurry entriesForPackage)
-  >>> (primEntries ++)
-  >>> fromListWithDuplicates
-  >>> SearchIndex
-
--- | A strategy for evaluating a SearchIndex in parallel.
-evalSearchIndex :: Strategy SearchIndex
-evalSearchIndex = fmap SearchIndex . evalTraversable rdeepseq . unSearchIndex
-
--- |
--- Given a list of packages (which should not include duplicates, or more than
--- one version of any given package), return a list of packages together with
--- the number of reverse dependencies each one has, in no particular order.
+-- | Everything the search index needs from a single package: the package's
+-- name, the names of its dependencies (for counting reverse dependencies),
+-- and the package's index entries.
 --
-countReverseDependencies :: [D.Package a] -> [(D.Package a, Int)]
-countReverseDependencies packages =
-  Map.elems $ foldl' go initialMap packages
+-- The reverse dependency count in each entry is zero until it is filled in
+-- by 'buildSearchIndex', once every package's dependencies have been seen.
+data PackageEntries = PackageEntries
+  { peName    :: PackageName
+  , peDeps    :: [PackageName]
+  , peEntries :: [(ByteString, IndexEntry)]
+  }
+  deriving (Generic)
+
+instance NFData PackageEntries
+
+-- | Compute the index entries for a single package. The index is built from
+-- one 'PackageEntries' per package (see 'buildSearchIndex') rather than from
+-- a list of packages, so that the decoded packages themselves need not all be
+-- held in memory at the same time: a fully evaluated 'PackageEntries' retains
+-- only the small parts of a package that the index needs.
+packageEntries :: D.Package a -> PackageEntries
+packageEntries pkg = PackageEntries
+  { peName    = D.packageName pkg
+  , peDeps    = map fst (D.pkgResolvedDependencies pkg)
+  , peEntries = entriesForPackage pkg 0
+  }
+
+-- | Given entries for each package (which should not include duplicates, or
+-- more than one version of any given package), assemble them into a search
+-- index, filling in the reverse dependency count of each entry.
+buildSearchIndex :: [PackageEntries] -> SearchIndex
+buildSearchIndex pkgs =
+  SearchIndex
+    (fromListWithDuplicates
+      (primEntries ++
+        concatMap entriesWithRevDeps (sortOn (Down . revDepsOf) pkgs)))
   where
-  initialMap =
-    Map.fromList $ map (\pkg -> (D.packageName pkg, (pkg, 0))) packages
+  revDepCounts :: Map.Map PackageName Int
+  revDepCounts =
+    Map.fromListWith (+) [ (dep, 1) | pkg <- pkgs, dep <- peDeps pkg ]
+
+  revDepsOf :: PackageEntries -> Int
+  revDepsOf pkg = Map.findWithDefault 0 (peName pkg) revDepCounts
 
-  go m pkg =
-    foldl' (flip increment) m
-      (map fst (D.pkgResolvedDependencies pkg))
+  entriesWithRevDeps :: PackageEntries -> [(ByteString, IndexEntry)]
+  entriesWithRevDeps pkg =
+    map (second (\entry -> entry { entryRevDeps = Just (Down (revDepsOf pkg)) }))
+        (peEntries pkg)
 
-  increment =
-    Map.adjust (second (+1))
+-- | A strategy for evaluating a SearchIndex in parallel.
+evalSearchIndex :: Strategy SearchIndex
+evalSearchIndex = fmap SearchIndex . evalTraversable rdeepseq . unSearchIndex
 
 primEntries :: [(ByteString, IndexEntry)]
 primEntries =