diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1c1242..c86d666 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,19 +14,21 @@ jobs: build_server: name: Build server # Note that this must be kept in sync with the version of Ubuntu which the - # Pursit server is running, otherwise the server binary may fail to run. - runs-on: ubuntu-latest + # Pursuit server is running, otherwise the server binary may fail to run + # (the binary links against the runner's glibc, and a binary built against + # a newer glibc than the server's will not start). + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: haskell/actions/setup@v1 + - uses: haskell-actions/setup@v2 with: enable-stack: true - stack-version: "2.5.1" + stack-version: "3.9.3" stack-no-global: true - name: Cache dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.stack key: ${{ runner.os }}-stack-${{ hashFiles('stack.yaml.lock') }}-${{ hashFiles('pursuit.cabal') }} @@ -49,7 +51,7 @@ jobs: tar czf pursuit.tar.gz -C ${{ env.BUNDLE_DIR }}/ . - name: Persist server assets - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: github.event_name == 'release' with: name: pursuit.tar.gz @@ -58,9 +60,7 @@ jobs: release: name: Release - # Note that this must be kept in sync with the version of Ubuntu which the - # Pursit server is running, otherwise the server binary may fail to run. - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: github.event_name == 'release' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -68,12 +68,12 @@ jobs: - build_server steps: - name: Retrieve server assets - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: pursuit.tar.gz - name: Upload assets - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: files: | pursuit.tar.gz diff --git a/CHANGELOG.md b/CHANGELOG.md index 241f78a..85edde9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ the most up-to-date version of this file. ## Unreleased +## v0.9.10 + +- Build the search index one package at a time (@thomashoneyman) + + Previously the hourly search index regeneration (which also runs at startup) + decoded every package in the database into memory at once before building + the index. The decoded packages now require more memory than the server has, + which caused the server to be OOM-killed several times an hour. Packages are + now decoded and indexed one at a time, so only the index entries themselves + are retained. + + A package file which cannot be read or decoded is now skipped (previously it + silently aborted the entire index build, leaving the search index empty), + and the new index is fully evaluated on the regeneration thread before being + published, rather than by the first search request to use it. + +- Use both CPU cores and cap the heap below available RAM (`-N2 -A64m -M3G`) + in `pursuit.service` (@thomashoneyman) +- Update GitHub Actions to current action versions and pin the runner to + `ubuntu-24.04` to match the server (@thomashoneyman) + ## v0.9.9 - Update pursuit.service to 3.5GB max memory (@thomashoneyman) diff --git a/deploy/pursuit.service b/deploy/pursuit.service index 8d3fb42..8963ca1 100644 --- a/deploy/pursuit.service +++ b/deploy/pursuit.service @@ -4,7 +4,10 @@ Description=Web service for hosting of PureScript API documentation [Service] Type=simple User=www-data -ExecStart=/usr/local/bin/pursuit +RTS -N1 -A128m -M3.5G -RTS +# The server has 2 vCPUs and 4GB of RAM. The heap limit needs to leave room +# for nginx and the rest of the system, so that the RTS runs out of heap +# before the kernel's OOM killer steps in. +ExecStart=/usr/local/bin/pursuit +RTS -N2 -A64m -M3G -RTS Restart=always RestartSec=5s Environment="PURSUIT_APPROOT=https://pursuit.purescript.org" diff --git a/pursuit.cabal b/pursuit.cabal index 24ac6af..a33cebb 100644 --- a/pursuit.cabal +++ b/pursuit.cabal @@ -1,5 +1,5 @@ name: pursuit -version: 0.9.9 +version: 0.9.10 cabal-version: >= 1.8 build-type: Simple license: MIT diff --git a/src/Application.hs b/src/Application.hs index b75f451..562a872 100644 --- a/src/Application.hs +++ b/src/Application.hs @@ -40,7 +40,7 @@ import Handler.Packages import Handler.Search import Handler.PackageBadges import Handler.Help -import SearchIndex (emptySearchIndex, createSearchIndex, evalSearchIndex) +import SearchIndex (emptySearchIndex, evalSearchIndex) -- This line actually creates our YesodDispatch instance. It is the second half -- of the call to mkYesodData which occurs in Foundation.hs. Please see the @@ -71,17 +71,18 @@ makeFoundation appSettings = do let hour = 60 * 60 * 1000 * 1000 -- microseconds in every hour $ do let emptySessionMap = mempty :: SessionMap - pkgs <- Unsafe.runFakeHandler + searchIndex <- Unsafe.runFakeHandler emptySessionMap appLogger foundation - getAllPackages - - traverse ( atomically - . writeTVar (appSearchIndex foundation) - . withStrategy evalSearchIndex - . createSearchIndex - ) pkgs + createSearchIndexFromDatabase + + -- Evaluate the new index fully on this thread before publishing + -- it, so that request handlers never have to pay for building it. + traverse ( \idx -> do + idx' <- evaluate (withStrategy evalSearchIndex idx) + atomically (writeTVar (appSearchIndex foundation) idx') + ) searchIndex -- | Convert our foundation to a WAI Application by calling @toWaiAppPlain@ and -- applyng some additional middlewares. diff --git a/src/Handler/Database.hs b/src/Handler/Database.hs index 0c9f099..4c2306a 100644 --- a/src/Handler/Database.hs +++ b/src/Handler/Database.hs @@ -3,7 +3,7 @@ -- Pursuit). module Handler.Database ( getAllPackageNames - , getAllPackages + , createSearchIndexFromDatabase , getLatestPackages , lookupPackage , availableVersionsFor @@ -26,6 +26,7 @@ import qualified Language.PureScript.Docs as D import Handler.Utils import Handler.Caching (clearCache) +import SearchIndex (SearchIndex, PackageEntries, packageEntries, buildSearchIndex) getAllPackageNames :: Handler [PackageName] getAllPackageNames = do @@ -46,16 +47,41 @@ getLatestPackages = do withVersion :: PackageName -> Handler (Maybe (PackageName, Version)) withVersion name = (map . map) (name,) (getLatestVersionFor name) --- | This is horribly inefficient, but it will do for now. Note that this --- only gets the latest version of each package in the database. -getAllPackages :: Handler [D.VerifiedPackage] -getAllPackages = do +-- | Build the search index from the latest version of each package in the +-- database. Packages are read and decoded one at a time, and only the +-- (comparatively small) index entries for each package are retained: decoding +-- every package up front requires several gigabytes of memory, which is more +-- than the server has. +createSearchIndexFromDatabase :: Handler SearchIndex +createSearchIndexFromDatabase = do pkgNames <- getAllPackageNames - pkgNamesAndVersions <- catMaybes <$> traverse withVersion pkgNames - catMaybes <$> traverse lookupPackageMay pkgNamesAndVersions + entries <- catMaybes <$> traverse entriesFor pkgNames + return (buildSearchIndex entries) where - withVersion name = (map . map) (name,) (getLatestVersionFor name) - lookupPackageMay = map hush . uncurry lookupPackage + -- A package which cannot be read or decoded is skipped, rather than + -- aborting the entire index build; in particular, decodePackageFile + -- responds with a 500 (thrown as an exception) on invalid JSON. + entriesFor :: PackageName -> Handler (Maybe PackageEntries) + entriesFor name = do + result <- tryAny $ do + mversion <- getLatestVersionFor name + case mversion of + Nothing -> return Nothing + Just version -> do + mpkg <- hush <$> lookupPackage name version + case mpkg of + Nothing -> return Nothing + Just pkg -> do + -- Force the entries fully before moving on to the next package, + -- so that the decoded package can be garbage collected. + let pkgEntries = packageEntries pkg + pkgEntries `deepseq` return (Just pkgEntries) + case result of + Right r -> return r + Left err -> do + $logError ("Skipping " <> runPackageName name <> + " while building the search index: " <> tshow err) + return Nothing data SomethingMissing = NoSuchPackage diff --git a/src/SearchIndex.hs b/src/SearchIndex.hs index b1b1e37..bb78b8e 100644 --- a/src/SearchIndex.hs +++ b/src/SearchIndex.hs @@ -5,7 +5,9 @@ module SearchIndex , searchResultTitle , SearchIndex , emptySearchIndex - , createSearchIndex + , PackageEntries + , packageEntries + , buildSearchIndex , evalSearchIndex , searchForName , searchForType @@ -114,38 +116,58 @@ instance NFData IndexEntry emptySearchIndex :: SearchIndex emptySearchIndex = SearchIndex Trie.empty --- | Given a list of packages, create a search index for them. -createSearchIndex :: [D.Package a] -> SearchIndex -createSearchIndex = - countReverseDependencies - >>> sortOn (Down . snd) - >>> concatMap (uncurry entriesForPackage) - >>> (primEntries ++) - >>> fromListWithDuplicates - >>> SearchIndex - --- | A strategy for evaluating a SearchIndex in parallel. -evalSearchIndex :: Strategy SearchIndex -evalSearchIndex = fmap SearchIndex . evalTraversable rdeepseq . unSearchIndex - --- | --- Given a list of packages (which should not include duplicates, or more than --- one version of any given package), return a list of packages together with --- the number of reverse dependencies each one has, in no particular order. +-- | Everything the search index needs from a single package: the package's +-- name, the names of its dependencies (for counting reverse dependencies), +-- and the package's index entries. -- -countReverseDependencies :: [D.Package a] -> [(D.Package a, Int)] -countReverseDependencies packages = - Map.elems $ foldl' go initialMap packages +-- The reverse dependency count in each entry is zero until it is filled in +-- by 'buildSearchIndex', once every package's dependencies have been seen. +data PackageEntries = PackageEntries + { peName :: PackageName + , peDeps :: [PackageName] + , peEntries :: [(ByteString, IndexEntry)] + } + deriving (Generic) + +instance NFData PackageEntries + +-- | Compute the index entries for a single package. The index is built from +-- one 'PackageEntries' per package (see 'buildSearchIndex') rather than from +-- a list of packages, so that the decoded packages themselves need not all be +-- held in memory at the same time: a fully evaluated 'PackageEntries' retains +-- only the small parts of a package that the index needs. +packageEntries :: D.Package a -> PackageEntries +packageEntries pkg = PackageEntries + { peName = D.packageName pkg + , peDeps = map fst (D.pkgResolvedDependencies pkg) + , peEntries = entriesForPackage pkg 0 + } + +-- | Given entries for each package (which should not include duplicates, or +-- more than one version of any given package), assemble them into a search +-- index, filling in the reverse dependency count of each entry. +buildSearchIndex :: [PackageEntries] -> SearchIndex +buildSearchIndex pkgs = + SearchIndex + (fromListWithDuplicates + (primEntries ++ + concatMap entriesWithRevDeps (sortOn (Down . revDepsOf) pkgs))) where - initialMap = - Map.fromList $ map (\pkg -> (D.packageName pkg, (pkg, 0))) packages + revDepCounts :: Map.Map PackageName Int + revDepCounts = + Map.fromListWith (+) [ (dep, 1) | pkg <- pkgs, dep <- peDeps pkg ] + + revDepsOf :: PackageEntries -> Int + revDepsOf pkg = Map.findWithDefault 0 (peName pkg) revDepCounts - go m pkg = - foldl' (flip increment) m - (map fst (D.pkgResolvedDependencies pkg)) + entriesWithRevDeps :: PackageEntries -> [(ByteString, IndexEntry)] + entriesWithRevDeps pkg = + map (second (\entry -> entry { entryRevDeps = Just (Down (revDepsOf pkg)) })) + (peEntries pkg) - increment = - Map.adjust (second (+1)) +-- | A strategy for evaluating a SearchIndex in parallel. +evalSearchIndex :: Strategy SearchIndex +evalSearchIndex = fmap SearchIndex . evalTraversable rdeepseq . unSearchIndex primEntries :: [(ByteString, IndexEntry)] primEntries =