From e4b2104a8bfdc5ec0905b5f4195073ba8df75396 Mon Sep 17 00:00:00 2001 From: Tim Fischer Date: Tue, 16 Jun 2026 14:22:23 +0200 Subject: [PATCH 1/2] config: allow listing submodules to clone --- CHANGELOG.md | 3 ++ book/src/dependencies.md | 20 ++++++++++ src/config.rs | 72 ++++++++++++++++++++++++++++++++++ src/sess.rs | 83 +++++++++++++++++++++++++++++++--------- 4 files changed, 160 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da509645..6caadade 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## Unreleased +### Added +- Add a per-dependency `git_submodules` list to the manifest (`Bender.yml`) that lets a package restrict which of its submodules are cloned (with optional per-entry `recursive` and `shallow` flags, both defaulting to `true`); when absent, all submodules are cloned recursively as before. + ## 0.32.1 - 2026-07-07 ### Added - Add `git_submodules` config field and `--git-submodules ` flag (env `BENDER_GIT_SUBMODULES`) to control cloning of dependency submodules; defaults to `true`, the flag overrides the configured value in either direction (https://github.com/pulp-platform/bender/pull/314). diff --git a/book/src/dependencies.md b/book/src/dependencies.md index 6f8b97c5..4423f6a7 100644 --- a/book/src/dependencies.md +++ b/book/src/dependencies.md @@ -120,6 +120,26 @@ Only disable submodules when none of your dependencies reference sources that li When submodules are disabled, Bender emits a warning for each dependency that carries submodules, listing the unchecked-out submodule paths and the `git submodule update --init --recursive` command to fetch them back into its checkout. +### Selecting submodules per dependency + +A package maintainer who knows which submodules are actually needed can restrict cloning to those submodules by adding a `git_submodules` list to the dependency's own `Bender.yml`: + +```yaml +git_submodules: + - sw/deps/printf # string form: clone this submodule + - submodule: sw/deps/cva6-sdk # map form + recursive: false # skip the submodule's own submodules (default: true) + shallow: false # fetch full history (default: true) + # pd/deps/ihp-130-pdk -> omitted, so it is not cloned +``` + +- **No `git_submodules` field** (the default): all submodules are cloned recursively. +- **`git_submodules` present**: only the listed submodules are cloned; an empty list (`git_submodules: []`) clones none. +- **`recursive`** (default `true`): also update the submodule's own nested submodules. Set it to `false` to fetch only the top-level submodule. +- **`shallow`** (default `true`): fetch the submodule with `--depth 1`. Set it to `false` to clone the full submodule history. + +The global `git_submodules: false` / `--git-submodules false` switch always wins: when submodule cloning is disabled globally, the per-dependency list is ignored and no submodules are cloned. + ## Version Resolution and the Lockfile When you run `bender update`, Bender performs the following: diff --git a/src/config.rs b/src/config.rs index b150f659..5840566e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -53,6 +53,12 @@ pub struct Manifest { pub workspace: Workspace, /// Vendorized dependencies pub vendor_package: Vec, + /// Per-dependency submodule selection. + /// + /// `None` means clone all submodules recursively (the default). `Some` + /// restricts cloning to the listed submodules only (an empty list clones + /// none). + pub git_submodules: Option>, } impl PrefixPaths for Manifest { @@ -455,6 +461,8 @@ pub struct PartialManifest { pub workspace: Option, /// External Import dependencies pub vendor_package: Option>, + /// Per-dependency submodule selection. + pub git_submodules: Option>>, /// Unknown extra fields #[serde(flatten)] extra: HashMap, @@ -604,6 +612,11 @@ impl Validate for PartialManifest { .wrap_err("Unable to parse vendor_package.")?, None => Vec::new(), }; + let git_submodules = self + .git_submodules + .map(|subs| subs.validate(vctx)) + .transpose() + .wrap_err(format!("In git_submodules of package `{}`:", pkg.name))?; if !vctx.pre_output { self.extra.iter().for_each(|(k, _)| { Warnings::IgnoreUnknownField { @@ -643,6 +656,7 @@ impl Validate for PartialManifest { frozen, workspace, vendor_package, + git_submodules, }) } } @@ -895,6 +909,64 @@ impl Validate for PartialIncludeDir { } } +/// A submodule of a dependency that should be cloned. +#[derive(Clone, Debug, Serialize)] +pub struct Submodule { + /// The path of the submodule within the dependency repository. + pub path: String, + /// Whether to also update the submodule's own submodules (`--recursive`). + pub recursive: bool, + /// Whether to fetch the submodule shallowly (`--depth 1`). + pub shallow: bool, +} + +/// A partial submodule selection entry. +#[derive(Serialize, Deserialize, Debug)] +pub struct PartialSubmodule { + /// The path of the submodule within the dependency repository. + pub submodule: String, + /// Whether to also update the submodule's own submodules (default: true). + pub recursive: Option, + /// Whether to fetch the submodule shallowly (default: true). + pub shallow: Option, + /// Unknown extra fields + #[serde(flatten)] + extra: HashMap, +} + +impl FromStr for PartialSubmodule { + type Err = Void; + fn from_str(s: &str) -> std::result::Result { + Ok(PartialSubmodule { + submodule: s.into(), + recursive: None, + shallow: None, + extra: HashMap::new(), + }) + } +} + +impl Validate for PartialSubmodule { + type Output = Submodule; + type Error = Error; + fn validate(self, vctx: &ValidationContext) -> Result { + if !vctx.pre_output { + self.extra.iter().for_each(|(k, _)| { + Warnings::IgnoreUnknownField { + field: k.clone(), + pkg: vctx.package_name.to_string(), + } + .emit(); + }); + } + Ok(Submodule { + path: self.submodule, + recursive: self.recursive.unwrap_or(true), + shallow: self.shallow.unwrap_or(true), + }) + } +} + /// A partial filtered preprocessor define #[derive(Serialize, Deserialize, Debug, Default)] pub struct PartialDefine { diff --git a/src/sess.rs b/src/sess.rs index 272933d0..092aa91f 100644 --- a/src/sess.rs +++ b/src/sess.rs @@ -1326,24 +1326,71 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> { } if path.join(".gitmodules").exists() { if self.sess.config.git_submodules { - let pb = Some(ProgressHandler::new( - self.sess.multiprogress.clone(), - GitProgressOps::Submodule, - name, - )); - local_git - .clone() - .spawn_with( - move |c| { - c.arg("submodule") - .arg("update") - .arg("--init") - .arg("--recursive") - .arg("--progress") - }, - pb, - ) - .await?; + // Re-read the dependency's own manifest (now checked out on + // disk) to see whether it restricts which of its submodules + // should be cloned. Source/include-dir existence is ignored + // here (via `validate_ignore_sources`) so that excluding a + // submodule that holds such paths does not defeat the + // selection; a missing or unparseable manifest falls back to + // cloning all submodules recursively. + let selection = std::fs::File::open(path.join("Bender.yml")) + .ok() + .and_then(|file| { + serde_yaml_ng::from_reader::<_, PartialManifest>(file).ok() + }) + .and_then(|partial| partial.validate_ignore_sources().ok()) + .and_then(|m| m.git_submodules); + + match selection { + // No `git_submodules` field: clone all submodules recursively. + None => { + let pb = Some(ProgressHandler::new( + self.sess.multiprogress.clone(), + GitProgressOps::Submodule, + name, + )); + local_git + .clone() + .spawn_with( + move |c| { + c.arg("submodule") + .arg("update") + .arg("--init") + .arg("--recursive") + .arg("--progress") + }, + pb, + ) + .await?; + } + // `git_submodules` present: clone only the listed + // submodules (an empty list clones none). + Some(subs) => { + for sub in subs { + let pb = Some(ProgressHandler::new( + self.sess.multiprogress.clone(), + GitProgressOps::Submodule, + name, + )); + local_git + .clone() + .spawn_with( + move |c| { + c.arg("submodule").arg("update").arg("--init"); + if sub.recursive { + c.arg("--recursive"); + } + if sub.shallow { + c.arg("--depth").arg("1"); + } + c.arg("--progress").arg("--").arg(&sub.path) + }, + pb, + ) + .await?; + } + } + } } else { // Submodules were disabled via the `--git-submodules` flag, // so they are left unchecked out. Warn the user, listing the From b59df6e70d0691911c9bd4ed5e2c3fd90ea0f826 Mon Sep 17 00:00:00 2001 From: Tim Fischer Date: Mon, 3 Aug 2026 15:05:30 +0200 Subject: [PATCH 2/2] feat!: disable submodule clone by default + change CLI/config flags to enum variants --- CHANGELOG.md | 14 +++- book/src/configuration.md | 8 +- book/src/dependencies.md | 21 +++-- src/cli.rs | 13 ++- src/config.rs | 36 ++++++--- src/diagnostic.rs | 20 ++++- src/sess.rs | 162 ++++++++++++++++++++++---------------- 7 files changed, 169 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6caadade..ba2792c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,20 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## Unreleased +### Breaking Changes +- **Submodules of a dependency are no longer cloned by default.** A package now declares which of its submodules it needs with a `git_submodules` list in its own `Bender.yml`, and only those are cloned. Bender emits warning `W37` for each dependency that carries submodules without declaring any. + - As a package maintainer, add the submodules your sources actually need to your manifest, or `git_submodules: []` to state that none are needed and silence the warning. + - As a consumer of a package that has not declared its submodules yet, run with `--git-submodules all` (or set `git_submodules: all` in your configuration) to restore the previous behaviour of cloning every submodule recursively. +- **`--git-submodules` and the `git_submodules` config field take a mode instead of a boolean.** The env variable `BENDER_GIT_SUBMODULES` takes the same values. Booleans are no longer accepted, so replace them: + - `git_submodules: true` -> `git_submodules: all`, cloning all submodules of every dependency recursively, ignoring the per-package lists. + - `git_submodules: false` -> `git_submodules: none`, cloning no submodules at all, ignoring the per-package lists. + - Not setting it at all is now `manifest`, cloning what each package selects in its manifest. + ### Added -- Add a per-dependency `git_submodules` list to the manifest (`Bender.yml`) that lets a package restrict which of its submodules are cloned (with optional per-entry `recursive` and `shallow` flags, both defaulting to `true`); when absent, all submodules are cloned recursively as before. +- Add a per-dependency `git_submodules` list to the manifest (`Bender.yml`) that selects which of a package's submodules are cloned (with optional per-entry `recursive` and `shallow` flags, both defaulting to `true`). + +### Changed +- Bender clones submodules shallowly by default (`--depth 1`), unless a package disables it explicitly by specifying `shallow: false` in its `git_submodules` list. ## 0.32.1 - 2026-07-07 ### Added diff --git a/book/src/configuration.md b/book/src/configuration.md index 2102ccde..9d0ddfa3 100644 --- a/book/src/configuration.md +++ b/book/src/configuration.md @@ -57,12 +57,12 @@ Enable or disable Git Large File Storage (LFS) support. Requires `git-lfs` to be - **Example:** `git_lfs: false` ### `git_submodules` -Clone the git submodules of dependencies. Disabling this skips `git submodule update` during checkout, which can substantially speed up the initial dependency fetch when dependencies carry submodules (e.g. software or tooling) that are not needed for the hardware build. Disable it only when none of your dependencies reference sources that live inside a submodule. +Selects which git submodules are cloned for dependencies. `manifest` (the default) clones the submodules that each dependency selects via the [`git_submodules`](./dependencies.md#selecting-submodules-per-dependency) list in its own manifest. The other two modes ignore those lists: `none` skips `git submodule update` during checkout entirely, which can substantially speed up the initial dependency fetch when dependencies carry submodules (e.g. software or tooling) that are not needed for the hardware build; `all` clones all submodules of every dependency recursively, which helps when a dependency has not declared a submodule that you need. Only set it to `none` when none of your dependencies reference sources that live inside a submodule. - **Config Key:** `git_submodules` -- **CLI Flag:** `--git-submodules ` (overrides the configured value in either direction) +- **CLI Flag:** `--git-submodules ` (takes precedence over the configured value) - **Env Var:** `BENDER_GIT_SUBMODULES` -- **Default:** `true` -- **Example:** `git_submodules: false` +- **Default:** `manifest` +- **Example:** `git_submodules: none` ### `overrides` Forces specific dependencies to use a particular version or local path. This is primarily used in [`Bender.local`](./local.md) for development. diff --git a/book/src/dependencies.md b/book/src/dependencies.md index 4423f6a7..2f91173f 100644 --- a/book/src/dependencies.md +++ b/book/src/dependencies.md @@ -109,20 +109,17 @@ Bender detects whether a dependency uses **Git Large File Storage (LFS)** via it ## Submodules -If a dependency contains a `.gitmodules` file, Bender initializes and updates its Git submodules recursively after checkout by default. +If a dependency contains a `.gitmodules` file, Bender only checks out the submodules that the dependency explicitly asks for in its own manifest. A dependency that does not list any submodules gets none of them checked out, and Bender emits warning `W37` listing the submodules it skipped. -Cloning submodules is often the slowest part of fetching dependencies, and submodules frequently hold software or tooling that is irrelevant to the hardware build. You can therefore disable submodule cloning: +Cloning submodules is often the slowest part of fetching dependencies, and submodules frequently hold software or tooling that is irrelevant to the hardware build. As a consumer of a dependency, you can override its selection in either direction, with `git_submodules` in your [configuration](./configuration.md#git_submodules) or with `--git-submodules ` (env `BENDER_GIT_SUBMODULES`) for a single invocation. The flag takes precedence over the configuration files. -- Set `git_submodules: false` in your [configuration](./configuration.md#git_submodules) to skip submodules persistently for a project. -- Pass `--git-submodules false` (or set `BENDER_GIT_SUBMODULES=false`) to skip them for a single invocation. The flag overrides the configured value in either direction. - -Only disable submodules when none of your dependencies reference sources that live inside a submodule. - -When submodules are disabled, Bender emits a warning for each dependency that carries submodules, listing the unchecked-out submodule paths and the `git submodule update --init --recursive` command to fetch them back into its checkout. +- **`manifest`** (the default): honor each dependency's own selection, as described below. +- **`none`**: clone no submodules at all, ignoring what the dependencies select. Use this when none of your dependencies reference sources that live inside a submodule, or when the submodule remotes are not reachable from your machine. Bender emits warning `W36` for each dependency that *selects* submodules, listing the paths it skipped; a dependency that selects none is not reported, since nothing is missing from it that the default mode would have cloned. +- **`all`**: clone all submodules of every dependency recursively, ignoring what the dependencies select. Use this when a dependency has not declared a submodule that you need — for instance because it does not use `git_submodules` yet. ### Selecting submodules per dependency -A package maintainer who knows which submodules are actually needed can restrict cloning to those submodules by adding a `git_submodules` list to the dependency's own `Bender.yml`: +A package maintainer declares which submodules the package actually needs with a `git_submodules` list in the package's own `Bender.yml`: ```yaml git_submodules: @@ -133,12 +130,12 @@ git_submodules: # pd/deps/ihp-130-pdk -> omitted, so it is not cloned ``` -- **No `git_submodules` field** (the default): all submodules are cloned recursively. -- **`git_submodules` present**: only the listed submodules are cloned; an empty list (`git_submodules: []`) clones none. +- **`git_submodules` present**: only the listed submodules are cloned. +- **No `git_submodules` field** (the default): no submodules are cloned, and Bender warns (`W37`) with the list of submodules it skipped. Add the submodules you need to the manifest to silence the warning, or use `git_submodules: []` to state explicitly that none are needed. - **`recursive`** (default `true`): also update the submodule's own nested submodules. Set it to `false` to fetch only the top-level submodule. - **`shallow`** (default `true`): fetch the submodule with `--depth 1`. Set it to `false` to clone the full submodule history. -The global `git_submodules: false` / `--git-submodules false` switch always wins: when submodule cloning is disabled globally, the per-dependency list is ignored and no submodules are cloned. +Note that the list only takes effect for a package that is checked out *as a dependency*; the one in your own root manifest is not applied to your working copy. A `git_submodules` setting in the consumer's configuration or on the command line overrides the list, as described above. ## Version Resolution and the Lockfile diff --git a/src/cli.rs b/src/cli.rs index b16e6f30..13e47212 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -25,7 +25,7 @@ use crate::Result; use crate::cmd; use crate::cmd::fusesoc::FusesocArgs; use crate::config::{ - Config, Manifest, Merge, PartialConfig, PrefixPaths, Validate, ValidationContext, + Config, Manifest, Merge, PartialConfig, PrefixPaths, SubmoduleMode, Validate, ValidationContext, }; use crate::diagnostic::{Diagnostics, Warnings}; use crate::lockfile::*; @@ -61,14 +61,15 @@ struct Cli { )] git_throttle: Option, - /// Clone the git submodules of dependencies [default: true] + /// Override which submodules are cloned for dependencies #[arg( long, global = true, + value_name = "MODE", help_heading = "Global Options", env = "BENDER_GIT_SUBMODULES" )] - git_submodules: Option, + git_submodules: Option, /// Suppresses specific warnings. Use `all` to suppress all warnings. #[arg(long, global = true, action = ArgAction::Append, help_heading = "Global Options", env = "BENDER_SUPPRESS_WARNINGS")] @@ -210,10 +211,8 @@ pub fn main() -> Result<()> { let mut config = load_config(&root_dir, matches!(cli.command, Commands::Update(_)))?; // The `--git-submodules` CLI flag (or `BENDER_GIT_SUBMODULES`) takes - // precedence over the configuration files in both directions. - if let Some(v) = cli.git_submodules { - config.git_submodules = v; - } + // precedence over the configuration files. + config.git_submodules = cli.git_submodules.unwrap_or(config.git_submodules); log::debug!("{:#?}", config); // Determine git throttle. The precedence is: CLI argument, env variable, config file, default (4). diff --git a/src/config.rs b/src/config.rs index 5840566e..98d005ba 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1679,6 +1679,21 @@ where } } +/// Which submodules of a dependency are cloned. +/// +/// The variant docs double as the `--git-submodules` help text. +#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug, Default, clap::ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum SubmoduleMode { + /// Clone all submodules of every dependency, recursively + All, + /// Clone no submodules at all + None, + /// Clone the submodules each dependency selects in its own manifest [default] + #[default] + Manifest, +} + /// A configuration. /// /// This struct encapsulates every setting of the tool that can be changed by @@ -1702,8 +1717,9 @@ pub struct Config { pub git_throttle: Option, /// Enable git LFS support, requires git-lfs (default: true) pub git_lfs: bool, - /// Clone the git submodules of dependencies (default: true) - pub git_submodules: bool, + /// Which submodules of a dependency are cloned (default: the selection in + /// each dependency's own manifest) + pub git_submodules: SubmoduleMode, } /// A partial configuration. @@ -1723,8 +1739,9 @@ pub struct PartialConfig { pub git_throttle: Option, /// Enable git LFS support, requires git-lfs (default: true) pub git_lfs: Option, - /// Clone the git submodules of dependencies (default: true) - pub git_submodules: Option, + /// Which submodules of a dependency are cloned (default: the selection in + /// each dependency's own manifest) + pub git_submodules: Option, } impl PartialConfig { @@ -1774,11 +1791,10 @@ impl Merge for PartialConfig { (Some(v1), Some(v2)) => Some(v1 | v2), (None, None) => None, }, - git_submodules: match (self.git_submodules, other.git_submodules) { - (Some(v), None) | (None, Some(v)) => Some(v), - (Some(v1), Some(v2)) => Some(v1 | v2), - (None, None) => None, - }, + // Unlike `git_lfs`, this is not an "either side may enable it" + // switch but a mode selection, so the higher-precedence side wins + // outright. + git_submodules: self.git_submodules.or(other.git_submodules), } } } @@ -1814,7 +1830,7 @@ impl Validate for PartialConfig { }, git_throttle: self.git_throttle, git_lfs: self.git_lfs.unwrap_or(true), - git_submodules: self.git_submodules.unwrap_or(true), + git_submodules: self.git_submodules.unwrap_or_default(), }) } } diff --git a/src/diagnostic.rs b/src/diagnostic.rs index 4e11915f..b41045c6 100644 --- a/src/diagnostic.rs +++ b/src/diagnostic.rs @@ -434,18 +434,34 @@ pub enum Warnings { OverrideFilesIgnored(String), #[error( - "Git submodules are disabled, dependency {} has submodules that were not checked out:\n{}", + "Git submodules are disabled, dependency {} selects submodules that were not checked out:\n{}", fmt_pkg!(.0), .1.iter().map(|p| format!(" - {}", fmt_path!(p))).collect::>().join("\n") )] #[diagnostic( code(W36), help( - "Re-run with `--git-submodules true` (or set `git_submodules: true` in the configuration), or run `git -C \"$(bender path {})\" submodule update --init --recursive` to fetch them manually.", + "Drop the `git_submodules: none` setting to clone the submodules {} selects, or use `all` to clone all of them.\nTo fetch them manually, run `git -C \"$(bender path {})\" submodule update --init -- `.", + fmt_pkg!(.0), fmt_pkg!(.0) ) )] SubmodulesDisabled(String, Vec), + + #[error( + "Dependency {} has submodules but does not declare which of them to clone, so none were checked out:\n{}", + fmt_pkg!(.0), + .1.iter().map(|p| format!(" - {}", fmt_path!(p))).collect::>().join("\n") + )] + #[diagnostic( + code(W37), + help( + "Add a `git_submodules` list to the {} manifest to select the submodules it needs, or `git_submodules: []` if none are needed to silence this warning.\nTo fetch them anyway, re-run with `--git-submodules all` or run `git -C \"$(bender path {})\" submodule update --init --recursive`.", + fmt_pkg!(.0), + fmt_pkg!(.0) + ) + )] + SubmodulesUnspecified(String, Vec), } #[derive(Error, Diagnostic, Debug, Clone)] diff --git a/src/sess.rs b/src/sess.rs index 092aa91f..81882037 100644 --- a/src/sess.rs +++ b/src/sess.rs @@ -34,7 +34,7 @@ use tokio::sync::Semaphore; use typed_arena::Arena; use crate::cli::read_manifest; -use crate::config::{self, Config, Manifest, PartialManifest}; +use crate::config::{self, Config, Manifest, PartialManifest, SubmoduleMode}; use crate::diagnostic::{Diagnostics, Errors, Warnings}; use crate::git::Git; use crate::lock::FsLock; @@ -1325,78 +1325,87 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> { } } if path.join(".gitmodules").exists() { - if self.sess.config.git_submodules { - // Re-read the dependency's own manifest (now checked out on - // disk) to see whether it restricts which of its submodules - // should be cloned. Source/include-dir existence is ignored - // here (via `validate_ignore_sources`) so that excluding a - // submodule that holds such paths does not defeat the - // selection; a missing or unparseable manifest falls back to - // cloning all submodules recursively. - let selection = std::fs::File::open(path.join("Bender.yml")) - .ok() - .and_then(|file| { - serde_yaml_ng::from_reader::<_, PartialManifest>(file).ok() - }) - .and_then(|partial| partial.validate_ignore_sources().ok()) - .and_then(|m| m.git_submodules); - - match selection { - // No `git_submodules` field: clone all submodules recursively. - None => { - let pb = Some(ProgressHandler::new( - self.sess.multiprogress.clone(), - GitProgressOps::Submodule, - name, - )); - local_git - .clone() - .spawn_with( - move |c| { - c.arg("submodule") - .arg("update") - .arg("--init") - .arg("--recursive") - .arg("--progress") - }, - pb, - ) - .await?; + match self.sess.config.git_submodules { + // Submodules were forced on via the `--git-submodules` flag, + // which overrides the dependency's own selection and clones + // all of them recursively. + SubmoduleMode::All => { + let pb = Some(ProgressHandler::new( + self.sess.multiprogress.clone(), + GitProgressOps::Submodule, + name, + )); + local_git + .clone() + .spawn_with( + move |c| { + c.arg("submodule") + .arg("update") + .arg("--init") + .arg("--recursive") + .arg("--progress") + }, + pb, + ) + .await?; + } + // Submodules were disabled via the `--git-submodules` flag, + // so they are left unchecked out. Only warn about the + // submodules the dependency actually selects: those are the + // ones that are missing compared to the default mode, for a + // dependency that selects none nothing is lost. + SubmoduleMode::None => { + let selected: Vec = submodule_selection(path) + .unwrap_or_default() + .into_iter() + .map(|sub| sub.path) + .collect(); + if !selected.is_empty() { + Warnings::SubmodulesDisabled(name.to_string(), selected).emit(); } - // `git_submodules` present: clone only the listed - // submodules (an empty list clones none). - Some(subs) => { - for sub in subs { - let pb = Some(ProgressHandler::new( - self.sess.multiprogress.clone(), - GitProgressOps::Submodule, - name, - )); - local_git - .clone() - .spawn_with( - move |c| { - c.arg("submodule").arg("update").arg("--init"); - if sub.recursive { - c.arg("--recursive"); - } - if sub.shallow { - c.arg("--depth").arg("1"); - } - c.arg("--progress").arg("--").arg(&sub.path) - }, - pb, - ) - .await?; + } + // No override: consult the dependency's own manifest (now + // checked out on disk) for which of its submodules should be + // cloned. + SubmoduleMode::Manifest => { + match submodule_selection(path) { + // No `git_submodules` field: the dependency does not + // declare which submodules it needs, so none are + // cloned. Warn, listing the affected submodules. + None => { + let submodules = local_git.clone().submodule_paths().await?; + Warnings::SubmodulesUnspecified(name.to_string(), submodules) + .emit(); + } + // `git_submodules` present: clone only the listed + // submodules (an empty list clones none). + Some(subs) => { + for sub in subs { + let pb = Some(ProgressHandler::new( + self.sess.multiprogress.clone(), + GitProgressOps::Submodule, + name, + )); + local_git + .clone() + .spawn_with( + move |c| { + c.arg("submodule").arg("update").arg("--init"); + if sub.recursive { + c.arg("--recursive"); + } + if sub.shallow { + c.arg("--depth").arg("1"); + } + c.arg("--progress").arg("--").arg(&sub.path) + }, + pb, + ) + .await?; + } } } } - } else { - // Submodules were disabled via the `--git-submodules` flag, - // so they are left unchecked out. Warn the user, listing the - // affected submodules and how to fetch them back. - let submodules = local_git.clone().submodule_paths().await?; - Warnings::SubmodulesDisabled(name.to_string(), submodules).emit(); } } } @@ -1984,6 +1993,21 @@ impl<'io, 'sess: 'io, 'ctx: 'sess> SessionIo<'sess, 'ctx> { } } +/// Read the `git_submodules` selection from a checked-out dependency's own +/// manifest. +/// +/// Source/include-dir existence is ignored here (via `validate_ignore_sources`) +/// so that excluding a submodule that holds such paths does not defeat the +/// selection; a missing or unparseable manifest is treated like a manifest +/// without a selection. +fn submodule_selection(path: &Path) -> Option> { + std::fs::File::open(path.join("Bender.yml")) + .ok() + .and_then(|file| serde_yaml_ng::from_reader::<_, PartialManifest>(file).ok()) + .and_then(|partial| partial.validate_ignore_sources().ok()) + .and_then(|manifest| manifest.git_submodules) +} + /// An arena container where all incremental, temporary things are allocated. pub struct SessionArenas { /// An arena to allocate paths in.