package manager: no dependency named 'X' in 'build.zig.zon'; but it is

[emidoots]

Zig Version

0.11.0-dev.3886+0c1bfe271

Steps to Reproduce and Observed Behavior

I'm encountering an issue where we get this error:

no dependency named 'vulkan_headers' in '/Users/slimsag/.cache/zig/p/12200a907165afc4c099d4d19e2f7ce9923a72116006d8f5f5af3a9a071fa66171d2/build.zig.zon'. All packages used in build.zig must be declared in this file.

The .cache/zig/.../build.zig.zon file in the error message has these contents:

.{
    .name = "glfw",
    .version = "0.0.0",
    .dependencies = .{
        .vulkan_headers = .{
            .url = "https://github.com/hexops/vulkan-headers/archive/e1b061ff624531e82a7daea5dbd56c07e08bf0be.tar.gz",
            .hash = "12204967f9d07fda3af6a21d6ab71fb832da2186176c530a7a14aa96043537399e6b",
        },
        .wayland_headers = .{
            .url = "https://github.com/hexops/wayland-headers/archive/7bf46f3d60edab2171643ff85770c392ef62704d.tar.gz",
            .hash = "1220689912c40c0880b094f748199c490810d4fa4c58ce1ef9058ce97f46ee28496a",
        },
        .x11_headers = .{
            .url = "https://github.com/hexops/x11-headers/archive/1db4df130531c6889df664541bc62161510b9189.tar.gz",
            .hash = "12208b3ccaec785e9344472c08bada6137d60666f0d52fc8cf62aae4251056db7737",
        },
    },
}

My project has a package dependency tree which looks like this (I've excluded some deps for brevity):

-> mach_gpu_dawn
  -> vulkan_headers
  -> mach_glfw
    -> glfw
      -> vulkan_headers

When I build the mach_glfw package, I do not encounter this error. Only when building the mach_gpu_dawn package does this error occur.

The generated `dependencies.zig` in `zig-cache` is this

pub const imports = struct {
    pub const @"mach_glfw.glfw.vulkan_headers" = @import("12204967f9d07fda3af6a21d6ab71fb832da2186176c530a7a14aa96043537399e6b");
    pub const @"mach_glfw.glfw.wayland_headers" = @import("1220689912c40c0880b094f748199c490810d4fa4c58ce1ef9058ce97f46ee28496a");
    pub const @"mach_glfw.glfw.x11_headers" = @import("12208b3ccaec785e9344472c08bada6137d60666f0d52fc8cf62aae4251056db7737");
    pub const @"mach_glfw.glfw" = @import("12200a907165afc4c099d4d19e2f7ce9923a72116006d8f5f5af3a9a071fa66171d2");
    pub const @"mach_glfw.direct3d_headers" = @import("12200d2155216c5eb5f111282cd355b5433cad6a68fd040294e695149cba329f7c12");
    pub const @"mach_glfw.vulkan_headers" = @import("12204967f9d07fda3af6a21d6ab71fb832da2186176c530a7a14aa96043537399e6b");
    pub const @"mach_glfw.wayland_headers" = @import("1220689912c40c0880b094f748199c490810d4fa4c58ce1ef9058ce97f46ee28496a");
    pub const @"mach_glfw.x11_headers" = @import("1220e6bd3186841c1da38d862d52ba88dec9633d24f409eda27627321937419a0ddb");
    pub const mach_glfw = @import("122017075bd50eebd5ee79655ffae799eb5d5193ff06e610c4de5c45752fa127b0fa");
    pub const glfw = @import("12200a907165afc4c099d4d19e2f7ce9923a72116006d8f5f5af3a9a071fa66171d2");
    pub const direct3d_headers = @import("12200d2155216c5eb5f111282cd355b5433cad6a68fd040294e695149cba329f7c12");
    pub const vulkan_headers = @import("12204967f9d07fda3af6a21d6ab71fb832da2186176c530a7a14aa96043537399e6b");
    pub const wayland_headers = @import("1220689912c40c0880b094f748199c490810d4fa4c58ce1ef9058ce97f46ee28496a");
    pub const x11_headers = @import("1220e6bd3186841c1da38d862d52ba88dec9633d24f409eda27627321937419a0ddb");
};
pub const build_root = struct {
    pub const mach_glfw = "/Users/slimsag/.cache/zig/p/122017075bd50eebd5ee79655ffae799eb5d5193ff06e610c4de5c45752fa127b0fa";
    pub const @"mach_glfw.glfw" = "/Users/slimsag/.cache/zig/p/12200a907165afc4c099d4d19e2f7ce9923a72116006d8f5f5af3a9a071fa66171d2";
    pub const @"mach_glfw.glfw.vulkan_headers" = "/Users/slimsag/.cache/zig/p/12204967f9d07fda3af6a21d6ab71fb832da2186176c530a7a14aa96043537399e6b";
    pub const @"mach_glfw.glfw.wayland_headers" = "/Users/slimsag/.cache/zig/p/1220689912c40c0880b094f748199c490810d4fa4c58ce1ef9058ce97f46ee28496a";
    pub const @"mach_glfw.glfw.x11_headers" = "/Users/slimsag/.cache/zig/p/12208b3ccaec785e9344472c08bada6137d60666f0d52fc8cf62aae4251056db7737";
    pub const @"mach_glfw.direct3d_headers" = "/Users/slimsag/.cache/zig/p/12200d2155216c5eb5f111282cd355b5433cad6a68fd040294e695149cba329f7c12";
    pub const @"mach_glfw.vulkan_headers" = "/Users/slimsag/.cache/zig/p/12204967f9d07fda3af6a21d6ab71fb832da2186176c530a7a14aa96043537399e6b";
    pub const @"mach_glfw.wayland_headers" = "/Users/slimsag/.cache/zig/p/1220689912c40c0880b094f748199c490810d4fa4c58ce1ef9058ce97f46ee28496a";
    pub const @"mach_glfw.x11_headers" = "/Users/slimsag/.cache/zig/p/1220e6bd3186841c1da38d862d52ba88dec9633d24f409eda27627321937419a0ddb";
    pub const glfw = "/Users/slimsag/.cache/zig/p/12200a907165afc4c099d4d19e2f7ce9923a72116006d8f5f5af3a9a071fa66171d2";
    pub const direct3d_headers = "/Users/slimsag/.cache/zig/p/12200d2155216c5eb5f111282cd355b5433cad6a68fd040294e695149cba329f7c12";
    pub const vulkan_headers = "/Users/slimsag/.cache/zig/p/12204967f9d07fda3af6a21d6ab71fb832da2186176c530a7a14aa96043537399e6b";
    pub const wayland_headers = "/Users/slimsag/.cache/zig/p/1220689912c40c0880b094f748199c490810d4fa4c58ce1ef9058ce97f46ee28496a";
    pub const x11_headers = "/Users/slimsag/.cache/zig/p/1220e6bd3186841c1da38d862d52ba88dec9633d24f409eda27627321937419a0ddb";
};

Notably, we can see in the above that the dependency is there:

    pub const @"mach_glfw.glfw.vulkan_headers" = @import("12204967f9d07fda3af6a21d6ab71fb832da2186176c530a7a14aa96043537399e6b");

I modified this code in std.Build to help debug this:

zig/lib/std/Build.zig

Lines 1523 to 1541 in b9fc0d2

	pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
	const build_runner = @import("root");
	const deps = build_runner.dependencies;
	inline for (@typeInfo(deps.imports).Struct.decls) |decl| {
	if (mem.startsWith(u8, decl.name, b.dep_prefix) and
	mem.endsWith(u8, decl.name, name) and
	decl.name.len == b.dep_prefix.len + name.len)
	{
	const build_zig = @field(deps.imports, decl.name);
	const build_root = @field(deps.build_root, decl.name);
	return dependencyInner(b, name, build_root, build_zig, args);
	}
	}
	const full_path = b.pathFromRoot("build.zig.zon");
	std.debug.print("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file.\n", .{ name, full_path });
	process.exit(1);
	}

I printed the decl.name for each dependency, and we can see it ends up there:

mach_glfw.glfw.vulkan_headers <-- should match, but doesn't
mach_glfw.glfw.wayland_headers
mach_glfw.glfw.x11_headers
mach_glfw.glfw
mach_glfw.direct3d_headers
mach_glfw.vulkan_headers
mach_glfw.wayland_headers
mach_glfw.x11_headers
mach_glfw
glfw
direct3d_headers
vulkan_headers
wayland_headers
x11_headers

In short, it's looking for glfw.vulkan_headers but mach_glfw.glfw.vulkan_headers is what should be matching:

• name is "vulkan_headers"
• b.dep_prefix is "glfw."
• mem.startsWith(u8, decl.name, b.dep_prefix) is false
• mem.endsWith(u8, decl.name, name) is true
• decl.name.len == b.dep_prefix.len + name.len is false

But this is where my knowledge falls apart; I can see the problem clearly, but I'm not sure what to do about it:

• Does b.dep_prefix need to be corrected to be "mach_glfw.glfw."? If so, how?
• Or does b.dependency logic need to be relaxed to allow for glfw.vulkan_headers to match mach_glfw.glfw.vulkan_headers?
• Or should dependencies.zig prefixes be corrected to contain glfw.vulkan_headers instead of mach_glfw.glfw.vulkan_headers?

Expected Behavior

dependency matching works as expected, the import succeeds

[emidoots]

Apologies, I think I know why this is happening now and believe it's my own fault. I'm confirming now.

[andrewrk]

Even if it ends up being ultimately on your end, feel free to suggest an enhancement to the zig toolchain that could have helped you diagnose the problem more easily 🙂

[emidoots]

Confirmed, this was my fault / not a bug in Zig. Here's the summary:

1. When a dependency is listed in build.zig.zon, it is possible to import that dependency's build.zig file. For example, .mach_glfw = .{} can be imported as @import("mach_glfw") giving you access to mach_glfw's build.zig file.

• I do not know if this was intentional behavior of the package manager, I suspect not, and forbidding it may prevent this issue. But for now, we are relying on this behavior in Mach in a few places (and I know others are too in other Zig projects.)

2. When abusing (1) above to import a dependency's build.zig file, if that file then uses b.dependency("foo") then b.dep_prefix doesn't reflect the dependency's prefix, but rather the parent packages' prefix. It's no longer a well-defined value.

Ultimately I think this is caused by abusing (1), which is probably unintended behavior of the package manager, and there should instead be a different/better way for a package to expose build-time code.

[andrewrk]

Ability to import directly the dependency in build.zig (and have the import root be the package's build.zig file) is intentional behavior, and it sounds like it's useful given that several projects have already discovered it and started using it.

I'm not sure I understand your second point - where would b come from? I was thinking that the package would provide pub declarations inside build.zig which are potentially useful to use in a build script. If your goal is to call the build function on a dependency in this manner, that is precisely what b.dependency() does (and making sure to create a proper *Build object for the dependency package, so that stuff like b.dep_prefix will work correctly).

[linkpy]

I do think there is a bug with how dependencies are discovered (or replicated when the same dependency is present multiple time within a project dependency tree).

I'm trying to build mach-gpu, https://github.com/hexops/mach-gpu/tree/1266bd935c20d539d74390380364ec566be4eec9 this commit exactly but changed to use mach_gpu as a zig dependency instead of a git submodule. Thus, this was added in the build.zig.zon :

        .mach_gpu_dawn = .{
            .url = "https://github.com/hexops/mach-gpu-dawn/archive/a33e4030dcc0d6fa6c0abf3419000b504074511a.tar.gz",
            .hash = "122078af196fd94860b4050edb55605e3cd4332405b8edb56718767e2433dd47975d",
        },

By modifying std/Build.zig to get more debug info, I found out that for the package glfw (https://github.com/hexops/glfw/tree/06bd1276fb05c8bc999e104cf21c76788386f263 this commit exactly) doesnt have its dependencies properly created in @import("@dependencies"). It searches for the dependency mach_glfw.glfw.vulkan_headers which exists in the build.zig.zon (pointing to https://github.com/hexops/vulkan-headers/tree/e1b061ff624531e82a7daea5dbd56c07e08bf0be exactly) and yet, build_runner.dependencies.imports doesnt contain it :

.{
	.{.name = "mach_gpu_dawn.mach_glfw.glfw.vulkan_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw.glfw.wayland_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw.glfw.x11_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw.glfw", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw.direct3d_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw.vulkan_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw.wayland_headers", .is_pub = true},
	.{.name = "mach_gpu_dawn.mach_glfw.x11_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.glfw", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.direct3d_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.vulkan_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.wayland_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.x11_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn", .is_pub = true}, 
	.{.name = "mach_glfw.glfw", .is_pub = true}, 
	.{.name = "mach_glfw.direct3d_headers", .is_pub = true}, 
	.{.name = "mach_glfw.vulkan_headers", .is_pub = true}, 
	.{.name = "mach_glfw.wayland_headers", .is_pub = true}, 
	.{.name = "mach_glfw.x11_headers", .is_pub = true}, 
	.{.name = "mach_glfw", .is_pub = true}, 
	.{.name = "glfw.vulkan_headers", .is_pub = true}, 
	.{.name = "glfw.wayland_headers", .is_pub = true}, 
	.{.name = "glfw.x11_headers", .is_pub = true}, 
	.{.name = "glfw", .is_pub = true}, 
	.{.name = "direct3d_headers", .is_pub = true}, 
	.{.name = "vulkan_headers", .is_pub = true}, 
	.{.name = "wayland_headers", .is_pub = true}, 
	.{.name = "x11_headers", .is_pub = true} 
}

(result of @compileLog(@typeInfo(build_runner.dependencies.imports).Struct.decls))

Since the "dependency resolution" seemed to have worked fine in mach_gpu_dawn for glfw (as u can see at the top, mach_gpu_dawn.mach_glfw.glfw.vulkan_headers is present), could it be that once that process have been done once on a package, it is not repeated afterward even if used elsewhere?

Or maybe that the result is not replicated when the dep_prefix is different? I am not really sure where that process takes place in the code (seems to be done directly within the compiler), so on my end I can't really tell.

The issue with my idea is that, even if the dependencies werent duplicated for glfw, it was correctly done for mach_glfw :

.{
	.{.name = "mach_gpu_dawn.mach_glfw.glfw.vulkan_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw.glfw.wayland_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw.glfw.x11_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw.glfw", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw.direct3d_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw.vulkan_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw.wayland_headers", .is_pub = true},
	.{.name = "mach_gpu_dawn.mach_glfw.x11_headers", .is_pub = true}, 
	.{.name = "mach_gpu_dawn.mach_glfw", .is_pub = true}, 
	[...]
	.{.name = "mach_glfw.glfw", .is_pub = true}, 
	.{.name = "mach_glfw.direct3d_headers", .is_pub = true}, 
	.{.name = "mach_glfw.vulkan_headers", .is_pub = true}, 
	.{.name = "mach_glfw.wayland_headers", .is_pub = true}, 
	.{.name = "mach_glfw.x11_headers", .is_pub = true}, 
	.{.name = "mach_glfw", .is_pub = true}, 
        [...]
}

Edit: I thought that maybe it was an issue with the commit/hash of the packages, so I went around and ensured that all of the commits and hashs of each package in every build.zig.zon files matched, but the issue still happens.

[mlugg]

Ah, yes, looking at the code, there is a bug there. I have no idea why it worked for mach_glfw, but in general, we'll only recurse into fetchAndAddDependencies for a given package one time, so we only recurse into its children for the first FQN-path encountered.

Fixing this leads to a design question for @andrewrk: should recursive dependencies between packages be allowed? (If so, we'll need to move away from the FQN thing, since a.b.a.b.a.b.a.b... would be a valid name.) I have no strong opinion on this.

[emidoots]

I'm not sure I understand your second point - where would b come from?

You're right, I didn't think this through enough before drawing my conclusion.

I have no idea why it worked for mach_glfw

Yes, I'm not sure either - but the behavior did change when I stopped importing the build.zig. It's probably not useful to continue discussing Mach code here, if needed I can put together a smaller reproduction case of the issue - but it sounds like @mlugg sees the issue so I'll hold off.

[mitchellh]

I don't have a reproduction, but this issue is blocking me from being able to update to use the package manager.

I have multiple triangle-shaped dependencies. Concretely, this is what it looks like. All of the transitive dependencies (i.e. zlib) currently match exact hashes in all repositories. More info below the code block.

ghostty
|-- zlib
|-- libpng
    |-- zlib
|-- freetype
    |-- zlib
    |-- libpng
         |-- zlib
|-- harfbuzz
    |-- freetype
        |-- <all of the above>
|-- fontconfig
    |-- freetype
        |-- <all of the above>

• When I added zlib, it was okay.
• When I added libpng, it was okay.
• When I added freetype, I got:

$ zig build run
no dependency named 'zlib' in '/Users/mitchellh/.cache/zig/p/1220d0134e1e0887e4697322ea31bbf84fc4583917ac048b49c6ebf8505e2c916a91/build.zig.zon'. All packages used in build.zig must be declared in this file.
error: the following build command failed with exit code 1:
/Users/mitchellh/code/go/src/github.com/mitchellh/ghostty/zig-cache/o/da150eed8490a69cd88546b32dbbe647/build /nix/store/gnzzphczp5v5vqmdbhbvnhbl5xw5kixc-zig-0.11.0-dev.4404+4f6013bf5/bin/zig /Users/mitchellh/code/go/src/github.com/mitchellh/ghostty /Users/mitchellh/code/go/src/github.com/mitchellh/ghostty/zig-cache /Users/mitchellh/.cache/zig run

And the build.zig.zon linked is the libpng one.

So it seems: once you have two dependencies that share a transitive dependency grandchild-level deep, something breaks. I can't find a workaround for this scenario. @slimsag shared a workaround with me that works for the mach dependency (I hit the original issue too) but it doesn't work for my freetype scenario.

[mitchellh]

I've found a workaround based on what @slimsag suggested and it looks like this. Yikes!

const freetype_dep = b.dependency("freetype", .{ .target = step.target, .optimize = step.optimize });
const libpng_dep = b.dependency("freetype.libpng", .{
    .target = step.target,
    .optimize = step.optimize,
});
const zlib_dep = b.dependency("freetype.libpng.zlib", .{
    .target = step.target,
    .optimize = step.optimize,
});

What I'd rather do which probably is correct once this bug is fixed:

const freetype_dep = b.dependency("freetype", .{ .target = step.target, .optimize = step.optimize });
const libpng_dep = b.dependency("libpng", .{ .target = step.target, .optimize = step.optimize });
const zlib_dep = b.dependency("zlib", .{ .target = step.target, .optimize = step.optimize });

It's possible as I continue going further down the transitive path here that I may not be able to work around it. I'll update this if so.