diff --git a/Dockerfile b/Dockerfile index 57abb96a..67791779 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,33 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ -ldflags "-X main.version=${VERSION} -X main.gitCommit=${GIT_COMMIT} -X main.gitDirty=${GIT_DIRTY} -X main.buildDate=${BUILD_DATE}" \ -o manager ./cmd +# Bundle git and all its shared-library deps (musl libc, openssl, curl, etc.) so that +# the final distroless image can shell out to git for Azure DevOps repositories (which +# require multi_ack capability that go-git v5 does not implement). +FROM alpine:3.24@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b AS git-bundle +RUN set -eux; \ + apk add --no-cache git openssh-client; \ + mkdir -p /bundle/bin /bundle/lib /bundle/usr/lib /bundle/usr/libexec; \ + cp -L /usr/bin/git /bundle/bin/; \ + cp -L /usr/bin/ssh /bundle/bin/; \ + cp -rL /usr/libexec/git-core /bundle/usr/libexec/; \ + cp -L /lib/ld-musl*.so* /bundle/lib/; \ + for f in /usr/bin/git \ + /usr/bin/ssh \ + /usr/libexec/git-core/git-remote-http \ + /usr/libexec/git-core/git-remote-https; do \ + [ -f "$f" ] || continue; \ + ldd "$f" 2>/dev/null \ + | awk '/ => /{ print $3 }' \ + | while read -r so; do \ + [ -f "$so" ] || continue; \ + case "$so" in \ + /lib/*) cp -Ln "$so" /bundle/lib/ 2>/dev/null || true ;; \ + /usr/lib/*) cp -Ln "$so" /bundle/usr/lib/ 2>/dev/null || true ;; \ + esac; \ + done; \ + done + FROM alpine:3.24@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b AS sops-downloader ARG TARGETARCH # Keep current: the CI image-scan gate fails on fixable CRITICALs in this @@ -69,6 +96,15 @@ FROM gcr.io/distroless/static:debug@sha256:e741251ccc55dd6cec4a99ff21c0766df3189 WORKDIR / COPY --from=builder /workspaces/manager . COPY --from=sops-downloader /usr/local/bin/sops /usr/local/bin/sops +# git, ssh, and their musl-linked deps for the ADO system-git fallback. +# busybox provides /bin/sh so git can invoke GIT_SSH_COMMAND via shell parsing. +# Alpine's busybox is musl-linked and needs only the libs already copied below. +COPY --from=git-bundle /bin/busybox /bin/sh +COPY --from=git-bundle /bundle/bin/git /usr/bin/git +COPY --from=git-bundle /bundle/bin/ssh /usr/bin/ssh +COPY --from=git-bundle /bundle/usr/libexec/git-core /usr/libexec/git-core +COPY --from=git-bundle /bundle/lib/ /lib/ +COPY --from=git-bundle /bundle/usr/lib/ /usr/lib/ USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/docs/configuration.md b/docs/configuration.md index 5c5aaf4a..ee4baeb0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -92,6 +92,29 @@ spec: - main ``` +### Azure DevOps repositories + +Azure DevOps (`dev.azure.com`, `*.visualstudio.com`, `ssh.dev.azure.com`) is supported. +Use **HTTPS with a Personal Access Token** as the recommended credential. ADO PATs must be +sent as HTTP Basic auth with an empty username and the PAT as the password: + +```yaml +spec: + url: https://dev.azure.com///_git/ + secretRef: + name: ado-creds # Secret with keys: username (empty string) and password (your PAT) +``` + +Microsoft Entra ID (OAuth) access tokens use the `bearerToken` Secret key instead. +SSH is also supported using the `ssh.dev.azure.com` URL format with standard `ssh-privatekey` and `known_hosts` credentials. + +> **Implementation note:** go-git v5 does not implement the `multi_ack` capability that ADO +> requires (ADO rejects requests without it with HTTP 400). The operator automatically routes +> ADO URLs through the system `git` binary instead. This is transparent — no configuration +> change is needed — but requires `git` to be present in the container image. The published +> image includes it. This fallback will be removed once go-git v6 ships with full `multi_ack` +> support (tracked in go-git PR #1204). + ### `GitProvider.spec.secretRef`: the credentials Secret The referenced Secret holds the Git credentials. The examples use the **Kubernetes-native** keys, diff --git a/internal/git/ado_system_git.go b/internal/git/ado_system_git.go new file mode 100644 index 00000000..2b67532d --- /dev/null +++ b/internal/git/ado_system_git.go @@ -0,0 +1,482 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "bufio" + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/storage/filesystem" + "sigs.k8s.io/controller-runtime/pkg/log" + + sshpkg "github.com/ConfigButler/gitops-reverser/internal/ssh" +) + +// lsRemoteFields is the number of tab-separated fields in a git ls-remote line. +const lsRemoteFields = 2 + +// credentialPattern matches HTTP/HTTPS URLs that embed credentials as userinfo +// (https://user:password@host/...). Used to redact secrets from error messages. +var credentialPattern = regexp.MustCompile(`(?i)(https?://)[^:@\s]+:[^@\s]+@`) + +// IsADOURL reports whether rawURL targets Azure DevOps. go-git v5 cannot negotiate +// the multi_ack capability that ADO requires; those repositories must use the system +// git fallback. +// +// Detection is based on the parsed hostname, not substring matching, to prevent +// crafted URLs (e.g. ext:: remote helpers) from hijacking the routing decision. +func IsADOURL(rawURL string) bool { + host, ok := parseGitURLHost(rawURL) + if !ok { + return false + } + lower := strings.ToLower(host) + return lower == "dev.azure.com" || lower == "ssh.dev.azure.com" || + strings.HasSuffix(lower, ".visualstudio.com") +} + +// parseGitURLHost extracts the hostname from a Git URL and validates that the +// scheme is safe for subprocess execution. It handles both standard hierarchical +// URLs (https://, ssh://) and SCP-style shorthand ([user@]host:path). +// Returns ("", false) for URLs with dangerous schemes (ext::, file::) or that +// cannot be parsed. +func parseGitURLHost(rawURL string) (string, bool) { + if u, err := url.Parse(rawURL); err == nil && u.Host != "" { + switch strings.ToLower(u.Scheme) { + case "https", "http", "ssh", "git+ssh", "ssh+git", "git": + return u.Hostname(), true + } + return "", false + } + // SCP-style: [user@]host:path — url.Parse yields no host. + // Reject anything containing whitespace first. + if strings.ContainsAny(rawURL, " \t\n\r") { + return "", false + } + s := rawURL + if at := strings.Index(s, "@"); at >= 0 { + s = s[at+1:] + } + if colon := strings.Index(s, ":"); colon > 0 { + return s[:colon], true + } + return "", false +} + +// systemGitEnv holds the environment variables needed to run git with the caller's +// credentials, and a cleanup function that removes any temp files it created. +type systemGitEnv struct { + vars []string + cleanup func() +} + +// newSystemGitEnv translates a go-git AuthMethod into environment variables consumed +// by a system git subprocess. +// +// - *http.BasicAuth → GIT_CONFIG_GLOBAL with http.extraHeader = Authorization: Basic +// - *http.TokenAuth → GIT_CONFIG_GLOBAL with http.extraHeader = Authorization: Bearer +// - *sshpkg.KeyAuth → GIT_SSH_COMMAND with -i and optional known_hosts +// - nil → anonymous (no extra env) +// - anything else → error +// +// HTTP auth types write a 0600 temp gitconfig and set GIT_CONFIG_GLOBAL. +// Credentials never appear in process argv. +// +// The repoURL parameter is used to scope HTTP credential headers to the remote +// origin so they are never sent to any other host git contacts. +func newSystemGitEnv(repoURL string, auth transport.AuthMethod) (*systemGitEnv, error) { + // Scope HTTP credential headers to the remote origin so they are never sent to + // any other host git might contact (e.g. a redirect or a submodule remote). + httpSection := "http" + if u, err := url.Parse(repoURL); err == nil && u.Scheme != "" && u.Host != "" { + httpSection = fmt.Sprintf("http %q", u.Scheme+"://"+u.Host+"/") + } + + tmpDir, err := os.MkdirTemp("", "reverser-git-*") + if err != nil { + return nil, fmt.Errorf("create temp dir for git credentials: %w", err) + } + cleanup := func() { _ = os.RemoveAll(tmpDir) } + + base := []string{ + "GIT_TERMINAL_PROMPT=0", + "GIT_CONFIG_NOSYSTEM=1", + "PATH=" + os.Getenv("PATH"), + } + + switch a := auth.(type) { + case *http.BasicAuth: + encoded := base64.StdEncoding.EncodeToString([]byte(a.Username + ":" + a.Password)) + cfgPath := filepath.Join(tmpDir, "gitconfig") + content := fmt.Sprintf("[%s]\n\textraHeader = Authorization: Basic %s\n", httpSection, encoded) + if err := os.WriteFile(cfgPath, []byte(content), 0600); err != nil { + cleanup() + return nil, fmt.Errorf("write git credential config: %w", err) + } + return &systemGitEnv{ + vars: append(base, "GIT_CONFIG_GLOBAL="+cfgPath), + cleanup: cleanup, + }, nil + + case *http.TokenAuth: + if strings.ContainsAny(a.Token, "\n\r") { + cleanup() + return nil, errors.New("bearer token contains invalid characters (newline)") + } + cfgPath := filepath.Join(tmpDir, "gitconfig") + content := fmt.Sprintf("[%s]\n\textraHeader = Authorization: Bearer %s\n", httpSection, a.Token) + if err := os.WriteFile(cfgPath, []byte(content), 0600); err != nil { + cleanup() + return nil, fmt.Errorf("write git credential config: %w", err) + } + return &systemGitEnv{ + vars: append(base, "GIT_CONFIG_GLOBAL="+cfgPath), + cleanup: cleanup, + }, nil + + case *sshpkg.KeyAuth: + if len(a.PrivateKeyPEM) == 0 { + cleanup() + return nil, errors.New( + "SSH key type is not supported by the system-git ADO fallback " + + "(key re-serialisation failed); use RSA, ECDSA, or Ed25519", + ) + } + keyFile := filepath.Join(tmpDir, "id_key") + if err := os.WriteFile(keyFile, a.PrivateKeyPEM, 0600); err != nil { + cleanup() + return nil, fmt.Errorf("write SSH private key: %w", err) + } + sshCmd := "ssh -i '" + keyFile + "' -o BatchMode=yes" + if a.KnownHosts != "" { + khFile := filepath.Join(tmpDir, "known_hosts") + if err := os.WriteFile(khFile, []byte(a.KnownHosts), 0600); err != nil { + cleanup() + return nil, fmt.Errorf("write SSH known_hosts: %w", err) + } + sshCmd += " -o StrictHostKeyChecking=yes -o UserKnownHostsFile='" + khFile + "'" + } else { + // No known_hosts supplied: accept new host keys but reject changed ones. + sshCmd += " -o StrictHostKeyChecking=accept-new" + } + return &systemGitEnv{ + vars: append(base, "GIT_SSH_COMMAND="+sshCmd), + cleanup: cleanup, + }, nil + + case nil: + return &systemGitEnv{vars: base, cleanup: cleanup}, nil + + default: + cleanup() + return nil, fmt.Errorf( + "system git fallback for ADO does not support auth type %T; use an HTTPS or SSH credential", + auth, + ) + } +} + +// run executes git with the configured environment. workDir is passed as -C when non-empty. +func (e *systemGitEnv) run(ctx context.Context, workDir string, args ...string) ([]byte, error) { + gitBin, err := exec.LookPath("git") + if err != nil { + return nil, errors.New( + "git binary not found in PATH; add git to the container image to enable the ADO fallback", + ) + } + + fullArgs := args + if workDir != "" { + fullArgs = append([]string{"-C", workDir}, args...) + } + + cmd := exec.CommandContext(ctx, gitBin, fullArgs...) + cmd.Env = e.vars + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + // Redact any embedded credentials (https://user:PAT@host) before logging. + safeArgs := make([]string, len(args)) + for i, a := range args { + safeArgs[i] = credentialPattern.ReplaceAllString(a, "${1}@") + } + safeStderr := credentialPattern.ReplaceAllString(stderr.String(), "${1}@") + return nil, fmt.Errorf("git %v: %w\nstderr: %s", safeArgs, err, safeStderr) + } + + return stdout.Bytes(), nil +} + +// remoteOriginURL returns the first URL configured for the "origin" remote, or an +// error if the remote does not exist or has no URLs (e.g. a hand-edited .git/config). +func remoteOriginURL(repo *gogit.Repository) (string, error) { + remote, err := repo.Remote("origin") + if err != nil { + return "", fmt.Errorf("get remote origin: %w", err) + } + urls := remote.Config().URLs + if len(urls) == 0 { + return "", errors.New("remote origin has no URLs configured") + } + return urls[0], nil +} + +// repoWorkDir returns the working-tree root of a filesystem-backed repository. +// Returns "" when the repository uses in-memory storage (tests). +func repoWorkDir(repo *gogit.Repository) string { + fsStorage, ok := repo.Storer.(*filesystem.Storage) + if !ok { + return "" + } + type rooter interface{ Root() string } + if r, ok := fsStorage.Filesystem().(rooter); ok { + return filepath.Dir(r.Root()) + } + return "" +} + +// parseADOLSRemote parses `git ls-remote --symref` output and returns: +// - refs: map of full-refname → SHA (e.g. "refs/heads/main" → "abc123") +// - defaultBranch: the short branch name HEAD resolves to (empty if HEAD is missing/broken) +func parseADOLSRemote(out []byte) (map[string]string, string) { + refs := make(map[string]string) + var defaultBranch string + + scanner := bufio.NewScanner(bytes.NewReader(out)) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "ref: ") { + // symbolic ref line: "ref: refs/heads/main\tHEAD" + parts := strings.SplitN(line, "\t", lsRemoteFields) + if len(parts) == lsRemoteFields && parts[1] == "HEAD" { + target := strings.TrimPrefix(parts[0], "ref: ") + if strings.HasPrefix(target, "refs/heads/") { + defaultBranch = strings.TrimPrefix(target, "refs/heads/") + } + } + continue + } + parts := strings.SplitN(line, "\t", lsRemoteFields) + if len(parts) == lsRemoteFields && parts[0] != "" && parts[1] != "" { + refs[parts[1]] = parts[0] + } + } + + return refs, defaultBranch +} + +// systemGitSmartFetch is the ADO path for SmartFetch. It uses system git to list +// remote refs and fetch the target (or default) branch, then repairs the +// remote-tracking symbolic HEAD so subsequent go-git checkout calls succeed. +func systemGitSmartFetch( + ctx context.Context, + repo *gogit.Repository, + target plumbing.ReferenceName, + auth transport.AuthMethod, +) (plumbing.ReferenceName, error) { + logger := log.FromContext(ctx) + + workDir := repoWorkDir(repo) + if workDir == "" { + return "", errors.New("systemGitSmartFetch: cannot determine repository path (in-memory storage?)") + } + + remoteURL, err := remoteOriginURL(repo) + if err != nil { + return "", err + } + + gitEnv, err := newSystemGitEnv(remoteURL, auth) + if err != nil { + return "", err + } + defer gitEnv.cleanup() + + // List remote refs to discover default branch and target existence. + out, err := gitEnv.run(ctx, workDir, "ls-remote", "--symref", "origin") + if err != nil { + return "", fmt.Errorf("ls-remote for ADO fetch: %w", err) + } + + refs, defaultBranch := parseADOLSRemote(out) + if len(refs) == 0 { + logger.Info("ADO remote is empty, nothing to fetch") + return "", nil + } + + targetFullStr := target.String() + _, targetExists := refs[targetFullStr] + defaultFull := "" + if defaultBranch != "" { + defaultFull = "refs/heads/" + defaultBranch + } + + // Determine which branch to use after fetch. + var result plumbing.ReferenceName + switch { + case targetExists: + result = target + case defaultFull != "": + result = plumbing.ReferenceName(defaultFull) + default: + return "", nil + } + + // Build refspecs (mirrors buildSmartRefSpecs logic). + remoteName := "origin" + refSpecs := buildSmartRefSpecs(remoteName, defaultFull, defaultBranch, target, targetExists) + if len(refSpecs) == 0 { + return result, nil + } + + fetchArgs := []string{"fetch", "--prune", "origin"} + for _, rs := range refSpecs { + fetchArgs = append(fetchArgs, string(rs)) + } + + logger.Info("ADO system-git fetch", "target", target.Short(), "refspecs", refSpecs) + if _, err := gitEnv.run(ctx, workDir, fetchArgs...); err != nil { + return "", fmt.Errorf("system git fetch for ADO: %w", err) + } + + repairRemoteSymbolicHead(repo, remoteName, defaultBranch) + return result, nil +} + +// systemGitPushAtomic is the ADO path for PushAtomic. It verifies that rootBranch +// has not moved (concurrent-update guard), then pushes the current branch with a +// force-with-lease so the push is rejected if another actor pushed concurrently. +func systemGitPushAtomic( + ctx context.Context, + repo *gogit.Repository, + rootHash plumbing.Hash, + rootBranch plumbing.ReferenceName, + auth transport.AuthMethod, +) error { + logger := log.FromContext(ctx) + + workDir := repoWorkDir(repo) + if workDir == "" { + return errors.New("systemGitPushAtomic: cannot determine repository path (in-memory storage?)") + } + + remoteURL, err := remoteOriginURL(repo) + if err != nil { + return err + } + + branch, localHash, err := GetCurrentBranch(repo) + if err != nil { + return fmt.Errorf("get current branch: %w", err) + } + + gitEnv, err := newSystemGitEnv(remoteURL, auth) + if err != nil { + return err + } + defer gitEnv.cleanup() + + // Query current remote state for the root branch and target branch. + lsOut, err := gitEnv.run(ctx, workDir, + "ls-remote", "origin", + rootBranch.String(), branch.String(), + ) + if err != nil { + return fmt.Errorf("ls-remote for ADO push: %w", err) + } + + remoteRefs, _ := parseADOLSRemote(lsOut) + remoteRootHash := remoteRefs[rootBranch.String()] + remoteBranchHash := remoteRefs[branch.String()] + + // Concurrency guard: abort if the root branch has been force-pushed since we fetched. + if !rootHash.IsZero() && remoteRootHash != rootHash.String() { + return errors.New("remote received unknown updates") + } + + // Nothing to push. + if remoteBranchHash == localHash.String() { + logger.Info("ADO remote already up-to-date", "branch", branch.Short()) + return nil + } + + branchShort := branch.Short() + pushArgs := []string{"push"} + + if remoteBranchHash != "" { + // Branch exists on remote: use force-with-lease to detect concurrent pushes. + pushArgs = append(pushArgs, + fmt.Sprintf("--force-with-lease=refs/heads/%s:%s", branchShort, remoteBranchHash), + ) + } + + pushArgs = append(pushArgs, "origin", fmt.Sprintf("HEAD:refs/heads/%s", branchShort)) + + logger.Info("ADO system-git push", "branch", branchShort, "hash", localHash) + if _, err := gitEnv.run(ctx, workDir, pushArgs...); err != nil { + return fmt.Errorf("system git push for ADO: %w", err) + } + + return nil +} + +// systemGitCheckRepo is the ADO path for CheckRepo. It uses `git ls-remote --symref` +// to retrieve repository metadata without a go-git transport session. +func systemGitCheckRepo(ctx context.Context, repoURL string, auth transport.AuthMethod) (*RepoInfo, error) { + logger := log.FromContext(ctx) + + gitEnv, err := newSystemGitEnv(repoURL, auth) + if err != nil { + return nil, err + } + defer gitEnv.cleanup() + + out, err := gitEnv.run(ctx, "", "ls-remote", "--symref", repoURL) + if err != nil { + return nil, fmt.Errorf("ls-remote for ADO check: %w", err) + } + + if len(bytes.TrimSpace(out)) == 0 { + logger.Info("ADO repository is empty", "url", repoURL) + return &RepoInfo{DefaultBranch: nil, RemoteBranchCount: 0}, nil + } + + refs, defaultBranch := parseADOLSRemote(out) + + info := &RepoInfo{} + for refName := range refs { + if strings.HasPrefix(refName, "refs/heads/") { + info.RemoteBranchCount++ + } + } + + if defaultBranch != "" { + defaultFullRef := "refs/heads/" + defaultBranch + sha := refs[defaultFullRef] + info.DefaultBranch = &BranchInfo{ + ShortName: defaultBranch, + Sha: sha, + Unborn: sha == "", + } + } + + logger.V(1).Info("ADO repository check completed", "remoteBranches", info.RemoteBranchCount) + return info, nil +} diff --git a/internal/git/ado_system_git_test.go b/internal/git/ado_system_git_test.go new file mode 100644 index 00000000..9c772993 --- /dev/null +++ b/internal/git/ado_system_git_test.go @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: Apache-2.0 + +package git + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "encoding/pem" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/go-git/go-git/v5/plumbing/transport/http" + gossh "golang.org/x/crypto/ssh" + + sshpkg "github.com/ConfigButler/gitops-reverser/internal/ssh" +) + +func TestIsADOURL(t *testing.T) { + tests := []struct { + url string + want bool + }{ + {"https://dev.azure.com/org/project/_git/repo", true}, + {"https://DEV.AZURE.COM/org/project/_git/repo", true}, + {"https://org.visualstudio.com/project/_git/repo", true}, + {"ssh://git@ssh.dev.azure.com:v3/org/project/repo", true}, + {"git@ssh.dev.azure.com:v3/org/project/repo", true}, + {"https://github.com/org/repo.git", false}, + {"https://gitlab.com/org/repo.git", false}, + {"git@github.com:org/repo.git", false}, + {"https://bitbucket.org/org/repo.git", false}, + {"", false}, + // Injection attempts must not match via substring. + {"ext::sh -c 'touch /tmp/x #dev.azure.com'", false}, + {"ext::dev.azure.com", false}, + {"file:///dev.azure.com", false}, + {"https://attacker.com/dev.azure.com", false}, + } + + for _, tc := range tests { + t.Run(tc.url, func(t *testing.T) { + if got := IsADOURL(tc.url); got != tc.want { + t.Errorf("IsADOURL(%q) = %v, want %v", tc.url, got, tc.want) + } + }) + } +} + +func TestNewSystemGitEnv_BasicAuth(t *testing.T) { + auth := &http.BasicAuth{Username: "user", Password: "test-password"} + env, err := newSystemGitEnv("https://dev.azure.com/org/proj/_git/repo", auth) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer env.cleanup() + + var cfgPath string + for _, v := range env.vars { + if strings.HasPrefix(v, "GIT_CONFIG_GLOBAL=") { + cfgPath = strings.TrimPrefix(v, "GIT_CONFIG_GLOBAL=") + } + } + if cfgPath == "" { + t.Fatal("GIT_CONFIG_GLOBAL not set") + } + + data, err := os.ReadFile(cfgPath) + if err != nil { + t.Fatalf("read gitconfig: %v", err) + } + content := string(data) + if !strings.Contains(content, "extraHeader") { + t.Error("gitconfig must contain http.extraHeader") + } + if !strings.Contains(content, "Basic ") { + t.Error("gitconfig must use Basic scheme") + } + // Credential must appear base64-encoded, not as plain text in the header. + if strings.Contains(content, "test-password") { + t.Error("raw password must not appear in gitconfig; should be base64-encoded") + } + + // File must be user-only readable. + info, err := os.Stat(cfgPath) + if err != nil { + t.Fatalf("stat gitconfig: %v", err) + } + if info.Mode().Perm()&0077 != 0 { + t.Errorf("gitconfig is group/world readable: mode %o", info.Mode().Perm()) + } +} + +func TestNewSystemGitEnv_TokenAuth(t *testing.T) { + auth := &http.TokenAuth{Token: "test-token"} + env, err := newSystemGitEnv("https://dev.azure.com/org/proj/_git/repo", auth) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer env.cleanup() + + var cfgPath string + for _, v := range env.vars { + if strings.HasPrefix(v, "GIT_CONFIG_GLOBAL=") { + cfgPath = strings.TrimPrefix(v, "GIT_CONFIG_GLOBAL=") + } + } + + if cfgPath == "" { + t.Fatal("GIT_CONFIG_GLOBAL not set") + } + + data, err := os.ReadFile(cfgPath) + if err != nil { + t.Fatalf("read gitconfig: %v", err) + } + if !strings.Contains(string(data), "test-token") { + t.Error("gitconfig must contain the bearer token") + } + if !strings.Contains(string(data), "extraHeader") { + t.Error("gitconfig must contain http.extraHeader") + } + + // File must be user-only readable. + info, err := os.Stat(cfgPath) + if err != nil { + t.Fatalf("stat gitconfig: %v", err) + } + if info.Mode().Perm()&0077 != 0 { + t.Errorf("gitconfig is group/world readable: mode %o", info.Mode().Perm()) + } +} + +func TestNewSystemGitEnv_Anonymous(t *testing.T) { + env, err := newSystemGitEnv("https://dev.azure.com/org/proj/_git/repo", nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + env.cleanup() + + for _, v := range env.vars { + if strings.HasPrefix(v, "GIT_ASKPASS=") || strings.HasPrefix(v, "GIT_CONFIG_GLOBAL=") { + t.Errorf("unexpected env var for anonymous auth: %s", v) + } + } +} + +type unsupportedAuth struct{} + +func (unsupportedAuth) Name() string { return "unsupported" } +func (unsupportedAuth) String() string { return "unsupported" } + +func TestNewSystemGitEnv_UnsupportedAuth(t *testing.T) { + _, err := newSystemGitEnv("https://dev.azure.com/org/proj/_git/repo", unsupportedAuth{}) + if err == nil { + t.Fatal("expected error for unsupported auth type, got nil") + } +} + +func TestNewSystemGitEnv_Cleanup(t *testing.T) { + auth := &http.BasicAuth{Username: "u", Password: "p"} + env, err := newSystemGitEnv("https://dev.azure.com/org/proj/_git/repo", auth) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Find the temp dir from the gitconfig path. + var tmpDir string + for _, v := range env.vars { + if strings.HasPrefix(v, "GIT_CONFIG_GLOBAL=") { + tmpDir = filepath.Dir(strings.TrimPrefix(v, "GIT_CONFIG_GLOBAL=")) + } + } + if tmpDir == "" { + t.Fatal("could not find temp dir") + } + + env.cleanup() + + if _, err := os.Stat(tmpDir); !os.IsNotExist(err) { + t.Errorf("temp dir %q still exists after cleanup", tmpDir) + } +} + +func TestParseADOLSRemote(t *testing.T) { + raw := []byte("ref: refs/heads/main\tHEAD\n" + + "abc123def456\tHEAD\n" + + "abc123def456\trefs/heads/main\n" + + "deadbeef0000\trefs/heads/feature\n") + + refs, defaultBranch := parseADOLSRemote(raw) + + if defaultBranch != "main" { + t.Errorf("defaultBranch = %q, want %q", defaultBranch, "main") + } + if refs["refs/heads/main"] != "abc123def456" { + t.Errorf("refs[main] = %q, want %q", refs["refs/heads/main"], "abc123def456") + } + if refs["refs/heads/feature"] != "deadbeef0000" { + t.Errorf("refs[feature] = %q, want %q", refs["refs/heads/feature"], "deadbeef0000") + } +} + +func TestParseADOLSRemote_Empty(t *testing.T) { + refs, defaultBranch := parseADOLSRemote(nil) + if defaultBranch != "" { + t.Errorf("defaultBranch = %q, want empty", defaultBranch) + } + if len(refs) != 0 { + t.Errorf("refs has %d entries, want 0", len(refs)) + } +} + +// generateTestSSHKey returns a PEM-encoded unencrypted ECDSA private key and a +// known_hosts line for a fake host, suitable for unit-testing newSystemGitEnv. +func generateTestSSHKey(t *testing.T) ([]byte, string) { + t.Helper() + privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate test key: %v", err) + } + block, err := gossh.MarshalPrivateKey(privKey, "") + if err != nil { + t.Fatalf("marshal test key: %v", err) + } + keyPEM := pem.EncodeToMemory(block) + + // Build a valid known_hosts line using a separate host key. + hostKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate host key: %v", err) + } + pubKey, err := gossh.NewPublicKey(hostKey.Public()) + if err != nil { + t.Fatalf("derive public key: %v", err) + } + knownHostsLine := "ssh.dev.azure.com " + strings.TrimRight(string(gossh.MarshalAuthorizedKey(pubKey)), "\n") + return keyPEM, knownHostsLine +} + +// assertSSHFilesNotWorldReadable checks that every path in sshCmd that lives under +// the reverser-git temp dir is not group- or world-readable. +// singleQuotedPath matches every single-quoted token in a shell command string, +// e.g. '-i '/tmp/reverser-git-x/id_key'' and 'UserKnownHostsFile='/tmp/.../''. +var singleQuotedPath = regexp.MustCompile(`'([^']+)'`) + +func assertSSHFilesNotWorldReadable(t *testing.T, sshCmd string) { + t.Helper() + for _, m := range singleQuotedPath.FindAllStringSubmatch(sshCmd, -1) { + path := m[1] + if !strings.Contains(path, "reverser-git") { + continue + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %q: %v", path, err) + } + if info.Mode().Perm()&0077 != 0 { + t.Errorf("file %q is group/world readable: %o", path, info.Mode().Perm()) + } + } +} + +func TestNewSystemGitEnv_SSHAuth_WithKnownHosts(t *testing.T) { + keyPEM, knownHostsLine := generateTestSSHKey(t) + + auth, err := sshpkg.NewSSHKeyAuth(string(keyPEM), "", knownHostsLine, false) + if err != nil { + t.Fatalf("NewSSHKeyAuth: %v", err) + } + + env, err := newSystemGitEnv("ssh://git@ssh.dev.azure.com:v3/org/proj/repo", auth) + if err != nil { + t.Fatalf("newSystemGitEnv: %v", err) + } + defer env.cleanup() + + var sshCmd string + for _, v := range env.vars { + if strings.HasPrefix(v, "GIT_SSH_COMMAND=") { + sshCmd = strings.TrimPrefix(v, "GIT_SSH_COMMAND=") + } + } + if sshCmd == "" { + t.Fatal("GIT_SSH_COMMAND not set") + } + if !strings.Contains(sshCmd, "-i ") { + t.Error("GIT_SSH_COMMAND must include -i ") + } + if !strings.Contains(sshCmd, "StrictHostKeyChecking=yes") { + t.Error("GIT_SSH_COMMAND must use StrictHostKeyChecking=yes when known_hosts is provided") + } + if !strings.Contains(sshCmd, "UserKnownHostsFile=") { + t.Error("GIT_SSH_COMMAND must reference a UserKnownHostsFile") + } + assertSSHFilesNotWorldReadable(t, sshCmd) +} + +func TestNewSystemGitEnv_SSHAuth_WithoutKnownHosts(t *testing.T) { + keyPEM, _ := generateTestSSHKey(t) + auth, err := sshpkg.NewSSHKeyAuth(string(keyPEM), "", "", true) // allowMissing=true + if err != nil { + t.Fatalf("NewSSHKeyAuth: %v", err) + } + + env, err := newSystemGitEnv("ssh://git@ssh.dev.azure.com:v3/org/proj/repo", auth) + if err != nil { + t.Fatalf("newSystemGitEnv: %v", err) + } + defer env.cleanup() + + var sshCmd string + for _, v := range env.vars { + if strings.HasPrefix(v, "GIT_SSH_COMMAND=") { + sshCmd = strings.TrimPrefix(v, "GIT_SSH_COMMAND=") + } + } + if !strings.Contains(sshCmd, "StrictHostKeyChecking=accept-new") { + t.Errorf("expected accept-new when no known_hosts, got: %s", sshCmd) + } +} diff --git a/internal/git/credentials_test.go b/internal/git/credentials_test.go index 912d996a..55c2eaa7 100644 --- a/internal/git/credentials_test.go +++ b/internal/git/credentials_test.go @@ -11,7 +11,6 @@ import ( "testing" gogithttp "github.com/go-git/go-git/v5/plumbing/transport/http" - gogitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" gossh "golang.org/x/crypto/ssh" @@ -24,6 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" configv1alpha3 "github.com/ConfigButler/gitops-reverser/api/v1alpha3" + sshpkg "github.com/ConfigButler/gitops-reverser/internal/ssh" ) func credTestSSHKey(t *testing.T) ([]byte, string) { @@ -61,7 +61,7 @@ func TestAuthFromSecretData_SSHKeyDialects(t *testing.T) { auth, err := AuthFromSecretData( context.Background(), c, &configv1alpha3.GitProvider{}, secret, SSHHostKeyConfig{}) require.NoError(t, err) - assert.IsType(t, &gogitssh.PublicKeys{}, auth) + assert.IsType(t, &sshpkg.KeyAuth{}, auth) }) } } @@ -78,7 +78,7 @@ func TestAuthFromSecretData_PasswordIsSSHPassphraseWhenKeyPresent(t *testing.T) }} auth, err := AuthFromSecretData(context.Background(), c, &configv1alpha3.GitProvider{}, secret, SSHHostKeyConfig{}) require.NoError(t, err) - assert.IsType(t, &gogitssh.PublicKeys{}, auth) + assert.IsType(t, &sshpkg.KeyAuth{}, auth) } func TestAuthFromSecretData_HTTPBasicAndBearer(t *testing.T) { @@ -155,7 +155,7 @@ func TestResolveKnownHosts_Priority(t *testing.T) { secret := &corev1.Secret{Data: map[string][]byte{"ssh-privatekey": privateKey}} auth, err := AuthFromSecretData(context.Background(), c, provider, secret, SSHHostKeyConfig{}) require.NoError(t, err) - assert.IsType(t, &gogitssh.PublicKeys{}, auth) + assert.IsType(t, &sshpkg.KeyAuth{}, auth) }) t.Run("knownHostsRef ConfigMap (Argo ssh_known_hosts key)", func(t *testing.T) { @@ -169,7 +169,7 @@ func TestResolveKnownHosts_Priority(t *testing.T) { secret := &corev1.Secret{Data: map[string][]byte{"ssh-privatekey": privateKey}} auth, err := AuthFromSecretData(context.Background(), c, provider, secret, SSHHostKeyConfig{}) require.NoError(t, err) - assert.IsType(t, &gogitssh.PublicKeys{}, auth) + assert.IsType(t, &sshpkg.KeyAuth{}, auth) }) t.Run("knownHostsRef Secret", func(t *testing.T) { @@ -187,7 +187,7 @@ func TestResolveKnownHosts_Priority(t *testing.T) { secret := &corev1.Secret{Data: map[string][]byte{"ssh-privatekey": privateKey}} auth, err := AuthFromSecretData(context.Background(), c, provider, secret, SSHHostKeyConfig{}) require.NoError(t, err) - assert.IsType(t, &gogitssh.PublicKeys{}, auth) + assert.IsType(t, &sshpkg.KeyAuth{}, auth) }) t.Run("knownHostsRef missing object is an error", func(t *testing.T) { @@ -210,7 +210,7 @@ func TestResolveKnownHosts_Priority(t *testing.T) { hostKeys := SSHHostKeyConfig{ControllerNamespace: "ns", DefaultKnownHostsConfigMap: "cluster-hosts"} auth, err := AuthFromSecretData(context.Background(), c, &configv1alpha3.GitProvider{}, secret, hostKeys) require.NoError(t, err) - assert.IsType(t, &gogitssh.PublicKeys{}, auth) + assert.IsType(t, &sshpkg.KeyAuth{}, auth) }) t.Run("absent install-level default falls through to fail-closed", func(t *testing.T) { @@ -237,7 +237,7 @@ func TestResolveKnownHosts_Priority(t *testing.T) { context.Background(), c, &configv1alpha3.GitProvider{}, secret, SSHHostKeyConfig{AllowMissingKnownHosts: true}) require.NoError(t, err) - assert.IsType(t, &gogitssh.PublicKeys{}, auth) + assert.IsType(t, &sshpkg.KeyAuth{}, auth) }) } diff --git a/internal/git/git.go b/internal/git/git.go index efceb02b..da676ab1 100644 --- a/internal/git/git.go +++ b/internal/git/git.go @@ -66,6 +66,10 @@ func CheckRepo(ctx context.Context, repoURL string, auth transport.AuthMethod) ( logger := log.FromContext(ctx) logger.V(1).Info("Checking repository connectivity and metadata", "url", repoURL) + if IsADOURL(repoURL) { + return systemGitCheckRepo(ctx, repoURL, auth) + } + // Use remote.List() for lightweight connectivity check remote := git.NewRemote(nil, &config.RemoteConfig{ Name: "origin", diff --git a/internal/git/git_atomic_push.go b/internal/git/git_atomic_push.go index 8572c671..1b18ca81 100644 --- a/internal/git/git_atomic_push.go +++ b/internal/git/git_atomic_push.go @@ -205,6 +205,12 @@ func PushAtomic( return errors.New("rootBranch is not a branch") } + if remote, err := repo.Remote("origin"); err == nil { + if urls := remote.Config().URLs; len(urls) > 0 && IsADOURL(urls[0]) { + return systemGitPushAtomic(ctx, repo, rootHash, rootBranch, auth) + } + } + logger := log.FromContext(ctx) session, err := getPushSession(ctx, repo, auth) diff --git a/internal/git/git_smart_fetch.go b/internal/git/git_smart_fetch.go index 02a90a79..f11e2f92 100644 --- a/internal/git/git_smart_fetch.go +++ b/internal/git/git_smart_fetch.go @@ -6,6 +6,7 @@ import ( "context" "errors" "fmt" + "io" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" @@ -33,6 +34,10 @@ func SmartFetch( return "", fmt.Errorf("failed to get remote %s: %w", remoteName, err) } + if urls := remote.Config().URLs; len(urls) > 0 && IsADOURL(urls[0]) { + return systemGitSmartFetch(ctx, repo, target, auth) + } + // 1. Audit: List refs refs, err := listRemoteRefs(remote, auth) if err != nil { @@ -65,9 +70,9 @@ func SmartFetch( RemoteName: remoteName, Auth: auth, RefSpecs: refSpecs, - Depth: 1, Force: true, Prune: true, + Progress: io.Discard, }) if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { return "", fmt.Errorf("smart fetch failed: %w", err) diff --git a/internal/ssh/auth.go b/internal/ssh/auth.go index 2a140d02..f3b20525 100644 --- a/internal/ssh/auth.go +++ b/internal/ssh/auth.go @@ -5,64 +5,114 @@ package ssh import ( "context" + "crypto" + "encoding/pem" "errors" "fmt" "os" "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" + gogitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/go-logr/logr" gossh "golang.org/x/crypto/ssh" "sigs.k8s.io/controller-runtime/pkg/log" ) -// InsecureAllowMissingKnownHostsFlag is the controller flag, surfaced in error text, that opts -// out of SSH host key verification when no host-key source produced any known_hosts at all. -const InsecureAllowMissingKnownHostsFlag = "--insecure-allow-missing-known-hosts" +// KeyAuth wraps go-git's SSH public key auth and carries the raw material +// needed by the system-git fallback (unencrypted private key PEM and known_hosts text). +// It satisfies transport.AuthMethod via the embedded *gogitssh.PublicKeys so the +// go-git path uses it transparently; the system-git path type-asserts to *KeyAuth. +type KeyAuth struct { + *gogitssh.PublicKeys -// GetAuthMethod returns an SSH public key authentication method from a private key. -// -// Host key verification fails closed: a known_hosts source is required. A known_hosts value that -// is present but cannot be parsed is always a hard error — if a host key is declared it must be -// valid. When no known_hosts is available at all, GetAuthMethod returns an error unless -// allowMissingKnownHosts is set (the controller's --insecure-allow-missing-known-hosts flag), -// which disables host key verification and is intended for throwaway/dev clusters only. -func GetAuthMethod(privateKey, password, knownHosts string, allowMissingKnownHosts bool) (transport.AuthMethod, error) { + PrivateKeyPEM []byte // always unencrypted; written to a 0600 temp file for system git + KnownHosts string // raw known_hosts content; empty means no pinning (accept-new) +} + +// NewSSHKeyAuth constructs a KeyAuth from a PEM private key, optional passphrase, and +// optional known_hosts content. It validates and sets up host key verification on the embedded +// PublicKeys (same policy as GetAuthMethod), and also decrypts and re-serialises the private +// key without a passphrase so the system-git subprocess can use it without an ssh-agent. +func NewSSHKeyAuth(privateKey, password, knownHosts string, allowMissingKnownHosts bool) (*KeyAuth, error) { logger := log.FromContext(context.Background()) if privateKey == "" { return nil, errors.New("private key cannot be empty") } - // Create the public key authentication - publicKeys, err := ssh.NewPublicKeys("git", []byte(privateKey), password) + publicKeys, err := gogitssh.NewPublicKeys("git", []byte(privateKey), password) if err != nil { return nil, fmt.Errorf("failed to create SSH public keys: %w", err) } - if knownHosts != "" { - // A declared host key must parse: this is a hard error regardless of the - // allow-missing opt-out, which only ever covers the no-key-at-all case. + switch { + case knownHosts != "": callback, err := setupKnownHostsCallback(logger, knownHosts) if err != nil { return nil, fmt.Errorf("failed to parse known_hosts for SSH host key verification: %w", err) } publicKeys.HostKeyCallback = callback - return publicKeys, nil - } - - if !allowMissingKnownHosts { + case !allowMissingKnownHosts: return nil, errors.New( "known_hosts is required for SSH host key verification: add a 'known_hosts' entry to the " + "Git credentials Secret, point spec.knownHostsRef at a ConfigMap/Secret, or configure an " + "install-level default known-hosts ConfigMap; set '" + InsecureAllowMissingKnownHostsFlag + "' on the controller for throwaway/dev clusters only", ) + default: + logInsecureHostKey(logger, "no known_hosts provided") + //nolint:gosec // explicit development opt-out via --insecure-allow-missing-known-hosts + publicKeys.HostKeyCallback = gossh.InsecureIgnoreHostKey() + } + + // Attempt to re-serialise the key without a passphrase for the system-git + // ADO fallback. This fails for unsupported types (e.g. DSA) but that must + // not break non-ADO SSH providers — defer the error to newSystemGitEnv, + // which is the only caller that actually needs PrivateKeyPEM. + unencryptedPEM, _ := decryptPrivateKeyPEM([]byte(privateKey), []byte(password)) + + return &KeyAuth{ + PublicKeys: publicKeys, + PrivateKeyPEM: unencryptedPEM, // nil when key type is unsupported by MarshalPrivateKey + KnownHosts: knownHosts, + }, nil +} + +// decryptPrivateKeyPEM parses the PEM private key (with optional passphrase) and +// re-serialises it without a passphrase. This lets the system-git subprocess load +// the key via -i without needing an ssh-agent or interactive prompt. +func decryptPrivateKeyPEM(pemBytes, passphrase []byte) ([]byte, error) { + var rawKey interface{} + var err error + if len(passphrase) > 0 { + rawKey, err = gossh.ParseRawPrivateKeyWithPassphrase(pemBytes, passphrase) + } else { + rawKey, err = gossh.ParseRawPrivateKey(pemBytes) } - logInsecureHostKey(logger, "no known_hosts provided") - //nolint:gosec // explicit development opt-out via --insecure-allow-missing-known-hosts - publicKeys.HostKeyCallback = gossh.InsecureIgnoreHostKey() - return publicKeys, nil + if err != nil { + return nil, fmt.Errorf("parse SSH private key: %w", err) + } + privKey, ok := rawKey.(crypto.PrivateKey) + if !ok { + return nil, errors.New("parsed SSH key does not implement crypto.PrivateKey") + } + block, err := gossh.MarshalPrivateKey(privKey, "") + if err != nil { + return nil, fmt.Errorf("marshal unencrypted SSH private key: %w", err) + } + return pem.EncodeToMemory(block), nil +} + +// InsecureAllowMissingKnownHostsFlag is the controller flag, surfaced in error text, that opts +// out of SSH host key verification when no host-key source produced any known_hosts at all. +const InsecureAllowMissingKnownHostsFlag = "--insecure-allow-missing-known-hosts" + +// GetAuthMethod returns an SSH public key authentication method from a private key. +// It delegates to NewSSHKeyAuth and returns the result as transport.AuthMethod. +// The concrete type is *KeyAuth, which the system-git ADO fallback can type-assert +// to access the raw private key bytes and known_hosts needed to invoke system git. +func GetAuthMethod(privateKey, password, knownHosts string, allowMissingKnownHosts bool) (transport.AuthMethod, error) { + return NewSSHKeyAuth(privateKey, password, knownHosts, allowMissingKnownHosts) } // logInsecureHostKey emits a loud warning whenever SSH host key verification is disabled. @@ -90,7 +140,7 @@ func setupKnownHostsCallback(logger logr.Logger, knownHosts string) (gossh.HostK return nil, err } - callback, err := ssh.NewKnownHostsCallback(tmpFile.Name()) + callback, err := gogitssh.NewKnownHostsCallback(tmpFile.Name()) if err != nil { logger.Info("Warning: failed to parse known_hosts", "error", err) return nil, err