From e44c299de97ca87977938f871521b69fda3198a5 Mon Sep 17 00:00:00 2001 From: "James O. D. Hunt" Date: Mon, 27 Nov 2017 17:42:16 +0000 Subject: [PATCH] proxy: Test merge of "one proxy / VM" changes Update the runtime based on https://github.com/containers/virtcontainers/pull/483, and accompanying changes for the virtcontainers API changes. Signed-off-by: James O. D. Hunt --- Makefile | 5 +- cc-env.go | 12 ++-- config.go | 18 ++++-- config/configuration.toml.in | 6 +- .../containers/virtcontainers/api.go | 6 ++ .../containers/virtcontainers/cc_proxy.go | 61 +++++++++++++------ .../containers/virtcontainers/container.go | 2 +- .../containers/virtcontainers/noop_proxy.go | 6 ++ .../containers/virtcontainers/pod.go | 37 +++++++++-- .../containers/virtcontainers/proxy.go | 6 +- 10 files changed, 118 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index 7011f8b7..eecfd9a0 100644 --- a/Makefile +++ b/Makefile @@ -96,7 +96,6 @@ SHIMCMD := cc-shim SHIMPATH := $(PKGLIBEXECDIR)/$(SHIMCMD) PROXYCMD := cc-proxy -PROXYURL := unix://$(PKGRUNDIR)/proxy.sock PROXYPATH := $(PKGLIBEXECDIR)/$(PROXYCMD) # Default number of vCPUs @@ -165,7 +164,6 @@ USER_VARS += PKGLIBEXECDIR USER_VARS += PKGRUNDIR USER_VARS += PREFIX USER_VARS += PROXYPATH -USER_VARS += PROXYURL USER_VARS += QEMUBINDIR USER_VARS += QEMUCMD USER_VARS += QEMUPATH @@ -219,7 +217,6 @@ var defaultShimPath = "$(SHIMPATH)" const defaultKernelParams = "$(KERNELPARAMS)" const defaultMachineType = "$(MACHINETYPE)" -const defaultProxyURL = "$(PROXYURL)" const defaultRootDirectory = "$(PKGRUNDIR)" const defaultRuntimeLib = "$(PKGLIBDIR)" const defaultRuntimeRun = "$(PKGRUNDIR)" @@ -288,7 +285,7 @@ $(GENERATED_FILES): %: %.in Makefile VERSION -e "s|@KERNELPARAMS@|$(KERNELPARAMS)|g" \ -e "s|@LOCALSTATEDIR@|$(LOCALSTATEDIR)|g" \ -e "s|@PKGLIBEXECDIR@|$(PKGLIBEXECDIR)|g" \ - -e "s|@PROXYURL@|$(PROXYURL)|g" \ + -e "s|@PROXYPATH@|$(PROXYPATH)|g" \ -e "s|@QEMUPATH@|$(QEMUPATH)|g" \ -e "s|@MACHINETYPE@|$(MACHINETYPE)|g" \ -e "s|@SHIMPATH@|$(SHIMPATH)|g" \ diff --git a/cc-env.go b/cc-env.go index 79161da7..4a427273 100644 --- a/cc-env.go +++ b/cc-env.go @@ -64,6 +64,7 @@ type RuntimeConfigInfo struct { type RuntimeInfo struct { Version RuntimeVersionInfo Config RuntimeConfigInfo + Debug bool } // RuntimeVersionInfo stores details of the runtime version @@ -78,13 +79,15 @@ type HypervisorInfo struct { MachineType string Version string Path string + Debug bool } // ProxyInfo stores proxy details type ProxyInfo struct { Type string Version string - URL string + Path string + Debug bool } // ShimInfo stores shim details @@ -92,6 +95,7 @@ type ShimInfo struct { Type string Version string Path string + Debug bool } // AgentInfo stores agent details @@ -202,8 +206,6 @@ func getProxyInfo(config oci.RuntimeConfig) (ProxyInfo, error) { return ProxyInfo{}, errors.New("cannot determine proxy config") } - proxyURL := proxyConfig.URL - version, err := getCommandVersion(defaultProxyPath) if err != nil { version = unknown @@ -212,7 +214,8 @@ func getProxyInfo(config oci.RuntimeConfig) (ProxyInfo, error) { ccProxy := ProxyInfo{ Type: string(config.ProxyType), Version: version, - URL: proxyURL, + Path: proxyConfig.Path, + Debug: proxyConfig.Debug, } return ccProxy, nil @@ -239,6 +242,7 @@ func getShimInfo(config oci.RuntimeConfig) (ShimInfo, error) { Type: string(config.ShimType), Version: version, Path: shimPath, + Debug: shimConfig.Debug, } return ccShim, nil diff --git a/config.go b/config.go index ee7ddc42..e86dd4b1 100644 --- a/config.go +++ b/config.go @@ -89,7 +89,8 @@ type hypervisor struct { } type proxy struct { - URL string `toml:"url"` + Path string `toml:"path"` + Debug bool `toml:"enable_debug"` } type runtime struct { @@ -172,12 +173,16 @@ func (h hypervisor) defaultMemSz() uint32 { return h.DefaultMemSz } -func (p proxy) url() string { - if p.URL == "" { - return defaultProxyURL +func (p proxy) path() string { + if p.Path == "" { + return defaultProxyPath } - return p.URL + return p.Path +} + +func (p proxy) debug() bool { + return p.Debug } func (s shim) path() (string, error) { @@ -265,7 +270,8 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run switch k { case ccProxyTableType: pConfig := vc.CCProxyConfig{ - URL: proxy.url(), + Path: proxy.path(), + Debug: proxy.debug(), } config.ProxyType = vc.CCProxyType diff --git a/config/configuration.toml.in b/config/configuration.toml.in index 4f76482f..ea6620ec 100644 --- a/config/configuration.toml.in +++ b/config/configuration.toml.in @@ -61,7 +61,11 @@ disable_block_device_use = @DEFDISABLEBLOCK@ #disable_nesting_checks = true [proxy.cc] -url = "@PROXYURL@" +path = "@PROXYPATH@" + +# If enabled, proxy messages will be sent to the system log +# (default: disabled) +#enable_debug = true [shim.cc] path = "@SHIMPATH@" diff --git a/vendor/github.com/containers/virtcontainers/api.go b/vendor/github.com/containers/virtcontainers/api.go index 80c56b43..98f1dcd2 100644 --- a/vendor/github.com/containers/virtcontainers/api.go +++ b/vendor/github.com/containers/virtcontainers/api.go @@ -86,6 +86,12 @@ func createPodFromConfig(podConfig PodConfig) (*Pod, error) { return nil, err } + // Start the proxy + err = p.startProxy() + if err != nil { + return nil, err + } + // Start shims if err := p.startShims(); err != nil { return nil, err diff --git a/vendor/github.com/containers/virtcontainers/cc_proxy.go b/vendor/github.com/containers/virtcontainers/cc_proxy.go index 23144bee..5d737573 100644 --- a/vendor/github.com/containers/virtcontainers/cc_proxy.go +++ b/vendor/github.com/containers/virtcontainers/cc_proxy.go @@ -20,13 +20,15 @@ import ( "fmt" "net" "net/url" + "os/exec" + "path/filepath" "time" "github.com/clearcontainers/proxy/client" "github.com/sirupsen/logrus" ) -var defaultCCProxyURL = "unix:///run/cc-oci-runtime/proxy.sock" +var defaultCCProxyURL = "unix:///var/run/clear-containers/proxy.sock" const ( // Number of seconds to wait for the proxy to respond to a connection @@ -41,7 +43,8 @@ type ccProxy struct { // CCProxyConfig is a structure storing information needed for // the Clear Containers proxy initialization. type CCProxyConfig struct { - URL string + Path string + Debug bool } // connectProxyRetry repeatedly tries to connect to the proxy on the specified @@ -96,12 +99,12 @@ func (p *ccProxy) connectProxyRetry(scheme, address string) (conn net.Conn, err } } -func (p *ccProxy) connectProxy(proxyURL string) (*client.Client, error) { - if proxyURL == "" { - proxyURL = defaultCCProxyURL +func (p *ccProxy) connectProxy(uri string) (*client.Client, error) { + if uri == "" { + return nil, fmt.Errorf("no proxy URI") } - u, err := url.Parse(proxyURL) + u, err := url.Parse(uri) if err != nil { return nil, err } @@ -127,17 +130,44 @@ func (p *ccProxy) connectProxy(proxyURL string) (*client.Client, error) { return client.NewClient(conn), nil } +// start is the proxy start implementation for ccProxy. +func (p *ccProxy) start(pod Pod) (int, string, error) { + if pod.config == nil { + return -1, "", fmt.Errorf("Pod config cannot be nil") + } + + config, ok := newProxyConfig(*(pod.config)).(CCProxyConfig) + if !ok { + return -1, "", fmt.Errorf("Wrong proxy config type, should be CCProxyConfig type") + } + + if config.Path == "" { + return -1, "", fmt.Errorf("Proxy path cannot be empty") + } + + // construct the socket path the proxy instance will use + socketPath := filepath.Join(runStoragePath, pod.id, "proxy.sock") + uri := fmt.Sprintf("unix://%s", socketPath) + + args := []string{config.Path, "-uri", uri} + if config.Debug { + args = append(args, "-log", "debug") + } + + cmd := exec.Command(args[0], args[1:]...) + if err := cmd.Start(); err != nil { + return -1, "", err + } + + return cmd.Process.Pid, uri, nil +} + // register is the proxy register implementation for ccProxy. func (p *ccProxy) register(pod Pod) ([]ProxyInfo, string, error) { var err error var proxyInfos []ProxyInfo - ccConfig, ok := newProxyConfig(*(pod.config)).(CCProxyConfig) - if !ok { - return []ProxyInfo{}, "", fmt.Errorf("Wrong proxy config type, should be CCProxyConfig type") - } - - p.client, err = p.connectProxy(ccConfig.URL) + p.client, err = p.connectProxy(pod.state.URL) if err != nil { return []ProxyInfo{}, "", err } @@ -193,12 +223,7 @@ func (p *ccProxy) unregister(pod Pod) error { func (p *ccProxy) connect(pod Pod, createToken bool) (ProxyInfo, string, error) { var err error - ccConfig, ok := newProxyConfig(*(pod.config)).(CCProxyConfig) - if !ok { - return ProxyInfo{}, "", fmt.Errorf("Wrong proxy config type, should be CCProxyConfig type") - } - - p.client, err = p.connectProxy(ccConfig.URL) + p.client, err = p.connectProxy(pod.state.URL) if err != nil { return ProxyInfo{}, "", err } diff --git a/vendor/github.com/containers/virtcontainers/container.go b/vendor/github.com/containers/virtcontainers/container.go index 0d089a39..92fcc645 100644 --- a/vendor/github.com/containers/virtcontainers/container.go +++ b/vendor/github.com/containers/virtcontainers/container.go @@ -714,7 +714,7 @@ func (c *Container) processList(options ProcessListOptions) (ProcessList, error) func (c *Container) createShimProcess(token, url string, cmd Cmd) (*Process, error) { if c.pod.state.URL != url { - return &Process{}, fmt.Errorf("Pod URL %s and URL from proxy %s MUST be identical", c.pod.state.URL, url) + return &Process{}, fmt.Errorf("Pod URL %q and URL from proxy %q MUST be identical", c.pod.state.URL, url) } shimParams := ShimParams{ diff --git a/vendor/github.com/containers/virtcontainers/noop_proxy.go b/vendor/github.com/containers/virtcontainers/noop_proxy.go index d9c790ff..73a8a967 100644 --- a/vendor/github.com/containers/virtcontainers/noop_proxy.go +++ b/vendor/github.com/containers/virtcontainers/noop_proxy.go @@ -20,6 +20,12 @@ type noopProxy struct{} var noopProxyURL = "noopProxyURL" +// register is the proxy start implementation for testing purpose. +// It does nothing. +func (p *noopProxy) start(pod Pod) (int, string, error) { + return 0, noopProxyURL, nil +} + // register is the proxy register implementation for testing purpose. // It does nothing. func (p *noopProxy) register(pod Pod) ([]ProxyInfo, string, error) { diff --git a/vendor/github.com/containers/virtcontainers/pod.go b/vendor/github.com/containers/virtcontainers/pod.go index 1a4832d1..616dd14f 100644 --- a/vendor/github.com/containers/virtcontainers/pod.go +++ b/vendor/github.com/containers/virtcontainers/pod.go @@ -72,6 +72,9 @@ type State struct { // Bool to indicate if the drive for a container was hotplugged. HotpluggedDrive bool `json:"hotpluggedDrive"` + + // Process ID of the pods proxy instance + ProxyPid int } // valid checks that the pod state is valid. @@ -757,6 +760,29 @@ func (p *Pod) startVM(netNsPath string) error { }) } +// startProxy starts a proxy instance for the pod. +// +// Note that there is no corresponding stopProxy() since the proxy +// stops itself. +func (p *Pod) startProxy() error { + pid, uri, err := p.proxy.start(*p) + if err != nil { + return err + } + + // save state + p.state.URL = uri + p.state.ProxyPid = pid + + if err := p.setPodState(p.state); err != nil { + return err + } + + p.Logger().WithField("proxy-pid", pid).Info("proxy started") + + return nil +} + // startShims registers all containers to the proxy and starts one // shim per container. func (p *Pod) startShims() error { @@ -773,11 +799,6 @@ func (p *Pod) startShims() error { return fmt.Errorf("Retrieved %d proxy infos, expecting %d", len(proxyInfos), len(p.containers)) } - p.state.URL = url - if err := p.setPodState(p.state); err != nil { - return err - } - shimCount := 0 for idx := range p.containers { shimParams := ShimParams{ @@ -802,7 +823,11 @@ func (p *Pod) startShims() error { } } - p.Logger().WithField("shim-count", shimCount).Info("Started shims") + if shimCount > 0 { + p.Logger().WithField("shim-count", shimCount).Info("Started shims") + } else { + p.Logger().Info("No containers, so no shims started") + } return nil } diff --git a/vendor/github.com/containers/virtcontainers/proxy.go b/vendor/github.com/containers/virtcontainers/proxy.go index ff57c8cd..81616039 100644 --- a/vendor/github.com/containers/virtcontainers/proxy.go +++ b/vendor/github.com/containers/virtcontainers/proxy.go @@ -93,7 +93,7 @@ func newProxyConfig(config PodConfig) interface{} { } } -// ProxyInfo holds the token and url returned by the proxy. +// ProxyInfo holds the token returned by the proxy. // Each ProxyInfo relates to a process running inside a container. type ProxyInfo struct { Token string @@ -101,6 +101,10 @@ type ProxyInfo struct { // proxy is the virtcontainers proxy interface. type proxy interface { + // start launches a proxy instance for the specified pod, returning + // the PID of the process and the URL used to connect to it. + start(pod Pod) (int, string, error) + // register connects and registers the proxy to the given VM. // It also returns information related to containers workloads. register(pod Pod) ([]ProxyInfo, string, error)