unable to build static binary

[harche]

While pulling from dockerhub works fine, not able to pull the image from the locally hosted registry.

[harshal@oc1472752102 skopeo]$ ./skopeo copy  docker://nginx:latest oci:nginx_ocilayout:latest
Getting image source signatures
Copying blob f5d23c7fed46 done
Copying blob 918b255d86e5 done
Copying blob 8c0120a6f561 done
Copying config 54aa0bd0dc done
Writing manifest to image destination
Storing signatures
[harshal@oc1472752102 skopeo]$ 

[harshal@oc1472752102 skopeo]$ ./skopeo copy --src-tls-verify=false docker://localhost:5000/nginx:latest oci:nginx_ocilayout:latest
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x63 pc=0x7f710c7e156d]

runtime stack:
runtime.throw(0xc55535, 0x2a)
	/usr/lib/go-1.10/src/runtime/panic.go:616 +0x81
runtime.sigpanic()
	/usr/lib/go-1.10/src/runtime/signal_unix.go:372 +0x28e

goroutine 44 [syscall]:
runtime.cgocall(0x9f99c0, 0xc4204a95f8, 0x29)
	/usr/lib/go-1.10/src/runtime/cgocall.go:128 +0x64 fp=0xc4204a95b8 sp=0xc4204a9580 pc=0x4033d4
net._C2func_getaddrinfo(0xc4200fe5d0, 0x0, 0xc4201ea5d0, 0xc42000e070, 0x0, 0x0, 0x0)
	_cgo_gotypes.go:92 +0x55 fp=0xc4204a95f8 sp=0xc4204a95b8 pc=0x5ccaa5
net.cgoLookupIPCNAME.func1(0xc4200fe5d0, 0x0, 0xc4201ea5d0, 0xc42000e070, 0xa, 0xa, 0xc4204e20c0)
	/usr/lib/go-1.10/src/net/cgo_unix.go:149 +0x13b fp=0xc4204a9640 sp=0xc4204a95f8 pc=0x5d379b
net.cgoLookupIPCNAME(0xc4200fe5b2, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/lib/go-1.10/src/net/cgo_unix.go:149 +0x174 fp=0xc4204a9738 sp=0xc4204a9640 pc=0x5ce114
net.cgoIPLookup(0xc420092840, 0xc4200fe5b2, 0x9)
	/usr/lib/go-1.10/src/net/cgo_unix.go:201 +0x4d fp=0xc4204a97c8 sp=0xc4204a9738 pc=0x5ce7dd
runtime.goexit()
	/usr/lib/go-1.10/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc4204a97d0 sp=0xc4204a97c8 pc=0x457d41
created by net.cgoLookupIP
	/usr/lib/go-1.10/src/net/cgo_unix.go:211 +0xaf

goroutine 1 [select]:
net/http.(*Transport).getConn(0xc4203480f0, 0xc4201ea4b0, 0x0, 0xc4200a58f0, 0x4, 0xc4200fe5b2, 0xe, 0x0, 0x0, 0x28)
	/usr/lib/go-1.10/src/net/http/transport.go:962 +0x558
net/http.(*Transport).RoundTrip(0xc4203480f0, 0xc420392300, 0xc4203480f0, 0x0, 0x0)
	/usr/lib/go-1.10/src/net/http/transport.go:409 +0x632
net/http.send(0xc420392300, 0xcc6f20, 0xc4203480f0, 0x0, 0x0, 0x0, 0xc42000e060, 0x411ef9, 0xc4204d2cf0, 0x1)
	/usr/lib/go-1.10/src/net/http/client.go:252 +0x185
net/http.(*Client).send(0xc4204f5d40, 0xc420392300, 0x0, 0x0, 0x0, 0xc42000e060, 0x0, 0x1, 0x7f7110468d90)
	/usr/lib/go-1.10/src/net/http/client.go:176 +0xfa
net/http.(*Client).Do(0xc4204f5d40, 0xc420392300, 0xc4204d2f20, 0x2, 0x2)
	/usr/lib/go-1.10/src/net/http/client.go:615 +0x28d
github.com/containers/skopeo/vendor/github.com/containers/image/docker.(*dockerClient).makeRequestToResolvedURL(0xc420348000, 0xccdba0, 0xc4200a8010, 0xc39bef, 0x3, 0xc4200a58f0, 0x2f, 0xc4204d31d0, 0x0, 0x0, ...)
	/src/github.com/containers/skopeo/vendor/github.com/containers/image/docker/docker_client.go:442 +0x34e
github.com/containers/skopeo/vendor/github.com/containers/image/docker.(*dockerClient).makeRequest(0xc420348000, 0xccdba0, 0xc4200a8010, 0xc39bef, 0x3, 0xc4204e6d80, 0x1a, 0xc4204d31d0, 0x0, 0x0, ...)
	/src/github.com/containers/skopeo/vendor/github.com/containers/image/docker/docker_client.go:411 +0x25a
github.com/containers/skopeo/vendor/github.com/containers/image/docker.(*dockerImageSource).fetchManifest(0xc4204e38c0, 0xccdba0, 0xc4200a8010, 0xc4204e6ab5, 0x6, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/src/github.com/containers/skopeo/vendor/github.com/containers/image/docker/docker_image_src.go:144 +0x2e3
github.com/containers/skopeo/vendor/github.com/containers/image/docker.(*dockerImageSource).ensureManifestIsLoaded(0xc4204e38c0, 0xccdba0, 0xc4200a8010, 0xc4200b9000, 0xc3a14e)
	/src/github.com/containers/skopeo/vendor/github.com/containers/image/docker/docker_image_src.go:176 +0xa0
github.com/containers/skopeo/vendor/github.com/containers/image/docker.newImageSource(0xccdba0, 0xc4200a8010, 0xc4202f4c60, 0xccaea0, 0xc4200b8760, 0xc4204f6280, 0x7ffcc5d8606e, 0xf)
	/src/github.com/containers/skopeo/vendor/github.com/containers/image/docker/docker_image_src.go:87 +0x39d
github.com/containers/skopeo/vendor/github.com/containers/image/docker.dockerReference.NewImageSource(0xccaea0, 0xc4200b8760, 0xccdba0, 0xc4200a8010, 0xc4202f4c60, 0xc4204f62b0, 0xc4204d3600, 0x18, 0xc4204d35f8)
	/src/github.com/containers/skopeo/vendor/github.com/containers/image/docker/docker_transport.go:144 +0x57
github.com/containers/skopeo/vendor/github.com/containers/image/copy.Image(0xccdba0, 0xc4200a8010, 0xc4200b8700, 0xcd3760, 0xc4204f4ed0, 0xcd35e0, 0xc4204f19c0, 0xc4204d39b8, 0x0, 0x0, ...)
	/src/github.com/containers/skopeo/vendor/github.com/containers/image/copy/copy.go:151 +0x19e
main.(*copyOptions).run(0xc420333490, 0xc4200ae120, 0x2, 0x2, 0xcc7120, 0xc4200ac008, 0x0, 0x0)
	/src/github.com/containers/skopeo/cmd/skopeo/copy.go:150 +0xa51
main.(*copyOptions).(main.run)-fm(0xc4200ae120, 0x2, 0x2, 0xcc7120, 0xc4200ac008, 0xc4204e85a0, 0x0)
	/src/github.com/containers/skopeo/cmd/skopeo/copy.go:52 +0x5c
main.commandAction.func1(0xc4202f4840, 0x0, 0xc4202f4840)
	/src/github.com/containers/skopeo/cmd/skopeo/utils.go:26 +0x6b
github.com/containers/skopeo/vendor/github.com/urfave/cli.HandleAction(0xb5ffa0, 0xc4204f11a0, 0xc4202f4840, 0xc4204e8500, 0x0)
	/src/github.com/containers/skopeo/vendor/github.com/urfave/cli/app.go:488 +0x101
github.com/containers/skopeo/vendor/github.com/urfave/cli.Command.Run(0xc3a04a, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc5807f, 0x2f, 0x0, ...)
	/src/github.com/containers/skopeo/vendor/github.com/urfave/cli/command.go:210 +0xa36
github.com/containers/skopeo/vendor/github.com/urfave/cli.(*App).Run(0xc4200bab60, 0xc4200ae0f0, 0x5, 0x5, 0x0, 0x0)
	/src/github.com/containers/skopeo/vendor/github.com/urfave/cli/app.go:255 +0x6a0
main.main()
	/src/github.com/containers/skopeo/cmd/skopeo/main.go:124 +0x5c

goroutine 20 [syscall]:
os/signal.signal_recv(0x0)
	/usr/lib/go-1.10/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
	/usr/lib/go-1.10/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
	/usr/lib/go-1.10/src/os/signal/signal_unix.go:28 +0x41

goroutine 43 [select]:
net.cgoLookupIP(0xccdb60, 0xc4200b0280, 0xc4200fe5b2, 0x9, 0x0, 0x0, 0x0, 0x0, 0x2, 0x2)
	/usr/lib/go-1.10/src/net/cgo_unix.go:212 +0x19f
net.(*Resolver).lookupIP(0x10800a0, 0xccdb60, 0xc4200b0280, 0xc4200fe5b2, 0x9, 0x42d49b, 0xc420001500, 0xc4204e20c0, 0xc420001500, 0xc4204a9658)
	/usr/lib/go-1.10/src/net/lookup_unix.go:95 +0x12d
net.(*Resolver).(net.lookupIP)-fm(0xccdb60, 0xc4200b0280, 0xc4200fe5b2, 0x9, 0x42a1d9, 0xc400000008, 0xc4204e20c0, 0xc4200b0100, 0xc4204a96a0)
	/usr/lib/go-1.10/src/net/lookup.go:192 +0x56
net.glob..func10(0xccdb60, 0xc4200b0280, 0xc42013c2b0, 0xc4200fe5b2, 0x9, 0x0, 0xc42000e038, 0xc4201ea270, 0xc420092358, 0xc4201ea0f0)
	/usr/lib/go-1.10/src/net/hook.go:19 +0x52
net.(*Resolver).LookupIPAddr.func1(0x0, 0x0, 0x0, 0x0)
	/usr/lib/go-1.10/src/net/lookup.go:206 +0xd8
internal/singleflight.(*Group).doCall(0x1080090, 0xc4200ae3c0, 0xc4200fe5b2, 0x9, 0xc4201ea570)
	/usr/lib/go-1.10/src/internal/singleflight/singleflight.go:95 +0x2e
created by internal/singleflight.(*Group).DoChan
	/usr/lib/go-1.10/src/internal/singleflight/singleflight.go:88 +0x2d0

goroutine 42 [select]:
net.(*Resolver).LookupIPAddr(0x10800a0, 0xccdbe0, 0xc420092780, 0xc4200fe5b2, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/lib/go-1.10/src/net/lookup.go:212 +0x50d
net.(*Resolver).internetAddrList(0x10800a0, 0xccdbe0, 0xc420092780, 0xc39d5a, 0x3, 0xc4200fe5b2, 0xe, 0x0, 0x0, 0x0, ...)
	/usr/lib/go-1.10/src/net/ipsock.go:293 +0x5c4
net.(*Resolver).resolveAddrList(0x10800a0, 0xccdbe0, 0xc420092780, 0xc3a06e, 0x4, 0xc39d5a, 0x3, 0xc4200fe5b2, 0xe, 0x0, ...)
	/usr/lib/go-1.10/src/net/dial.go:193 +0x50c
net.(*Dialer).DialContext(0xc4204e8d20, 0xccdba0, 0xc4200a8010, 0xc39d5a, 0x3, 0xc4200fe5b2, 0xe, 0x0, 0x0, 0x0, ...)
	/usr/lib/go-1.10/src/net/dial.go:375 +0x22b
net.(*Dialer).Dial(0xc4204e8d20, 0xc39d5a, 0x3, 0xc4200fe5b2, 0xe, 0x60, 0x110, 0x110, 0xc420470120)
	/usr/lib/go-1.10/src/net/dial.go:320 +0x75
github.com/containers/skopeo/vendor/golang.org/x/net/proxy.(Dialer).Dial-fm(0xc39d5a, 0x3, 0xc4200fe5b2, 0xe, 0xc420092720, 0xc420139ae8, 0x405246, 0x73d009)
	/src/github.com/containers/skopeo/vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go:109 +0x5b
net/http.(*Transport).dial(0xc4203480f0, 0xccdba0, 0xc4200a8010, 0xc39d5a, 0x3, 0xc4200fe5b2, 0xe, 0x445741, 0x44516f, 0xe, ...)
	/usr/lib/go-1.10/src/net/http/transport.go:901 +0x78
net/http.(*Transport).dialConn(0xc4203480f0, 0xccdba0, 0xc4200a8010, 0x0, 0xc4200a58f0, 0x4, 0xc4200fe5b2, 0xe, 0x1, 0x0, ...)
	/usr/lib/go-1.10/src/net/http/transport.go:1143 +0x317
net/http.(*Transport).getConn.func4(0xc4203480f0, 0xccdba0, 0xc4200a8010, 0xc4201ea4e0, 0xc4200944e0)
	/usr/lib/go-1.10/src/net/http/transport.go:957 +0x78
created by net/http.(*Transport).getConn
	/usr/lib/go-1.10/src/net/http/transport.go:956 +0x363
[harshal@oc1472752102 skopeo]$ 

[harche]

For what it's worth, docker works fine with same registry,

[harshal@oc1472752102 skopeo]$ docker pull localhost:5000/nginx:latest
Trying to pull repository localhost:5000/nginx ... 
latest: Pulling from localhost:5000/nginx
Digest: sha256:00be67d6ba53d5318cd91c57771530f5251cfbe028b7be2c4b70526f988cfc9f
Status: Image is up to date for localhost:5000/nginx:latest
[harshal@oc1472752102 skopeo]$ 

[vrothberg]

@harche, thanks for opening the issue. Can you share how the binary was built and with which version of go?

[harche]

I cloned upstream master of the skopeo and built using, make binary-static DISABLE_CGO=1

[harshal@oc1472752102 skopeo]$ go version 
go version go1.12.5 linux/amd64
[harshal@oc1472752102 skopeo]$ 

[vrothberg]

That's odd since the stack trace claims: /usr/lib/go-1.10/src/net/cgo_unix.go:212 +0x19f

[harche]

Oh, it's built inside a container right?

[harshal@oc1472752102 skopeo]$ make binary-static DISABLE_CGO=1
docker build  -f Dockerfile.build -t skopeobuildimage .
Sending build context to Docker daemon 101.2 MB
Step 1/4 : FROM ubuntu:18.10
 ---> 6f73120de66c
Step 2/4 : RUN apt-get update && apt-get install -y     golang     libbtrfs-dev     git-core     libdevmapper-dev     libgpgme11-dev     go-md2man     libglib2.0-dev     libostree-dev
 ---> Using cache
 ---> bb2870e349d5
Step 3/4 : ENV GOPATH /
 ---> Using cache
 ---> 39c804ef056a
Step 4/4 : WORKDIR /src/github.com/containers/skopeo
 ---> Using cache
 ---> 1b5ca13b10a6
Successfully built 1b5ca13b10a6
docker run --rm --security-opt label=disable -v $(pwd):/src/github.com/containers/skopeo \
	skopeobuildimage make binary-local-static  BUILDTAGS='containers_image_ostree_stub exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp'
go build -ldflags "-extldflags \"-static\" -X main.gitCommit=5f451126781913c7c72695ad18fe029d2f0e8de8" -gcflags "" -tags "containers_image_ostree_stub exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp" -o skopeo ./cmd/skopeo
[harshal@oc1472752102 skopeo]$ 

[vrothberg]

Yes, it's built in the container. I assume CGO must be disabled. Trying to reproduce now.

[harche]

If I disable CGO I am not even able to build,

[harshal@oc1472752102 skopeo]$ make binary-static DISABLE_CGO=0
docker build  -f Dockerfile.build -t skopeobuildimage .
Sending build context to Docker daemon 101.2 MB
Step 1/4 : FROM ubuntu:18.10
 ---> 6f73120de66c
Step 2/4 : RUN apt-get update && apt-get install -y     golang     libbtrfs-dev     git-core     libdevmapper-dev     libgpgme11-dev     go-md2man     libglib2.0-dev     libostree-dev
 ---> Using cache
 ---> bb2870e349d5
Step 3/4 : ENV GOPATH /
 ---> Using cache
 ---> 39c804ef056a
Step 4/4 : WORKDIR /src/github.com/containers/skopeo
 ---> Using cache
 ---> 1b5ca13b10a6
Successfully built 1b5ca13b10a6
docker run --rm --security-opt label=disable -v $(pwd):/src/github.com/containers/skopeo \
	skopeobuildimage make binary-local-static  BUILDTAGS='   containers_image_ostree_stub '
go build -ldflags "-extldflags \"-static\" -X main.gitCommit=5f451126781913c7c72695ad18fe029d2f0e8de8" -gcflags "" -tags "containers_image_ostree_stub " -o skopeo ./cmd/skopeo
# github.com/containers/skopeo/cmd/skopeo
/usr/lib/go-1.10/pkg/tool/linux_amd64/link: running gcc failed: exit status 1
/usr/bin/ld: cannot find -ldevmapper
/usr/bin/ld: /tmp/go-link-031325273/000023.o: in function `mygetgrouplist':
/build/golang-1.10-0Gb4kq/golang-1.10-1.10.4/src/os/user/getgrouplist_unix.go:15: warning: Using 'getgrouplist' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
/usr/bin/ld: /tmp/go-link-031325273/000022.o: in function `mygetgrgid_r':
/build/golang-1.10-0Gb4kq/golang-1.10-1.10.4/src/os/user/cgo_lookup_unix.go:38: warning: Using 'getgrgid_r' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
/usr/bin/ld: /tmp/go-link-031325273/000022.o: in function `mygetgrnam_r':
/build/golang-1.10-0Gb4kq/golang-1.10-1.10.4/src/os/user/cgo_lookup_unix.go:43: warning: Using 'getgrnam_r' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
/usr/bin/ld: /tmp/go-link-031325273/000022.o: in function `mygetpwnam_r':
/build/golang-1.10-0Gb4kq/golang-1.10-1.10.4/src/os/user/cgo_lookup_unix.go:33: warning: Using 'getpwnam_r' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
/usr/bin/ld: /tmp/go-link-031325273/000022.o: in function `mygetpwuid_r':
/build/golang-1.10-0Gb4kq/golang-1.10-1.10.4/src/os/user/cgo_lookup_unix.go:28: warning: Using 'getpwuid_r' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
/usr/bin/ld: /tmp/go-link-031325273/000029.o: in function `_cgo_f7895c2c5a3a_C2func_getaddrinfo':
/tmp/go-build/cgo-gcc-prolog:46: warning: Using 'getaddrinfo' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
collect2: error: ld returned 1 exit status

make: *** [Makefile:101: binary-local-static] Error 2
make: *** [binary-static] Error 2
[harshal@oc1472752102 skopeo]$ 

[harche]

@vrothberg

[harshal@oc1472752102 skopeo]$ make binary-local
go build "-buildmode=pie" -ldflags "-X main.gitCommit=5f451126781913c7c72695ad18fe029d2f0e8de8" -gcflags "" -tags "   containers_image_ostree_stub " -o skopeo ./cmd/skopeo
[harshal@oc1472752102 skopeo]$ 
[harshal@oc1472752102 skopeo]$ 
[harshal@oc1472752102 skopeo]$ 
[harshal@oc1472752102 skopeo]$ ./skopeo copy  docker://nginx:latest oci:nginx_ocilayout:latest
Getting image source signatures
Copying blob f5d23c7fed46 skipped: already exists
Copying blob 918b255d86e5 skipped: already exists
Copying blob 8c0120a6f561 skipped: already exists
Copying config 54aa0bd0dc done
Writing manifest to image destination
Storing signatures
(reverse-i-search)`co': ./skopeo ^Cpy  docker://nginx:latest oci:nginx_ocilayout:latest
[harshal@oc1472752102 skopeo]$ ./skopeo copy --src-tls-verify=false docker://localhost:5000/nginx:latest oci:nginx_ocilayout:latest
Getting image source signatures
Copying blob fc7181108d40 done
Copying blob d2e987ca2267 done
Copying blob 0b760b431b11 done
Copying config d31a797051 done
Writing manifest to image destination
Storing signatures
[harshal@oc1472752102 skopeo]$ 

so make binary-local works, but if I try to generate static binaries it fails.

[vrothberg]

@harche, it took a while until I refreshed my brain. This is actually a known issue and addressed in PR #670. However, there are a few shortcomings when using static binaries as @mtrmac mentions here #670 (comment).

Let's keep this issue open until there's a consensus in #670 how we proceed.

[harche]

@vrothberg sure, thanks.

[mtrmac]

/usr/bin/ld: cannot find -ldevmapper

That just seems like a bug in the build container, FWIW.

(Still, static builds are problematic, and at least I don’t think we should make them default. Have the users opt-in after reading documentation with warning.)

[mtrmac]

The consensus from #670 was #755 … but in the mean time we got another static build facility via Nix :/ .

Either way, make binary-static no longer exists, so the cannot find -ldevmapper bug in that target can’t be fixed any more.