Skip to content

out of memory when docker image prune #1422

Closed
Closed
out of memory when docker image prune#1422
@denverdino

Description

@denverdino
denverdino
opened on Oct 9, 2018

Description

Get out of memory exception when prune many images

Steps to reproduce the issue:

# docker image prune -a --force --filter "until=720h"

Describe the results you received:

fatal error: runtime: out of memory

runtime stack:
runtime.throw(0x17e114c, 0x16)
/usr/local/go/src/runtime/panic.go:616 +0x81
runtime.sysMap(0xc426800000, 0x100000, 0x590000, 0x2555db8)
/usr/local/go/src/runtime/mem_linux.go:216 +0x20a
runtime.(*mheap).sysAlloc(0x253bbe0, 0x100000, 0x7f69cbfa4fb0)
/usr/local/go/src/runtime/malloc.go:470 +0xd4
runtime.(*mheap).grow(0x253bbe0, 0x20, 0x0)
/usr/local/go/src/runtime/mheap.go:907 +0x60
runtime.(*mheap).allocSpanLocked(0x253bbe0, 0x20, 0x2555dc8, 0x7f69cbfa46c8)
/usr/local/go/src/runtime/mheap.go:820 +0x301
runtime.(*mheap).alloc_m(0x253bbe0, 0x20, 0x101, 0x7f69cbfa46c8)
/usr/local/go/src/runtime/mheap.go:686 +0x118
runtime.(*mheap).alloc.func1()
/usr/local/go/src/runtime/mheap.go:753 +0x4d
runtime.(*mheap).alloc(0x253bbe0, 0x20, 0x7f69cb000101, 0x7f69cbfa46c8)
/usr/local/go/src/runtime/mheap.go:752 +0x8a
runtime.largeAlloc(0x3e971, 0x450100, 0x7f69cbfa46c8)
/usr/local/go/src/runtime/malloc.go:826 +0x94
runtime.mallocgc.func1()
/usr/local/go/src/runtime/malloc.go:721 +0x46
runtime.systemstack(0x0)
/usr/local/go/src/runtime/asm_amd64.s:409 +0x79
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1175

goroutine 1 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_amd64.s:363 fp=0xc4211ff880 sp=0xc4211ff878 pc=0x457b50
runtime.mallocgc(0x3e971, 0x0, 0xc4211ff900, 0x41403d)
/usr/local/go/src/runtime/malloc.go:720 +0x8a2 fp=0xc4211ff920 sp=0xc4211ff880 pc=0x4143d2
runtime.rawstring(0x3e971, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/runtime/string.go:245 +0x4f fp=0xc4211ff950 sp=0xc4211ff920 pc=0x447d9f
runtime.rawstringtmp(0x0, 0x3e971, 0x7f69cbff5000, 0xc420000180, 0xc420685cf0, 0x76, 0xc4205d2080)
/usr/local/go/src/runtime/string.go:115 +0x6f fp=0xc4211ff990 sp=0xc4211ff950 pc=0x4477ff
runtime.concatstrings(0x0, 0xc4211ffa70, 0x2, 0x2, 0xc42049d0e0, 0xc420152000)
/usr/local/go/src/runtime/string.go:46 +0x10a fp=0xc4211ffa28 sp=0xc4211ff990 pc=0x44735a
runtime.concatstring2(0x0, 0xc426786000, 0x3e920, 0xc42049d0e0, 0x51, 0xc426786000, 0x3e920)
/usr/local/go/src/runtime/string.go:55 +0x47 fp=0xc4211ffa68 sp=0xc4211ffa28 pc=0x447547
github.com/docker/cli/cli/command/image.runPrune(0x1985860, 0xc42020b6c0, 0x1420101, 0xc420699b30, 0x3, 0xc420619b98, 0xc4203a8de0, 0xc420619c68, 0x590630)
/go/src/github.com/docker/cli/cli/command/image/prune.go:81 +0x463 fp=0xc4211ffba0 sp=0xc4211ffa68 pc=0xb14fd3
github.com/docker/cli/cli/command/image.NewPruneCommand.func1(0xc4206be000, 0xc420695540, 0x0, 0x4, 0x0, 0x0)
/go/src/github.com/docker/cli/cli/command/image/prune.go:29 +0x64 fp=0xc4211ffc78 sp=0xc4211ffba0 pc=0xb1d4c4
github.com/docker/cli/vendor/github.com/spf13/cobra.(*Command).execute(0xc4206be000, 0xc42003a0a0, 0x4, 0x4, 0xc4206be000, 0xc42003a0a0)
/go/src/github.com/docker/cli/vendor/github.com/spf13/cobra/command.go:762 +0x468 fp=0xc4211ffd68 sp=0xc4211ffc78 pc=0x58f678
github.com/docker/cli/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc42034a000, 0xc420685f10, 0x1536ca0, 0xc420685f20)
/go/src/github.com/docker/cli/vendor/github.com/spf13/cobra/command.go:852 +0x30a fp=0xc4211ffea8 sp=0xc4211ffd68 pc=0x59008a
github.com/docker/cli/vendor/github.com/spf13/cobra.(*Command).Execute(0xc42034a000, 0xc42034a000, 0x194a740)
/go/src/github.com/docker/cli/vendor/github.com/spf13/cobra/command.go:800 +0x2b fp=0xc4211ffed8 sp=0xc4211ffea8 pc=0x58fd5b
main.main()
/go/src/github.com/docker/cli/cmd/docker/docker.go:174 +0xd0 fp=0xc4211fff88 sp=0xc4211ffed8 pc=0x1422600
runtime.main()
/usr/local/go/src/runtime/proc.go:198 +0x212 fp=0xc4211fffe0 sp=0xc4211fff88 pc=0x42f572
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc4211fffe8 sp=0xc4211fffe0 pc=0x45a6e1

goroutine 5 [syscall, 5 minutes]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:139 +0xa6
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:28 +0x41

goroutine 9 [chan receive]:
github.com/docker/cli/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0x2535180)
/go/src/github.com/docker/cli/vendor/github.com/golang/glog/glog.go:879 +0x8b
created by github.com/docker/cli/vendor/github.com/golang/glog.init.0
/go/src/github.com/docker/cli/vendor/github.com/golang/glog/glog.go:410 +0x203

goroutine 12 [IO wait, 5 minutes]:
internal/poll.runtime_pollWait(0x7f69cbf99f00, 0x72, 0xc42006d9a8)
/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420130b98, 0x72, 0xffffffffffffff00, 0x194ca20, 0x24beb88)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc420130b98, 0xc420265000, 0x1000, 0x1000)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc420130b80, 0xc420265000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc420130b80, 0xc420265000, 0x1000, 0x1000, 0x4565d0, 0xc420000180, 0x4)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4205e82c8, 0xc420265000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc42053ec60, 0xc420265000, 0x1000, 0x1000, 0xc42006db98, 0x4072a5, 0xc420046c00)
/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4202d67e0)
/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4202d67e0, 0x1, 0x0, 0x0, 0x0, 0xc420046b40, 0x0)
/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc42053ec60)
/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 13 [select, 5 minutes]:
net/http.(*persistConn).writeLoop(0xc42053ec60)
/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1238 +0x97f

goroutine 15 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f69cbf99e30, 0x72, 0xc4200709a8)
/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc420131898, 0x72, 0xffffffffffffff00, 0x194ca20, 0x24beb88)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc420131898, 0xc42015d000, 0x1000, 0x1000)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc420131880, 0xc42015d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc420131880, 0xc42015d000, 0x1000, 0x1000, 0x4565d0, 0xc420000180, 0x4)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc4205e8308, 0xc42015d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:176 +0x6a
net/http.(*persistConn).Read(0xc42053eea0, 0xc42015d000, 0x1000, 0x1000, 0xc420070b98, 0x4072a5, 0xc420046f00)
/usr/local/go/src/net/http/transport.go:1453 +0x136
bufio.(*Reader).fill(0xc4202d6c60)
/usr/local/go/src/bufio/bufio.go:100 +0x11e
bufio.(*Reader).Peek(0xc4202d6c60, 0x1, 0x0, 0x0, 0x0, 0xc420047080, 0x0)
/usr/local/go/src/bufio/bufio.go:132 +0x3a
net/http.(*persistConn).readLoop(0xc42053eea0)
/usr/local/go/src/net/http/transport.go:1601 +0x185
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1237 +0x95a

goroutine 16 [select, 6 minutes]:
net/http.(*persistConn).writeLoop(0xc42053eea0)
/usr/local/go/src/net/http/transport.go:1822 +0x14b
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1238 +0x97f

Describe the results you expected:

Prune image successfully

Additional information you deem important (e.g. issue happens only occasionally):

I have tons of out of images

Output of docker version:

Client:
 Version:           18.06.0-ce
 API version:       1.38
 Go version:        go1.10.3
 Git commit:        0ffa825
 Built:             Wed Jul 18 19:10:22 2018
 OS/Arch:           linux/amd64
 Experimental:      false

Server:
 Engine:
  Version:          18.06.0-ce
  API version:      1.38 (minimum version 1.12)
  Go version:       go1.10.3
  Git commit:       0ffa825
  Built:            Wed Jul 18 19:08:26 2018
  OS/Arch:          linux/amd64
  Experimental:     false

Output of docker info:

Containers: 1
 Running: 1
 Paused: 0
 Stopped: 0
Images: 411
Server Version: 18.06.0-ce
Storage Driver: aufs
 Root Dir: /var/lib/docker/aufs
 Backing Filesystem: extfs
 Dirs: 1064
 Dirperm1 Supported: false
Logging Driver: json-file
Cgroup Driver: cgroupfs
Plugins:
 Volume: local
 Network: bridge host macvlan null overlay
 Log: awslogs fluentd gcplogs gelf journald json-file logentries splunk syslog
Swarm: inactive
Runtimes: runc
Default Runtime: runc
Init Binary: docker-init
containerd version: d64c661f1d51c48782c9cec8fda7604785f93587
runc version: 69663f0bd4b60df09991c08812a60108003fa340
init version: fec3683
Security Options:
 apparmor
Kernel Version: 3.13.0-86-generic
Operating System: Ubuntu 14.04.5 LTS
OSType: linux
Architecture: x86_64
CPUs: 1
Total Memory: 993.9MiB
Name: iZ62lox823oZ
ID: CGJL:JQP3:BXST:KEYH:XO4G:Q2N4:IQ76:CIRN:BPJJ:TTVF:R7UU:CB4A
Docker Root Dir: /var/lib/docker
Debug Mode (client): false
Debug Mode (server): false
Registry: https://index.docker.io/v1/
Labels:
Experimental: false
Insecure Registries:
 127.0.0.0/8
Live Restore Enabled: false

WARNING: No swap limit support

Additional environment details (AWS, VirtualBox, physical, etc.):

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions