Question

我有一个代码，其中包含近5.9亿个元素的巨大uint8个切片。我将这个数组保存在代码中，以使其尽可能快地运行。代码的最终大小为1.3 GB。

当我尝试编译它时，它会引发fatal error: out of memory。以下

# command-line-arguments
fatal error: out of memory

runtime stack:
runtime.throw(0x8fb3f2, 0xd)
    /usr/local/go/src/runtime/panic.go:566 +0x95
runtime.(*mcache).refill(0x7f5c2afa3ba8, 0x1440000000a, 0x7f57dc46d5e0)
    /usr/local/go/src/runtime/mcache.go:123 +0x160
runtime.(*mcache).nextFree.func1()
    /usr/local/go/src/runtime/malloc.go:505 +0x33
runtime.systemstack(0xc420044000)
    /usr/local/go/src/runtime/asm_amd64.s:298 +0x79
runtime.mstart()
    /usr/local/go/src/runtime/proc.go:1079

goroutine 1 [running]:
runtime.systemstack_switch()
    /usr/local/go/src/runtime/asm_amd64.s:252 fp=0x131f300c970 sp=0x131f300c968
runtime.(*mcache).nextFree(0x7f5c2afa3ba8, 0xa, 0x0, 0x131f300ca80, 0x1)
    /usr/local/go/src/runtime/malloc.go:506 +0xb2 fp=0x131f300c9c8 sp=0x131f300c970
runtime.mallocgc(0x90, 0x8ed060, 0x1, 0x115f3f9c)
    /usr/local/go/src/runtime/malloc.go:658 +0x809 fp=0x131f300ca68 sp=0x131f300c9c8
runtime.newobject(0x8ed060, 0xe00b46a450)
    /usr/local/go/src/runtime/malloc.go:785 +0x38 fp=0x131f300ca98 sp=0x131f300ca68
cmd/compile/internal/gc.Nod(0xd6c186c643, 0xfca1028090, 0xd6c186c480, 0x1)
    /usr/local/go/src/cmd/compile/internal/gc/subr.go:335 +0x31 fp=0x131f300cac0 sp=0x131f300ca98
cmd/compile/internal/gc.arraylit(0x0, 0x1, 0xc420663290, 0xfca1028090, 0x131f300d6c0)
    /usr/local/go/src/cmd/compile/internal/gc/sinit.go:698 +0x18f fp=0x131f300cb48 sp=0x131f300cac0
cmd/compile/internal/gc.slicelit(0x0, 0xc420663290, 0xfca1018000, 0x131f300d6c0)
    /usr/local/go/src/cmd/compile/internal/gc/sinit.go:765 +0xb36 fp=0x131f300cc58 sp=0x131f300cb48
cmd/compile/internal/gc.anylit(0x0, 0xc420663290, 0xfca1018000, 0x131f300d6c0)
    /usr/local/go/src/cmd/compile/internal/gc/sinit.go:1106 +0xac3 fp=0x131f300cd30 sp=0x131f300cc58
cmd/compile/internal/gc.oaslit(0xfca1017ef0, 0x131f300d6c0, 0xfca1018000)
    /usr/local/go/src/cmd/compile/internal/gc/sinit.go:1192 +0x104 fp=0x131f300cd60 sp=0x131f300cd30
cmd/compile/internal/gc.walkexpr(0xfca1017ef0, 0x131f300d6c0, 0x0)
    /usr/local/go/src/cmd/compile/internal/gc/walk.go:725 +0xe9c fp=0x131f300d5b8 sp=0x131f300cd60
cmd/compile/internal/gc.walkstmt(0xfca1017ef0, 0xfca1018090)
    /usr/local/go/src/cmd/compile/internal/gc/walk.go:192 +0xf45 fp=0x131f300d7f0 sp=0x131f300d5b8
cmd/compile/internal/gc.walkstmtlist(0x131f3184000, 0x21, 0x40)
    /usr/local/go/src/cmd/compile/internal/gc/walk.go:80 +0x4d fp=0x131f300d820 sp=0x131f300d7f0
cmd/compile/internal/gc.walk(0xc42065ccf0)
    /usr/local/go/src/cmd/compile/internal/gc/walk.go:65 +0x1e4 fp=0x131f300d8f0 sp=0x131f300d820
cmd/compile/internal/gc.compile(0xc42065ccf0)
    /usr/local/go/src/cmd/compile/internal/gc/pgen.go:391 +0x1d4 fp=0x131f300dba0 sp=0x131f300d8f0
cmd/compile/internal/gc.funccompile(0xc42065ccf0)
    /usr/local/go/src/cmd/compile/internal/gc/dcl.go:1287 +0x186 fp=0x131f300dc18 sp=0x131f300dba0
cmd/compile/internal/gc.Main()
    /usr/local/go/src/cmd/compile/internal/gc/main.go:467 +0x19f8 fp=0x131f300de60 sp=0x131f300dc18
cmd/compile/internal/amd64.Main()
    /usr/local/go/src/cmd/compile/internal/amd64/galign.go:93 +0x2fa fp=0x131f300de98 sp=0x131f300de60
main.main()
    /usr/local/go/src/cmd/compile/main.go:33 +0x2a3 fp=0x131f300df28 sp=0x131f300de98
runtime.main()
    /usr/local/go/src/runtime/proc.go:183 +0x1f4 fp=0x131f300df80 sp=0x131f300df28
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0x131f300df88 sp=0x131f300df80

我认为机器的RAM足以完成此类任务但无法通过此问题。我用另一片2100万个元素尝试了代码，它可以正常工作。这是一项非常困难的计算任务，因为它需要非常快速地运行，所以我不能使用外部文件来处理数据。

已经对变量类型进行了优化。

有什么想法吗？

Answer 1

我从外部二进制文件加载数据作为字节。以下代码@BJ Black提供了帮助。

    buf, err := ioutil.ReadFile("somefile.bin")
    if err != nil { panic(err.Error()) }
    // Coerce type
    uintbuf := []uint8(buf)

Golang，致命错误：1 TB RAM计算机内存不足

1 个答案: