Question

我有一个包含约210万个日志字符串的切片，我想创建一个切片，其中字符串尽可能均匀分布。

这是我到目前为止所做的：

// logs is a slice with ~2.1 million strings in it.
var divided = make([][]string, 0)
NumCPU := runtime.NumCPU()
ChunkSize := len(logs) / NumCPU
for i := 0; i < NumCPU; i++ {
    temp := make([]string, 0)
    idx := i * ChunkSize
    end := i * ChunkSize + ChunkSize
    for x := range logs[idx:end] {
        temp = append(temp, logs[x])
    }
    if i == NumCPU {
        for x := range logs[idx:] {
            temp = append(temp, logs[x])
        }
    }
    divided = append(divided, temp)
}

idx := i * ChunkSize将为logs索引提供当前的“块开始”，而end := i * ChunkSize + ChunkSize将为我提供“块末端”或该范围的结尾块。我找不到任何关于如何在Go中分块或分割切片或迭代有限范围的文档或示例，所以这就是我提出的。但是，它只复制第一个块多次，因此不起作用。

我如何（尽可能均匀地）在Go中切片？

Answer 1

您不需要制作新切片，只需将logs的切片附加到divided切片。

http://play.golang.org/p/vyihJZlDVy

var divided [][]string

chunkSize := (len(logs) + numCPU - 1) / numCPU

for i := 0; i < len(logs); i += chunkSize {
    end := i + chunkSize

    if end > len(logs) {
        end = len(logs)
    }

    divided = append(divided, logs[i:end])
}

fmt.Printf("%#v\n", divided)

Answer 2

另一种变体。它的工作速度比 JimB 提出的方法快 2.5 倍。测试和基准是 here。

https://play.golang.org/p/WoXHqGjozMI

func chunks(xs []string, chunkSize int) [][]string {
    if len(xs) == 0 {
        return nil
    }
    divided := make([][]string, (len(xs)+chunkSize-1)/chunkSize)
    prev := 0
    i := 0
    till := len(xs) - chunkSize
    for prev < till {
        next := prev + chunkSize
        divided[i] = xs[prev:next]
        prev = next
        i++
    }
    divided[i] = xs[prev:]
    return divided
}

Answer 3

对任何 []T 使用反射

https://github.com/kirito41dd/xslice

package main

import (
    "fmt"
    "github.com/kirito41dd/xslice"
)

func main() {
    s := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
    i := xslice.SplitToChunks(s, 3)
    ss := i.([][]int)
    fmt.Println(ss) // [[0 1 2] [3 4 5] [6 7 8] [9]]
}

https://github.com/kirito41dd/xslice/blob/e50d91fa75241a3a03d262ad51c8e4cb2ea4b995/split.go#L12

func SplitToChunks(slice interface{}, chunkSize int) interface{} {
    sliceType := reflect.TypeOf(slice)
    sliceVal := reflect.ValueOf(slice)
    length := sliceVal.Len()
    if sliceType.Kind() != reflect.Slice {
        panic("parameter must be []T")
    }
    n := 0
    if length%chunkSize > 0 {
        n = 1
    }
    SST := reflect.MakeSlice(reflect.SliceOf(sliceType), 0, length/chunkSize+n)
    st, ed := 0, 0
    for st < length {
        ed = st + chunkSize
        if ed > length {
            ed = length
        }
        SST = reflect.Append(SST, sliceVal.Slice(st, ed))
        st = ed
    }
    return SST.Interface()
}

在Go中切片

3 个答案: