Question

我有一个CSV文件，包含~10k我需要HTTP的URL。将Go例程的并发性限制为一次不超过16的最简单方法是什么？

func getUrl(url string) {
    request := gorequest.New()
    resp, body, errs := request.Get(each[1]).End()
    _ = resp
    _ = body
    _ = errs
}

func main() {

    csvfile, err := os.Open("urls.csv")
    defer csvfile.Close()
    reader := csv.NewReader(csvfile)
    reader.FieldsPerRecord = -1 
    rawCSVdata, err := reader.ReadAll()

    completed := 0
    for _, each := range rawCSVdata {
        go getUrl(each[1])
        completed++
    }
}

Answer 1

生产者 - 消费者模式：

package main

import (
    "encoding/csv"
    "os"
    "sync"

    "github.com/parnurzeal/gorequest"
)

const workersCount = 16

func getUrlWorker(urlChan chan string) {
    for url := range urlChan {
        request := gorequest.New()
        resp, body, errs := request.Get(url).End()
        _ = resp
        _ = body
        _ = errs
    }
}

func main() {
    csvfile, err := os.Open("urls.csv")
    if err != nil {
        panic(err)
    }
    defer csvfile.Close()

    reader := csv.NewReader(csvfile)
    reader.FieldsPerRecord = -1
    rawCSVdata, err := reader.ReadAll()

    var wg sync.WaitGroup
    urlChan := make(chan string)

    wg.Add(workersCount)

    for i := 0; i < workersCount; i++ {
        go func() {
            getUrlWorker(urlChan)
            wg.Done()
        }()
    }

    completed := 0
    for _, each := range rawCSVdata {
        urlChan <- each[1]
        completed++
    }
    close(urlChan)

    wg.Wait()
}

在Go中限制并发性的简单方法

1 个答案: