Go中限制并发的简单方法

Question

我有一个包含大约 10k 个 URL 的 CSV 文件，我需要通过 HTTP 获取。将 Go 例程的并发限制为一次不超过 16 个的最简单方法是什么？func getUrl(url string) {    request := gorequest.New()    resp, body, errs := request.Get(each[1]).End()    _ = resp    _ = body    _ = errs}func main() {    csvfile, err := os.Open("urls.csv")    defer csvfile.Close()    reader := csv.NewReader(csvfile)    reader.FieldsPerRecord = -1     rawCSVdata, err := reader.ReadAll()    completed := 0    for _, each := range rawCSVdata {        go getUrl(each[1])        completed++    }}

Answer 1

生产者-消费者模式：

package main

import (

    "encoding/csv"

    "os"

    "sync"

    "github.com/parnurzeal/gorequest"

)

const workersCount = 16

func getUrlWorker(urlChan chan string) {

    for url := range urlChan {

        request := gorequest.New()

        resp, body, errs := request.Get(url).End()

        _ = resp

        _ = body

        _ = errs

    }

}

func main() {

    csvfile, err := os.Open("urls.csv")

    if err != nil {

        panic(err)

    }

    defer csvfile.Close()

    reader := csv.NewReader(csvfile)

    reader.FieldsPerRecord = -1

    rawCSVdata, err := reader.ReadAll()

    var wg sync.WaitGroup

    urlChan := make(chan string)

    wg.Add(workersCount)

    for i := 0; i < workersCount; i++ {

        go func() {

            getUrlWorker(urlChan)

            wg.Done()

        }()

    }

    completed := 0

    for _, each := range rawCSVdata {

        urlChan <- each[1]

        completed++

    }

    close(urlChan)

    wg.Wait()

}