我正在尝试构建用于使用Go提取数据的管道。这三个阶段是“下载批处理”,“转换每个消息”和“将消息放入队列”。
在我看来,很自然的逻辑是为每个阶段创建3个功能,并将这些功能与无缓冲通道绑定在一起。
在我的代码中的某个地方,我没有正确实现通道,还是没有使用等待组?由于只有一条消息到达最后阶段,程序似乎停止/阻塞。
func (c *worker) startWork() {
// channel for messages to be sent to the queue
chMessagesToEnqueue := make(chan types.Foo)
// channel for messages to be transformed
chMessagesToTransform := make(chan []types.UpstreamFooType)
// start the goroutines with the channels
go c.startTransformer(chMessagesToTransform, chMessagesToEnqueue)
go c.startEnqueuer(chMessagesToEnqueue)
go c.startDownloader(chMessagesToTransform)
}
func (c *worker) startDownloader(out chan []types.UpstreamFooType) {
// https://github.com/SebastiaanKlippert/go-soda
// uses a library here to fetch data from upstream APIs, but the gist is:
var wg sync.WaitGroup
for i := 0; i < c.workerCount; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for {
// i've cut out some meat out to make more concise
var results []types.UpstreamFooType
err = json.NewDecoder(resp.Body).Decode(&results)
out <- results
}
}()
}
wg.Wait()
}
func (c *worker) startTransformer(in <-chan []types.UpstreamFooType, out chan types.Foo) {
data := <-in
for _, record := range data {
msg := types.Foo{
name: record.fifa,
}
out <- msg
}
}
func (c *worker) startEnqueuer(in <-chan []types.Foo) {
data := <-in
c.logger.Infow("startEnqueuer", "data", data)
}