我的程序有3个功能。每个函数都会获取一个项目列表并填充某些信息。 例如
class Item {
String sku,upc,competitorName;
double price;
}
功能 F1 获取一个列表并填充 upc
功能 F2 获取列表（F1的输出）并填写价格。
功能 F3 获取列表（F2的输出）并填充 competitorName
F1 可以一次处理5个项目， F2 可以一次处理20个项目， F3 也是20。
现在我正在运行F1 - ＆gt; F2 - ＆gt; F3连续，因为F2需要来自F1的信息（UPC代码）。 F3需要F2的价格。
我想通过连续运行F1运行而不是等待F2和F3完成来使这个过程高效。 F1执行并输出到队列中，然后F2一次取20个项目并处理它们。然后是F3。
如何通过使用BlockingCollection和Queue实现此目的？
答案 0 :(得分：2)
这是 Apache Storm 的典型用例，以防您有连续项目进入F1。您可以在几分钟内在Storm中实现这一点，并且您将拥有快速且完美并行的系统。您的F1，F2和F3将成为螺栓，您的物品生产商将成为鲸鱼喷水。
由于你问过如何使用BlockingCollections这是一个实现。你总共需要3个线程。
ItemsProducer：一次生产5件物品并送到F1。
F2ExecutorThread：一次消耗20件物品并将其送到F2。
F3ExecutorThread：一次消耗20件物品并将其送到F3。
您还有2个阻止队列，一个用于从F1-> F2传输数据，另一个用于F2-> F3。如果需要，您还可以使用类似的方式将数据提供给F1。这取决于你如何获得物品。我使用Thread.sleep来模拟执行函数所需的时间。
每个函数将继续查找其指定队列中的项目，而不管其他函数正在执行什么操作，并等待队列中有项目。一旦他们处理了该项目，他们就会将其放入另一个队列中以获得另一个功能。如果它已满，它们将等到另一个队列有空间。
由于所有功能都在不同的线程中运行，因此F1不会等待F2或F3完成。如果您的F2和F3明显快于F1，您可以为F1分配更多线程并继续推送到相同的f2Queue。
public class App {
final BlockingQueue<Item> f2Queue = new ArrayBlockingQueue<>(100);
final BlockingQueue<Item> f3Queue = new ArrayBlockingQueue<>(100);
public static void main(String[] args) throws InterruptedException {
App app = new App();
app.start();
}
public void start() throws InterruptedException {
Thread t1 = new ItemsProducer(f2Queue);
Thread t2 = new F2ExecutorThread(f2Queue, f3Queue);
Thread t3 = new F3ExecutorThread(f3Queue);
t1.start();
t2.start();
t3.start();
t1.join();
t2.join();
t3.join();
}
}
/**
* Thread producing 5 items at a time and feeding it to f1()
*/
class ItemsProducer extends Thread {
private BlockingQueue<Item> f2Queue;
private static final int F1_BATCH_SIZE = 5;
public ItemsProducer(BlockingQueue<Item> f2Queue) {
this.f2Queue = f2Queue;
}
public void run() {
Random random = new Random();
while (true) {
try {
List<Item> items = new ArrayList<>();
for (int i = 0; i < F1_BATCH_SIZE; i++) {
Item item = new Item(String.valueOf(random.nextInt(100)));
Thread.sleep(20);
items.add(item);
System.out.println("Item produced: " + item);
}
// Feed items to f1
f1(items);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
void f1(List<Item> items) throws InterruptedException {
Random random = new Random();
for (Item item : items) {
Thread.sleep(100);
item.upc = String.valueOf(random.nextInt(100));
f2Queue.put(item);
}
}
}
/**
* Thread consuming items produced by f1(). It takes 20 items at a time, but if they are not
* available it waits and starts processesing as soon as one gets available
*/
class F2ExecutorThread extends Thread {
static final int F2_BATCH_SIZE = 20;
private BlockingQueue<Item> f2Queue;
private BlockingQueue<Item> f3Queue;
public F2ExecutorThread(BlockingQueue<Item> f2Queue, BlockingQueue<Item> f3Queue) {
this.f2Queue = f2Queue;
this.f3Queue = f3Queue;
}
public void run() {
try {
List<Item> items = new ArrayList<>();
while (true) {
items.clear();
if (f2Queue.drainTo(items, F2_BATCH_SIZE) == 0) {
items.add(f2Queue.take());
}
f2(items);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
void f2(List<Item> items) throws InterruptedException {
Random random = new Random();
for (Item item : items) {
Thread.sleep(100);
item.price = random.nextInt(100);
f3Queue.put(item);
}
}
}
/**
* Thread consuming items produced by f2(). It takes 20 items at a time, but if they are not
* available it waits and starts processesing as soon as one gets available.
*/
class F3ExecutorThread extends Thread {
static final int F3_BATCH_SIZE = 20;
private BlockingQueue<Item> f3Queue;
public F3ExecutorThread(BlockingQueue<Item> f3Queue) {
this.f3Queue = f3Queue;
}
public void run() {
try {
List<Item> items = new ArrayList<>();
while (true) {
items.clear();
if (f3Queue.drainTo(items, F3_BATCH_SIZE) == 0) {
items.add(f3Queue.take());
}
f3(items);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
private void f3(List<Item> items) throws InterruptedException {
Random random = new Random();
for (Item item : items) {
Thread.sleep(100);
item.competitorName = String.valueOf(random.nextInt(100));
System.out.println("Item done: " + item);
}
}
}
class Item {
String sku, upc, competitorName;
double price;
public Item(String sku) {
this.sku = sku;
}
public String toString() {
return "sku: " + sku + " upc: " + upc + " price: " + price + " compName: " + competitorName;
}
}
我猜你也可以在.Net中采用完全相同的方法。为了更好地理解，我建议您浏览http://storm.apache.org/releases/current/Tutorial.html
的基本架构答案 1 :(得分：1)
我试图在.NET中做同样的事情，我认为它正在发挥作用。
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace BlockingCollectionExample
{
class Program
{
static void Main(string[] args)
{
BlockingCollection<Listing> needUPCJobs = new BlockingCollection<Listing>();
BlockingCollection<Listing> needPricingJobs = new BlockingCollection<Listing>();
// This will have final output
List<Listing> output = new List<Listing>();
// start executor 1 which waits for data until available
var executor1 = Task.Factory.StartNew(() =>
{
int maxSimutenousLimit = 5;
int gg = 0;
while (true)
{
while (needUPCJobs.Count >= maxSimutenousLimit)
{
List<Listing> tempListings = new List<Listing>();
for (int i = 0; i < maxSimutenousLimit; i++)
{
Listing listing = new Listing();
if (needUPCJobs.TryTake(out listing))
tempListings.Add(listing);
}
// Simulating some delay for first executor
Thread.Sleep(1000);
foreach (var eachId in tempListings)
{
eachId.UPC = gg.ToString();
gg++;
needPricingJobs.Add(eachId);
}
}
if (needUPCJobs.IsAddingCompleted)
{
if (needUPCJobs.Count == 0)
break;
else
maxSimutenousLimit = needUPCJobs.Count;
}
}
needPricingJobs.CompleteAdding();
});
// start executor 2 which waits for data until available
var executor2 = Task.Factory.StartNew(() =>
{
int maxSimutenousLimit = 10;
int gg = 10;
while (true)
{
while (needPricingJobs.Count >= maxSimutenousLimit)
{
List<Listing> tempListings = new List<Listing>();
for (int i = 0; i < maxSimutenousLimit; i++)
{
Listing listing = new Listing();
if (needPricingJobs.TryTake(out listing))
tempListings.Add(listing);
}
// Simulating more delay for second executor
Thread.Sleep(10000);
foreach (var eachId in tempListings)
{
eachId.Price = gg;
gg++;
output.Add(eachId);
}
}
if (needPricingJobs.IsAddingCompleted)
{
if(needPricingJobs.Count==0)
break;
else
maxSimutenousLimit = needPricingJobs.Count;
}
}
});
// producer thread
var producer = Task.Factory.StartNew(() =>
{
for (int i = 0; i < 100; i++)
{
needUPCJobs.Add(new Listing() { ID = i });
}
needUPCJobs.CompleteAdding();
});
// wait for producer to finish producing
producer.Wait();
// wait for all executors to finish executing
Task.WaitAll(executor1, executor2);
Console.WriteLine();
Console.WriteLine();
}
}
public class Listing
{
public int ID;
public string UPC;
public double Price;
public Listing() { }
}
}