我有一个大型数据集,我正在执行以下操作:
data = %{
city: ["Chiselhurst", "Saskatchewan", "Boston"],
day: [1, 2, 3, 4, 5, 6, 7],
transportation: ["plane", "train", "automobile"]
}
defmodule Make do
def combos(data) do
for city <- data[:city],
day <- data[:day],
transportation <- data[:transportation] do
List.flatten([city, day, transportation])
end
end
def filter_map(data) do
data
|> Enum.filter_map(fn(x) -> x[:day] > 5 end, &(%{:trip => &1}))
end
def combos_filter(data) do
data
|> combos
|> filter_map
end
end
有没有办法让理解(Make.combos
)更有效率?
有没有办法异步过滤?我一直在使用Task.async
/ Task.await
,但我们无法将其与Enum.filter
一起使用。
更新
day_range = for n <- 1..5_000_000, do: n
data = %{
city: ["Chiselhurst", "Saskatchewan", "Boston"],
day: day_range,
transportation: ["plane", "train", "automobile"]
}
defmodule Make do
def combos(data) do
for city <- data[:city],
day <- data[:day],
transportation <- data[:transportation] do
[city, day, transportation]
end
end
def combos_async(data) do
chiselhurst = Task.async(fn -> partial_combos("Chiselhurst", data) end)
saskatchewan = Task.async(fn -> partial_combos("Saskatchewan", data) end)
boston = Task.async( fn -> partial_combos("Boston", data) end)
Task.await(chiselhurst, 1000000) ++ Task.await(saskatchewan, 1000000) ++ Task.await(boston, 1000000)
end
defp partial_combos(city, data) do
for day <- data[:day],
transportation <- data[:transportation] do
[city, day, transportation]
end
end
end