我有如下的话语（字符串）/语料库列表，
List<string> allUtterances = new List<string>
{
"c2's are above the hierarchy than c1's",
"c2's are better than c1's",
"get me a group of 10 c1's",
"he is a c2",
"he was a c two",
"hey i am a c1",
"jdsaxkjhasx",
"khndsmcsdfcs",
"my competency is c2",
"none intent",
"she is still a c 1",
"this is a none intent, please ignore",
"we are hiring fresh c1's"
};
这是类模式：
public class ListEntity
{
public string name { get; set; }
public List<Sublist> subLists { get; set; }
}
public class Sublist
{
public string canonicalForm { get; set; }
public List<string> list { get; set; }
}
这是一个示例POCO：
List<ListEntity> listEntities = new List<ListEntity>
{
new ListEntity
{
name = "Competency",
subLists = new List<Sublist>
{
new Sublist
{
canonicalForm = "C1",
list = new List<string>
{
"c1",
"c one",
"c 1",
"C 1",
"C1",
"C one",
"C ONE"
}
},
new Sublist
{
canonicalForm = "C2",
list = new List<string>
{
"c2",
"c two",
"c 2",
"C 2",
"C2",
"C two",
"C TWO"
}
}
}
}
};
var canonicalForms = listEntities.Select(x => x.subLists.Select(y => y.list).ToList()).ToList();
假设我从上述列表allUtterances中说出以下话：
"query": "C2's are better than C1's"
对于上述话，我想获得以下输出：
{
"entity": "c2",
"type": "Competency",
"startIndex": 0,
"endIndex": 1,
"resolution": {
"values": [
"C2"
]
}
},
{
"entity": "c1",
"type": "Competency",
"startIndex": 21,
"endIndex": 22,
"resolution": {
"values": [
"C1"
]
}
}
我要匹配的规则如下：
对于allUtterances list中的所有语音，如果语音文本包含类子列表的属性list中的值，我要提取开始和结束位置，并用适当的键（在这种情况下为canonicalForm进行标记），使用ListEntityClass中的name属性更新我的JSON有效负载中的类型键。
我尝试了以下方法：
using System;
using System.Linq;
using System.IO;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System.Collections.Generic;
namespace ListEntityProblem
{
class Program
{
static void Main(string[] args)
{
List<string> allUtterances = new List<string>
{
"c2's are above the hierarchy than c1's",
"c2's are better than c1's",
"get me a group of 10 c1's",
"he is a c2",
"he was a c two",
"hey i am a c1",
"jdsaxkjhasx",
"khndsmcsdfcs",
"my competency is c2",
"none intent",
"she is still a c 1",
"this is a none intent, please ignore",
"we are hiring fresh c1's"
};
List<ListEntity> listEntities = new List<ListEntity>
{
new ListEntity
{
name = "Competency",
subLists = new List<Sublist>
{
new Sublist
{
canonicalForm = "C1",
list = new List<string>
{
"c1",
"c one",
"c 1",
"C 1",
"C1",
"C one",
"C ONE"
}
},
new Sublist
{
canonicalForm = "C2",
list = new List<string>
{
"c2",
"c two",
"c 2",
"C 2",
"C2",
"C two",
"C TWO"
}
}
}
}
};
List<Tuple<string, string, List<string>>> ListEntityLookup = new List<Tuple<string, string, List<string>>>();
//n^2, construct lookup for list entities
foreach (var item in listEntities)
{
string listEntityName = item.name;
foreach (var innerItem in item.subLists)
{
string normalizedValue = innerItem.canonicalForm;
List<string> synonymValues = innerItem.list;
ListEntityLookup.Add(Tuple.Create<string, string, List<string>>(listEntityName, normalizedValue, synonymValues));
}
}
List<JObject> parsedEntities = new List<JObject>();
//n^3, populate the parsed payload with start and end indices
foreach (var item in allUtterances)
{
foreach (var ll in ListEntityLookup)
{
foreach (var cf in ll.Item3)
{
int start = 0, end = 0;
if (item.Contains(cf))
{
start = item.IndexOf(cf);
end = start + cf.Length;
parsedEntities.Add(new JObject
{
new JProperty("Start", start),
new JProperty("End", end),
new JProperty("Query", item),
new JProperty("CanonicalForm", ll.Item2),
new JProperty("ListEntity", ll.Item1)
});
}
}
}
}
//Group by query
var groupedParsedEntities = parsedEntities.GroupBy(x => x["Query"]).ToList();
}
}
}
编辑：
我试图重新编写for-each循环，但这导致了更多的嵌套。
foreach (var item in allUtterances)
{
foreach (var listEntity in listEntities)
{
foreach (var canonicalForm in listEntity.subLists)
{
foreach(var synonym in canonicalForm.list)
{
int start = item.IndexOf(synonym);
if(start != -1)
{
parsedEntities.Add(new JObject
{
new JProperty("Start", start),
new JProperty("End", start + synonym.Length),
new JProperty("Query", item),
new JProperty("CanonicalForm", canonicalForm.canonicalForm),
new JProperty("ListEntity", listEntity.name)
});
}
}
}
}
}
但是这种方法似乎对于大量话语来说放慢了速度，并且扩展性不是很好。由于主循环运行n ^ 3次。我们的服务器每秒必须执行太多计算。
我一直在考虑是否应该使用Regex，如果它可以给我带来一些性能上的好处。
请帮助我优化此算法。
任何帮助将不胜感激。
答案 0 :(得分：1)
您是否尝试过使用Linq查询而不是循环？
这并不能完全满足您的需求，但我相信它确实可以提取相关数据：
allUtterances
.AsParallel()
.SelectMany(utterance => listEntities.SelectMany(l => l.subLists
.Where(sl => sl.list.Any(sle => utterance.Contains(sle)))
.SelectMany(sl => sl.list
.Where(sle => utterance.Contains(sle))
.Select(sle => new {
canonicalForm = sl.canonicalForm,
matchedValue = sle,
startindex = utterance.IndexOf(sle),
endindex = utterance.IndexOf(sle) + sle.Length - 1
})
)
.Select(o => new {
// not sure if 'entity' and 'resolutionValue' are swopped around
utterance = utterance,
entity = o.matchedValue,
type = l.name,
startIndex = o.startindex,
endIndex = o.endindex,
resolutionValue = o.canonicalForm,
}
)
/*
or change the Select above to create the JObjects:
.Select(jo => new JObject {
new JProperty("Start", jo.startIndex),
new JProperty("End", jo.endIndex),
new JProperty("Query", jo.utterance),
new JProperty("CanonicalForm", jo.resolutionValue),
new JProperty("ListEntity", jo.entity)
})
*/
)).ToList();
或者，您可以尝试并行化循环：
allUtterances.AsParallel().ForAll(ut => { .... });