我正在尝试使用spaCy创建带有产品名称列表的新实体分类“ MYPRODUCT”。更新模型并将其保存到磁盘后，旧标签将被遗忘。这是正确的方法吗？我使用了一些现有示例并创建了我的脚本。谁能帮我吗？
我的代码：
# training data
TRAIN_DATA = [
('Smartstream is a Infor Product.', {'entities': [(0, 11, 'MYPRODUCT')]}),
("Horses are too tall and they pretend to care about your feelings", {'entities': [(0, 6, 'ANIMAL')]}),
("Do they bite?", {'entities': [] }),
("horses are too tall and they pretend to care about your feelings", { 'entities': [(0, 6, 'ANIMAL')] }),
("horses pretend to care about your feelings", { 'entities': [(0, 6, 'ANIMAL')] }),
("horses?", {'entities': [(0, 6, 'ANIMAL')]}),
("General",{"entities":[(0,7, 'MYPRODUCT')]}),
("SmartStream",{"entities":[(0,11, 'MYPRODUCT')]}),
("TotalHR",{"entities":[(0,7, 'MYPRODUCT')]}),
("E Series",{"entities":[(0,8, 'MYPRODUCT')]}),
("M Series",{"entities":[(0,8, 'MYPRODUCT')]}),
("Infor Support Portal - Customer Care",{"entities":[(0,36, 'MYPRODUCT')]}),
("ALLTAX",{"entities":[(0,6, 'MYPRODUCT')]}),
("Infor Partner Network",{"entities":[(0,21, 'MYPRODUCT')]}),
("Host General",{"entities":[(0,12, 'MYPRODUCT')]}),
("StreamLine",{"entities":[(0,10, 'MYPRODUCT')]}),
("System 21",{"entities":[(0,9, 'MYPRODUCT')]}),
("TIMS",{"entities":[(0,4, 'MYPRODUCT')]}),
("i2",{"entities":[(0,2, 'MYPRODUCT')]}),
("EnRoute",{"entities":[(0,7, 'MYPRODUCT')]}),
("Help - zSeries",{"entities":[(0,14, 'MYPRODUCT')]}),
("Expense Management",{"entities":[(0,18, 'MYPRODUCT')]}),
("PM",{"entities":[(0,2, 'MYPRODUCT')]}),
("Pathway",{"entities":[(0,7, 'MYPRODUCT')]}),
("Education",{"entities":[(0,9, 'MYPRODUCT')]}),
("Anael",{"entities":[(0,5, 'MYPRODUCT')]}),
("Library Solutions",{"entities":[(0,17, 'MYPRODUCT')]}),
("HR Tax & Reg",{"entities":[(0,12, 'MYPRODUCT')]}),
("Pegasus",{"entities":[(0,7, 'MYPRODUCT')]}),
("F9",{"entities":[(0,2, 'MYPRODUCT')]}),
("SunSystems",{"entities":[(0,10, 'MYPRODUCT')]}),
("Q&A/Vision",{"entities":[(0,10, 'MYPRODUCT')]}),
("Elevon",{"entities":[(0,6, 'MYPRODUCT')]}),
("MAX",{"entities":[(0,3, 'MYPRODUCT')]}),
("Prism",{"entities":[(0,5, 'MYPRODUCT')]}),
("Protean",{"entities":[(0,7, 'MYPRODUCT')]}),
("SXe",{"entities":[(0,3, 'MYPRODUCT')]}),
("Unison",{"entities":[(0,6, 'MYPRODUCT')]}),
("PRMS",{"entities":[(0,4, 'MYPRODUCT')]}),
("IRONSIDE",{"entities":[(0,8, 'MYPRODUCT')]}),
("Epiphany Sales and Service",{"entities":[(0,26, 'MYPRODUCT')]}),
("BI/Cognos",{"entities":[(0,9, 'MYPRODUCT')]}),
("FMS Masterpiece",{"entities":[(0,15, 'MYPRODUCT')]}),
("ERP LX",{"entities":[(0,6, 'MYPRODUCT')]}),
("BOSS",{"entities":[(0,4, 'MYPRODUCT')]}),
("INFINIUM",{"entities":[(0,8, 'MYPRODUCT')]}),
("SCM WM",{"entities":[(0,6, 'MYPRODUCT')]}),
("SCEM",{"entities":[(0,4, 'MYPRODUCT')]}),
("Epiphany Interaction Advisor",{"entities":[(0,28, 'MYPRODUCT')]}),
("Epiphany Marketing",{"entities":[(0,18, 'MYPRODUCT')]}),
("KBM",{"entities":[(0,3, 'MYPRODUCT')]}),
("TMS",{"entities":[(0,3, 'MYPRODUCT')]}),
("iSeries EAM",{"entities":[(0,11, 'MYPRODUCT')]}),
("Demand Planning",{"entities":[(0,15, 'MYPRODUCT')]}),
("CAS",{"entities":[(0,3, 'MYPRODUCT')]}),
("PROVIA",{"entities":[(0,6, 'MYPRODUCT')]})
]
@plac.annotations(
model=("Model name. Defaults to blank 'en' model.", "option", "m", str),
output_dir=("Optional output directory", "option", "o", Path),
n_iter=("Number of training iterations", "option", "n", int))
def main(model="en", output_dir="c:\\myproject\\models", n_iter=10):
"""Load the model, set up the pipeline and train the entity recognizer."""
if model is not None:
nlp = spacy.load(model) # load existing spaCy model
print("Loaded model '%s'" % model)
else:
nlp = spacy.blank('en') # create blank Language class
print("Created blank 'en' model")
# create the built-in pipeline components and add them to the pipeline
# nlp.create_pipe works for built-ins that are registered with spaCy
if 'ner' not in nlp.pipe_names:
ner = nlp.create_pipe('ner')
nlp.add_pipe(ner, last=True)
# otherwise, get it so we can add labels
else:
ner = nlp.get_pipe('ner')
# add labels
ner.add_label('MYPRODUCT')
ner.add_label('ANIMAL') # add new entity label to entity recognizer
ner.add_label('GOODBYE')
ner.add_label('CURSE')
ner.add_label('AFFIRM')
ner.add_label('GREETINGS')
ner.add_label('LOC-Q')
ner.add_label('WHQ')
ner.add_label('PERSON')
# get names of other pipes to disable them during training
other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']
with nlp.disable_pipes(*other_pipes): # only train NER
optimizer = nlp.begin_training()
for itn in range(n_iter):
random.shuffle(TRAIN_DATA)
losses = {}
batches = minibatch(TRAIN_DATA, size=compounding(4., 32., 1.001))
for batch in batches:
texts, annotations = zip(*batch)
nlp.update(
texts, # batch of texts
annotations, # batch of annotations
drop=0.5, # dropout - make it harder to memorise data
sgd=optimizer, # callable to update weights
losses=losses)
print('Losses', losses)
# test the trained model
for text, _ in TRAIN_DATA:
doc = nlp(text)
print('Entities', [(ent.text, ent.label_) for ent in doc.ents])
print('Tokens', [(t.text, t.ent_type_, t.ent_iob) for t in doc])
# save model to output directory
if output_dir is not None:
output_dir = Path(output_dir)
if not output_dir.exists():
output_dir.mkdir()
nlp.to_disk(output_dir)
print("Saved model to", output_dir)
# test the saved model
print("Loading from", output_dir)
nlp2 = spacy.load(output_dir)
for text, _ in TRAIN_DATA:
doc = nlp2(text)
print('Entities-Test', [(ent.text, ent.label_) for ent in doc.ents])
print('Tokens-Test', [(t.text, t.ent_type_, t.ent_iob) for t in doc])
if __name__ == '__main__':
plac.call(main)
是否可以使用Matcher或PatternMatcher完成此操作？
答案 0 :(得分：0)
基本上，您的错误是您致电nlp.begin_training()。
这是用随机权重初始化模型，这意味着开始一个新模型。这就是为什么您的模型会忘记所有内容的原因。
实际上，您根本不需要该命令。您可以在不指定sgd的情况下调用nlp.update()，并且将使用默认的优化器（Adam）。
我在没有nlp.begin_training()的情况下尝试了您的代码，并按预期工作。请确认这可以。
其他一些注意事项：
您需要更多数据才能添加新的实体类型。您应该期望在几千到一百万之间，这取决于将其与其他现有实体区分开来的困难程度
训练新的实体类型时，请不要忘记为已经训练的实体类型添加标签。如果不这样做，最终会遇到一个“遗忘问题”。
希望它会有所帮助：）